From ef59cbe08aabe9844159cb4192d6345e4e3caab2 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Mon, 30 Jan 2017 19:03:48 -0800 Subject: [PATCH 01/16] YARN-5951. Changes to allow CapacityScheduler to use configuration store --- .../scheduler/capacity/CapacityScheduler.java | 37 +++++----- .../CapacitySchedulerConfiguration.java | 9 +++ .../conf/CSConfigurationProvider.java | 46 +++++++++++++ .../FileBasedCSConfigurationProvider.java | 67 +++++++++++++++++++ .../scheduler/capacity/conf/package-info.java | 29 ++++++++ .../capacity/TestCapacityScheduler.java | 4 +- 6 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 7f50272240b..e74cbe17a24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; @@ -103,6 +102,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation; @@ -167,6 +168,8 @@ public class CapacityScheduler extends private int maxAssignPerHeartbeat; + private CSConfigurationProvider csConfProvider; + @Override public void setConf(Configuration conf) { yarnConf = conf; @@ -289,7 +292,18 @@ public class CapacityScheduler extends IOException { try { writeLock.lock(); - this.conf = loadCapacitySchedulerConfiguration(configuration); + String confProviderStr = configuration.get( + CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER); + if (confProviderStr.equals( + CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) { + this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext); + } else { + throw new IOException("Invalid CS configuration provider: " + + confProviderStr); + } + this.csConfProvider.init(configuration); + this.conf = this.csConfProvider.loadConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); @@ -399,7 +413,7 @@ public class CapacityScheduler extends writeLock.lock(); Configuration configuration = new Configuration(newConf); CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = loadCapacitySchedulerConfiguration(configuration); + this.conf = csConfProvider.loadConfiguration(configuration); validateConf(this.conf); try { LOG.info("Re-initializing queues..."); @@ -1831,23 +1845,6 @@ public class CapacityScheduler extends return true; } - private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( - Configuration configuration) throws IOException { - try { - InputStream CSInputStream = - this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(configuration, - YarnConfiguration.CS_CONFIGURATION_FILE); - if (CSInputStream != null) { - configuration.addResource(CSInputStream); - return new CapacitySchedulerConfiguration(configuration, false); - } - return new CapacitySchedulerConfiguration(configuration, true); - } catch (Exception e) { - throw new IOException(e); - } - } - private String getDefaultReservationQueueName(String planQueueName) { return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3a519ecf5f1..3821e24a74e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -315,6 +315,15 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur @Private public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + + public static final String CS_CONF_PROVIDER = PREFIX + + "configuration.provider"; + + @Private + public static final String FILE_CS_CONF_PROVIDER = "file"; + + @Private + public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER; AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java new file mode 100644 index 00000000000..c9984ac7378 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import java.io.IOException; + +/** + * Configuration provider for {@link CapacityScheduler}. + */ +public interface CSConfigurationProvider { + + /** + * Initialize the configuration provider with given conf. + * @param conf configuration to initialize with + */ + void init(Configuration conf); + + /** + * Loads capacity scheduler configuration object. + * @param conf initial bootstrap configuration + * @return CS configuration + * @throws IOException if fail to retrieve configuration + */ + CapacitySchedulerConfiguration loadConfiguration(Configuration conf) + throws IOException; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java new file mode 100644 index 00000000000..51c64fac45a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import java.io.IOException; +import java.io.InputStream; + +/** + * {@link CapacityScheduler} configuration provider based on local + * {@code capacity-scheduler.xml} file. + */ +public class FileBasedCSConfigurationProvider implements + CSConfigurationProvider { + + private RMContext rmContext; + + /** + * Construct file based CS configuration provider with given context. + * @param rmContext the RM context + */ + public FileBasedCSConfigurationProvider(RMContext rmContext) { + this.rmContext = rmContext; + } + + @Override + public void init(Configuration conf) {} + + @Override + public CapacitySchedulerConfiguration loadConfiguration(Configuration conf) + throws IOException { + try { + InputStream csInputStream = + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, + YarnConfiguration.CS_CONFIGURATION_FILE); + if (csInputStream != null) { + conf.addResource(csInputStream); + return new CapacitySchedulerConfiguration(conf, false); + } + return new CapacitySchedulerConfiguration(conf, true); + } catch (Exception e) { + throw new IOException(e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java new file mode 100644 index 00000000000..08d0522d318 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package + * org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf + * contains classes related to capacity scheduler configuration management. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 84405192dd1..1dea4eea75f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -241,13 +241,13 @@ public class TestCapacityScheduler { @Test (timeout = 30000) public void testConfValidation() throws Exception { - ResourceScheduler scheduler = new CapacityScheduler(); + CapacityScheduler scheduler = new CapacityScheduler(); scheduler.setRMContext(resourceManager.getRMContext()); Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { - scheduler.reinitialize(conf, mockContext); + scheduler.init(conf); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { From e3579a8c3b1dc58a38859b189973be5a2d23f730 Mon Sep 17 00:00:00 2001 From: Xuan Date: Fri, 24 Feb 2017 15:58:12 -0800 Subject: [PATCH 02/16] YARN-5946: Create YarnConfigurationStore interface and InMemoryConfigurationStore class. Contributed by Jonathan Hung --- .../conf/InMemoryConfigurationStore.java | 86 ++++++++++ .../capacity/conf/YarnConfigurationStore.java | 154 ++++++++++++++++++ .../conf/TestYarnConfigurationStore.java | 70 ++++++++ 3 files changed, 310 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java new file mode 100644 index 00000000000..a208fb9a96c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * A default implementation of {@link YarnConfigurationStore}. Doesn't offer + * persistent configuration storage, just stores the configuration in memory. + */ +public class InMemoryConfigurationStore implements YarnConfigurationStore { + + private Configuration schedConf; + private LinkedList pendingMutations; + private long pendingId; + + @Override + public void initialize(Configuration conf, Configuration schedConf) { + this.schedConf = schedConf; + this.pendingMutations = new LinkedList<>(); + this.pendingId = 0; + } + + @Override + public synchronized long logMutation(LogMutation logMutation) { + logMutation.setId(++pendingId); + pendingMutations.add(logMutation); + return pendingId; + } + + @Override + public synchronized boolean confirmMutation(long id, boolean isValid) { + LogMutation mutation = pendingMutations.poll(); + // If confirmMutation is called out of order, discard mutations until id + // is reached. + while (mutation != null) { + if (mutation.getId() == id) { + if (isValid) { + Map mutations = mutation.getUpdates(); + for (Map.Entry kv : mutations.entrySet()) { + schedConf.set(kv.getKey(), kv.getValue()); + } + } + return true; + } + mutation = pendingMutations.poll(); + } + return false; + } + + @Override + public synchronized Configuration retrieve() { + return schedConf; + } + + @Override + public synchronized List getPendingMutations() { + return pendingMutations; + } + + @Override + public List getConfirmedConfHistory(long fromId) { + // Unimplemented. + return null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java new file mode 100644 index 00000000000..22c0ef8f5ef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; + +import java.util.List; +import java.util.Map; + +/** + * YarnConfigurationStore exposes the methods needed for retrieving and + * persisting {@link CapacityScheduler} configuration via key-value + * using write-ahead logging. When configuration mutation is requested, caller + * should first log it with {@code logMutation}, which persists this pending + * mutation. This mutation is merged to the persisted configuration only after + * {@code confirmMutation} is called. + * + * On startup/recovery, caller should call {@code retrieve} to get all + * confirmed mutations, then get pending mutations which were not confirmed via + * {@code getPendingMutations}, and replay/confirm them via + * {@code confirmMutation} as in the normal case. + */ +public interface YarnConfigurationStore { + + /** + * LogMutation encapsulates the fields needed for configuration mutation + * audit logging and recovery. + */ + class LogMutation { + private Map updates; + private String user; + private long id; + + /** + * Create log mutation prior to logging. + * @param updates key-value configuration updates + * @param user user who requested configuration change + */ + public LogMutation(Map updates, String user) { + this(updates, user, 0); + } + + /** + * Create log mutation for recovery. + * @param updates key-value configuration updates + * @param user user who requested configuration change + * @param id transaction id of configuration change + */ + LogMutation(Map updates, String user, long id) { + this.updates = updates; + this.user = user; + this.id = id; + } + + /** + * Get key-value configuration updates. + * @return map of configuration updates + */ + public Map getUpdates() { + return updates; + } + + /** + * Get user who requested configuration change. + * @return user who requested configuration change + */ + public String getUser() { + return user; + } + + /** + * Get transaction id of this configuration change. + * @return transaction id + */ + public long getId() { + return id; + } + + /** + * Set transaction id of this configuration change. + * @param id transaction id + */ + public void setId(long id) { + this.id = id; + } + } + + /** + * Initialize the configuration store. + * @param conf configuration to initialize store with + * @param schedConf Initial key-value configuration to persist + */ + void initialize(Configuration conf, Configuration schedConf); + + /** + * Logs the configuration change to backing store. Generates an id associated + * with this mutation, sets it in {@code logMutation}, and returns it. + * @param logMutation configuration change to be persisted in write ahead log + * @return id which configuration store associates with this mutation + */ + long logMutation(LogMutation logMutation); + + /** + * Should be called after {@code logMutation}. Gets the pending mutation + * associated with {@code id} and marks the mutation as persisted (no longer + * pending). If isValid is true, merge the mutation with the persisted + * configuration. + * + * If {@code confirmMutation} is called with ids in a different order than + * was returned by {@code logMutation}, the result is implementation + * dependent. + * @param id id of mutation to be confirmed + * @param isValid if true, update persisted configuration with mutation + * associated with {@code id}. + * @return true on success + */ + boolean confirmMutation(long id, boolean isValid); + + /** + * Retrieve the persisted configuration. + * @return configuration as key-value + */ + Configuration retrieve(); + + /** + * Get the list of pending mutations, in the order they were logged. + * @return list of mutations + */ + List getPendingMutations(); + + /** + * Get a list of confirmed configuration mutations starting from a given id. + * @param fromId id from which to start getting mutations, inclusive + * @return list of configuration mutations + */ + List getConfirmedConfHistory(long fromId); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java new file mode 100644 index 00000000000..dff4e774700 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestYarnConfigurationStore { + + private YarnConfigurationStore confStore; + private Configuration schedConf; + + private static final String testUser = "testUser"; + + @Before + public void setUp() { + schedConf = new Configuration(false); + schedConf.set("key1", "val1"); + } + + @Test + public void testInMemoryConfigurationStore() { + confStore = new InMemoryConfigurationStore(); + confStore.initialize(new Configuration(), schedConf); + assertEquals("val1", confStore.retrieve().get("key1")); + + Map update1 = new HashMap<>(); + update1.put("keyUpdate1", "valUpdate1"); + LogMutation mutation1 = new LogMutation(update1, testUser); + long id = confStore.logMutation(mutation1); + assertEquals(1, confStore.getPendingMutations().size()); + confStore.confirmMutation(id, true); + assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1")); + assertEquals(0, confStore.getPendingMutations().size()); + + Map update2 = new HashMap<>(); + update2.put("keyUpdate2", "valUpdate2"); + LogMutation mutation2 = new LogMutation(update2, testUser); + id = confStore.logMutation(mutation2); + assertEquals(1, confStore.getPendingMutations().size()); + confStore.confirmMutation(id, false); + assertNull("Configuration should not be updated", + confStore.retrieve().get("keyUpdate2")); + assertEquals(0, confStore.getPendingMutations().size()); + } +} From ef686635c1aa0cd4bd52ca212a95769f2532aa1a Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Wed, 1 Mar 2017 16:03:01 -0800 Subject: [PATCH 03/16] YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store --- .../hadoop/yarn/conf/YarnConfiguration.java | 6 ++ .../src/main/resources/yarn-default.xml | 12 +++ .../MutableConfigurationProvider.java | 35 +++++++ .../scheduler/capacity/CapacityScheduler.java | 14 ++- .../CapacitySchedulerConfiguration.java | 3 + .../conf/CSConfigurationProvider.java | 3 +- .../conf/MutableCSConfigurationProvider.java | 94 +++++++++++++++++++ .../conf/YarnConfigurationStoreFactory.java | 46 +++++++++ .../TestMutableCSConfigurationProvider.java | 83 ++++++++++++++++ 9 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d99941d4176..92b9697b3bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -674,6 +674,12 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; + public static final String SCHEDULER_CONFIGURATION_STORE_CLASS = + YARN_PREFIX + "scheduler.configuration.store.class"; + public static final String MEMORY_CONFIGURATION_STORE = "memory"; + public static final String DEFAULT_CONFIGURATION_STORE = + MEMORY_CONFIGURATION_STORE; + public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX + "authorization-provider"; private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2bc233065e5..73f4717b0eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3367,4 +3367,16 @@ + + + The type of configuration store to use for storing scheduler + configurations, if using a mutable configuration provider. + Keywords such as "memory" map to certain configuration store + implementations. If keyword is not found, try to load this + value as a class. + + yarn.scheduler.configuration.store.class + memory + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java new file mode 100644 index 00000000000..da30a2bf88a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import java.util.Map; + +/** + * Interface for allowing changing scheduler configurations. + */ +public interface MutableConfigurationProvider { + + /** + * Update the scheduler configuration with the provided key value pairs. + * @param user User issuing the request + * @param confUpdate Key-value pairs for configurations to be updated. + */ + void mutateConfiguration(String user, Map confUpdate); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index e74cbe17a24..11f2f6ebaad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.MutableCSConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation; @@ -295,10 +296,15 @@ public class CapacityScheduler extends String confProviderStr = configuration.get( CapacitySchedulerConfiguration.CS_CONF_PROVIDER, CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER); - if (confProviderStr.equals( - CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) { - this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext); - } else { + switch (confProviderStr) { + case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER: + this.csConfProvider = + new FileBasedCSConfigurationProvider(rmContext); + break; + case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER: + this.csConfProvider = new MutableCSConfigurationProvider(rmContext); + break; + default: throw new IOException("Invalid CS configuration provider: " + confProviderStr); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3821e24a74e..40cb893b6d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -322,6 +322,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur @Private public static final String FILE_CS_CONF_PROVIDER = "file"; + @Private + public static final String STORE_CS_CONF_PROVIDER = "store"; + @Private public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java index c9984ac7378..0d2c8bb0040 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java @@ -32,8 +32,9 @@ public interface CSConfigurationProvider { /** * Initialize the configuration provider with given conf. * @param conf configuration to initialize with + * @throws IOException if initialization fails due to misconfiguration */ - void init(Configuration conf); + void init(Configuration conf) throws IOException; /** * Loads capacity scheduler configuration object. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java new file mode 100644 index 00000000000..267ab6a35cc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; + +import java.io.IOException; +import java.util.Map; + +/** + * CS configuration provider which implements + * {@link MutableConfigurationProvider} for modifying capacity scheduler + * configuration. + */ +public class MutableCSConfigurationProvider implements CSConfigurationProvider, + MutableConfigurationProvider { + + private Configuration schedConf; + private YarnConfigurationStore confStore; + private RMContext rmContext; + private Configuration conf; + + public MutableCSConfigurationProvider(RMContext rmContext) { + this.rmContext = rmContext; + } + + @Override + public void init(Configuration config) throws IOException { + String store = config.get( + YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.DEFAULT_CONFIGURATION_STORE); + switch (store) { + case YarnConfiguration.MEMORY_CONFIGURATION_STORE: + this.confStore = new InMemoryConfigurationStore(); + break; + default: + this.confStore = YarnConfigurationStoreFactory.getStore(config); + break; + } + Configuration initialSchedConf = new Configuration(false); + initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); + this.schedConf = initialSchedConf; + confStore.initialize(config, initialSchedConf); + this.conf = config; + } + + @Override + public CapacitySchedulerConfiguration loadConfiguration(Configuration + configuration) throws IOException { + Configuration loadedConf = new Configuration(configuration); + loadedConf.addResource(schedConf); + return new CapacitySchedulerConfiguration(loadedConf, false); + } + + @Override + public void mutateConfiguration(String user, + Map confUpdate) { + Configuration oldConf = new Configuration(schedConf); + LogMutation log = new LogMutation(confUpdate, user); + long id = confStore.logMutation(log); + for (Map.Entry kv : confUpdate.entrySet()) { + schedConf.set(kv.getKey(), kv.getValue()); + } + try { + rmContext.getScheduler().reinitialize(conf, rmContext); + } catch (IOException e) { + schedConf = oldConf; + confStore.confirmMutation(id, false); + return; + } + confStore.confirmMutation(id, true); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java new file mode 100644 index 00000000000..60249c849f9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * Factory class for creating instances of {@link YarnConfigurationStore}. + */ +public final class YarnConfigurationStoreFactory { + + private static final Log LOG = LogFactory.getLog( + YarnConfigurationStoreFactory.class); + + private YarnConfigurationStoreFactory() { + // Unused. + } + + public static YarnConfigurationStore getStore(Configuration conf) { + Class storeClass = + conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + InMemoryConfigurationStore.class, YarnConfigurationStore.class); + LOG.info("Using YarnConfigurationStore implementation - " + storeClass); + return ReflectionUtils.newInstance(storeClass, conf); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java new file mode 100644 index 00000000000..3f103b1bd87 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests {@link MutableCSConfigurationProvider}. + */ +public class TestMutableCSConfigurationProvider { + + private MutableCSConfigurationProvider confProvider; + private RMContext rmContext; + private Map goodUpdate; + private Map badUpdate; + private CapacityScheduler cs; + + private static final String TEST_USER = "testUser"; + + @Before + public void setUp() { + cs = mock(CapacityScheduler.class); + rmContext = mock(RMContext.class); + when(rmContext.getScheduler()).thenReturn(cs); + confProvider = new MutableCSConfigurationProvider(rmContext); + goodUpdate = new HashMap<>(); + goodUpdate.put("goodKey", "goodVal"); + badUpdate = new HashMap<>(); + badUpdate.put("badKey", "badVal"); + } + + @Test + public void testInMemoryBackedProvider() throws IOException { + Configuration conf = new Configuration(); + confProvider.init(conf); + assertNull(confProvider.loadConfiguration(conf) + .get("goodKey")); + + doNothing().when(cs).reinitialize(any(Configuration.class), + any(RMContext.class)); + confProvider.mutateConfiguration(TEST_USER, goodUpdate); + assertEquals("goodVal", confProvider.loadConfiguration(conf) + .get("goodKey")); + + assertNull(confProvider.loadConfiguration(conf).get("badKey")); + doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class), + any(RMContext.class)); + confProvider.mutateConfiguration(TEST_USER, badUpdate); + assertNull(confProvider.loadConfiguration(conf).get("badKey")); + } +} From d8a5136b1f3761166a08969eda7a7f66d1578d7e Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 3 Apr 2017 10:12:01 -0700 Subject: [PATCH 04/16] YARN-5952. Create REST API for changing YARN scheduler configurations. (Jonathan Hung via wangda) --- .../scheduler/MutableConfScheduler.java | 40 ++ .../MutableConfigurationProvider.java | 5 +- .../scheduler/capacity/CapacityScheduler.java | 16 +- .../conf/InMemoryConfigurationStore.java | 6 +- .../conf/MutableCSConfigurationProvider.java | 24 +- .../resourcemanager/webapp/RMWebServices.java | 172 ++++++- .../webapp/dao/QueueConfigInfo.java | 57 +++ .../webapp/dao/QueueConfigsUpdateInfo.java | 60 +++ .../TestMutableCSConfigurationProvider.java | 6 +- ...estRMWebServicesConfigurationMutation.java | 477 ++++++++++++++++++ 10 files changed, 851 insertions(+), 12 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java new file mode 100644 index 00000000000..35e36e1c23b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.IOException; +import java.util.Map; + +/** + * Interface for a scheduler that supports changing configuration at runtime. + * + */ +public interface MutableConfScheduler extends ResourceScheduler { + + /** + * Update the scheduler's configuration. + * @param user Caller of this update + * @param confUpdate key-value map of the configuration update + * @throws IOException if update is invalid + */ + void updateConfiguration(UserGroupInformation user, + Map confUpdate) throws IOException; + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index da30a2bf88a..889c3bc1f03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import java.io.IOException; import java.util.Map; /** @@ -29,7 +30,9 @@ public interface MutableConfigurationProvider { * Update the scheduler configuration with the provided key value pairs. * @param user User issuing the request * @param confUpdate Key-value pairs for configurations to be updated. + * @throws IOException if scheduler could not be reinitialized */ - void mutateConfiguration(String user, Map confUpdate); + void mutateConfiguration(String user, Map confUpdate) + throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 11f2f6ebaad..7c5839bf310 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -86,6 +86,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; @@ -150,7 +152,7 @@ import com.google.common.util.concurrent.SettableFuture; public class CapacityScheduler extends AbstractYarnScheduler implements PreemptableResourceScheduler, CapacitySchedulerContext, Configurable, - ResourceAllocationCommitter { + ResourceAllocationCommitter, MutableConfScheduler { private static final Log LOG = LogFactory.getLog(CapacityScheduler.class); @@ -2610,4 +2612,16 @@ public class CapacityScheduler extends // In seconds return ((LeafQueue) queue).getMaximumApplicationLifetime(); } + + @Override + public void updateConfiguration(UserGroupInformation user, + Map confUpdate) throws IOException { + if (csConfProvider instanceof MutableConfigurationProvider) { + ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( + user.getShortUserName(), confUpdate); + } else { + throw new UnsupportedOperationException("Configured CS configuration " + + "provider does not support updating configuration."); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java index a208fb9a96c..b97be1bd4c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -58,7 +58,11 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore { if (isValid) { Map mutations = mutation.getUpdates(); for (Map.Entry kv : mutations.entrySet()) { - schedConf.set(kv.getKey(), kv.getValue()); + if (kv.getValue() == null) { + schedConf.unset(kv.getKey()); + } else { + schedConf.set(kv.getKey(), kv.getValue()); + } } } return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 267ab6a35cc..ea1b3c070f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -60,34 +60,44 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } Configuration initialSchedConf = new Configuration(false); initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); - this.schedConf = initialSchedConf; - confStore.initialize(config, initialSchedConf); + this.schedConf = new Configuration(false); + // We need to explicitly set the key-values in schedConf, otherwise + // these configuration keys cannot be deleted when + // configuration is reloaded. + for (Map.Entry kv : initialSchedConf) { + schedConf.set(kv.getKey(), kv.getValue()); + } + confStore.initialize(config, schedConf); this.conf = config; } @Override public CapacitySchedulerConfiguration loadConfiguration(Configuration configuration) throws IOException { - Configuration loadedConf = new Configuration(configuration); - loadedConf.addResource(schedConf); + Configuration loadedConf = new Configuration(schedConf); + loadedConf.addResource(configuration); return new CapacitySchedulerConfiguration(loadedConf, false); } @Override public void mutateConfiguration(String user, - Map confUpdate) { + Map confUpdate) throws IOException { Configuration oldConf = new Configuration(schedConf); LogMutation log = new LogMutation(confUpdate, user); long id = confStore.logMutation(log); for (Map.Entry kv : confUpdate.entrySet()) { - schedConf.set(kv.getKey(), kv.getValue()); + if (kv.getValue() == null) { + schedConf.unset(kv.getKey()); + } else { + schedConf.set(kv.getKey(), kv.getValue()); + } } try { rmContext.getScheduler().reinitialize(conf, rmContext); } catch (IOException e) { schedConf = oldConf; confStore.confirmMutation(id, false); - return; + throw e; } confStore.confirmMutation(id, true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 6dc3d9ab094..215e5117cd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -55,7 +55,8 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; - +import com.google.common.base.Joiner; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -134,11 +135,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; @@ -2454,4 +2458,170 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { GetContainersRequest request) throws YarnException, IOException { return rm.getClientRMService().getContainers(request).getContainerList(); } + + @PUT + @Path("/queues") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo + mutationInfo, @Context HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + ApplicationACLsManager aclsManager = rm.getApplicationACLsManager(); + if (aclsManager.areACLsEnabled()) { + if (callerUGI == null || !aclsManager.isAdmin(callerUGI)) { + String msg = "Only admins can carry out this operation."; + throw new ForbiddenException(msg); + } + } + + ResourceScheduler scheduler = rm.getResourceScheduler(); + if (scheduler instanceof MutableConfScheduler) { + try { + callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, YarnException { + Map confUpdate = + constructKeyValueConfUpdate(mutationInfo); + ((CapacityScheduler) scheduler).updateConfiguration(callerUGI, + confUpdate); + return null; + } + }); + } catch (IOException e) { + return Response.status(Status.BAD_REQUEST).entity(e.getMessage()) + .build(); + } + return Response.status(Status.OK).entity("Configuration change " + + "successfully applied.").build(); + } else { + return Response.status(Status.BAD_REQUEST) + .entity("Configuration change only supported by CapacityScheduler.") + .build(); + } + } + + private Map constructKeyValueConfUpdate( + QueueConfigsUpdateInfo mutationInfo) throws IOException { + CapacitySchedulerConfiguration currentConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + CapacitySchedulerConfiguration proposedConf = + new CapacitySchedulerConfiguration(currentConf, false); + Map confUpdate = new HashMap<>(); + for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) { + removeQueue(queueToRemove, proposedConf, confUpdate); + } + for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) { + addQueue(addQueueInfo, proposedConf, confUpdate); + } + for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) { + updateQueue(updateQueueInfo, proposedConf, confUpdate); + } + return confUpdate; + } + + private void removeQueue( + String queueToRemove, CapacitySchedulerConfiguration proposedConf, + Map confUpdate) throws IOException { + if (queueToRemove == null) { + return; + } else { + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + String queueName = queueToRemove.substring( + queueToRemove.lastIndexOf('.') + 1); + CSQueue queue = cs.getQueue(queueName); + if (queue == null || + !queue.getQueuePath().equals(queueToRemove)) { + throw new IOException("Queue " + queueToRemove + " not found"); + } else if (queueToRemove.lastIndexOf('.') == -1) { + throw new IOException("Can't remove queue " + queueToRemove); + } + String parentQueuePath = queueToRemove.substring(0, queueToRemove + .lastIndexOf('.')); + String[] siblingQueues = proposedConf.getQueues(parentQueuePath); + List newSiblingQueues = new ArrayList<>(); + for (String siblingQueue : siblingQueues) { + if (!siblingQueue.equals(queueName)) { + newSiblingQueues.add(siblingQueue); + } + } + proposedConf.setQueues(parentQueuePath, newSiblingQueues + .toArray(new String[0])); + String queuesConfig = CapacitySchedulerConfiguration.PREFIX + + parentQueuePath + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.QUEUES; + if (newSiblingQueues.size() == 0) { + confUpdate.put(queuesConfig, null); + } else { + confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues)); + } + for (Map.Entry confRemove : proposedConf.getValByRegex( + ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*") + .entrySet()) { + proposedConf.unset(confRemove.getKey()); + confUpdate.put(confRemove.getKey(), null); + } + } + } + + private void addQueue( + QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf, + Map confUpdate) throws IOException { + if (addInfo == null) { + return; + } else { + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + String queuePath = addInfo.getQueue(); + String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1); + if (cs.getQueue(queueName) != null) { + throw new IOException("Can't add existing queue " + queuePath); + } else if (queuePath.lastIndexOf('.') == -1) { + throw new IOException("Can't add invalid queue " + queuePath); + } + String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); + String[] siblings = proposedConf.getQueues(parentQueue); + List siblingQueues = siblings == null ? new ArrayList<>() : + new ArrayList<>(Arrays.asList(siblings)); + siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1)); + proposedConf.setQueues(parentQueue, + siblingQueues.toArray(new String[0])); + confUpdate.put(CapacitySchedulerConfiguration.PREFIX + + parentQueue + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.QUEUES, + Joiner.on(',').join(siblingQueues)); + String keyPrefix = CapacitySchedulerConfiguration.PREFIX + + queuePath + CapacitySchedulerConfiguration.DOT; + for (Map.Entry kv : addInfo.getParams().entrySet()) { + if (kv.getValue() == null) { + proposedConf.unset(keyPrefix + kv.getKey()); + } else { + proposedConf.set(keyPrefix + kv.getKey(), kv.getValue()); + } + confUpdate.put(keyPrefix + kv.getKey(), kv.getValue()); + } + } + } + + private void updateQueue(QueueConfigInfo updateInfo, + CapacitySchedulerConfiguration proposedConf, + Map confUpdate) { + if (updateInfo == null) { + return; + } else { + String queuePath = updateInfo.getQueue(); + String keyPrefix = CapacitySchedulerConfiguration.PREFIX + + queuePath + CapacitySchedulerConfiguration.DOT; + for (Map.Entry kv : updateInfo.getParams().entrySet()) { + if (kv.getValue() == null) { + proposedConf.unset(keyPrefix + kv.getKey()); + } else { + proposedConf.set(keyPrefix + kv.getKey(), kv.getValue()); + } + confUpdate.put(keyPrefix + kv.getKey(), kv.getValue()); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java new file mode 100644 index 00000000000..b20eda6a136 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Information for adding or updating a queue to scheduler configuration + * for this queue. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueConfigInfo { + + @XmlElement(name = "queueName") + private String queue; + + private HashMap params = new HashMap<>(); + + public QueueConfigInfo() { } + + public QueueConfigInfo(String queue, Map params) { + this.queue = queue; + this.params = new HashMap<>(params); + } + + public String getQueue() { + return this.queue; + } + + public HashMap getParams() { + return this.params; + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java new file mode 100644 index 00000000000..644ec90f279 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Information for making scheduler configuration changes (supports adding, + * removing, or updating a queue). + */ +@XmlRootElement(name = "schedConf") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueConfigsUpdateInfo { + + @XmlElement(name = "add") + private ArrayList addQueueInfo = new ArrayList<>(); + + @XmlElement(name = "remove") + private ArrayList removeQueueInfo = new ArrayList<>(); + + @XmlElement(name = "update") + private ArrayList updateQueueInfo = new ArrayList<>(); + + public QueueConfigsUpdateInfo() { + // JAXB needs this + } + + public ArrayList getAddQueueInfo() { + return addQueueInfo; + } + + public ArrayList getRemoveQueueInfo() { + return removeQueueInfo; + } + + public ArrayList getUpdateQueueInfo() { + return updateQueueInfo; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 3f103b1bd87..254da318934 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -77,7 +77,11 @@ public class TestMutableCSConfigurationProvider { assertNull(confProvider.loadConfiguration(conf).get("badKey")); doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class), any(RMContext.class)); - confProvider.mutateConfiguration(TEST_USER, badUpdate); + try { + confProvider.mutateConfiguration(TEST_USER, badUpdate); + } catch (IOException e) { + // Expected exception. + } assertNull(confProvider.loadConfiguration(conf).get("badKey")); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java new file mode 100644 index 00000000000..d149055a452 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -0,0 +1,477 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.Status; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Test scheduler configuration mutation via REST API. + */ +public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { + + private static final File CONF_FILE = new File(new File("target", + "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE); + private static final File OLD_CONF_FILE = new File(new File("target", + "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp"); + + private static MockRM rm; + private static String userName; + private static CapacitySchedulerConfiguration csConf; + private static YarnConfiguration conf; + + private static class WebServletModule extends ServletModule { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + try { + userName = UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException ioe) { + throw new RuntimeException("Unable to get current user name " + + ioe.getMessage(), ioe); + } + csConf = new CapacitySchedulerConfiguration(new Configuration(false), + false); + setupQueueConfiguration(csConf); + conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName); + try { + if (CONF_FILE.exists()) { + if (!CONF_FILE.renameTo(OLD_CONF_FILE)) { + throw new RuntimeException("Failed to rename conf file"); + } + } + FileOutputStream out = new FileOutputStream(CONF_FILE); + csConf.writeXml(out); + out.close(); + } catch (IOException e) { + throw new RuntimeException("Failed to write XML file", e); + } + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + filter("/*").through(TestRMWebServicesAppsModification + .TestRMCustomAuthFilter.class); + } + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); + } + + private static void setupQueueConfiguration( + CapacitySchedulerConfiguration config) { + config.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[]{"a", "b", "c"}); + + final String a = CapacitySchedulerConfiguration.ROOT + ".a"; + config.setCapacity(a, 25f); + config.setMaximumCapacity(a, 50f); + + final String a1 = a + ".a1"; + final String a2 = a + ".a2"; + config.setQueues(a, new String[]{"a1", "a2"}); + config.setCapacity(a1, 100f); + config.setCapacity(a2, 0f); + + final String b = CapacitySchedulerConfiguration.ROOT + ".b"; + config.setCapacity(b, 75f); + + final String c = CapacitySchedulerConfiguration.ROOT + ".c"; + config.setCapacity(c, 0f); + + final String c1 = c + ".c1"; + config.setQueues(c, new String[] {"c1"}); + config.setCapacity(c1, 0f); + } + + public TestRMWebServicesConfigurationMutation() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testAddNestedQueue() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Add parent queue root.d with two children d1 and d2. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + Map d1Capacity = new HashMap<>(); + d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); + d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25"); + Map nearEmptyCapacity = new HashMap<>(); + nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4"); + nearEmptyCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, + "1E-4"); + Map d2Capacity = new HashMap<>(); + d2Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "75"); + d2Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "75"); + QueueConfigInfo d1 = new QueueConfigInfo("root.d.d1", d1Capacity); + QueueConfigInfo d2 = new QueueConfigInfo("root.d.d2", d2Capacity); + QueueConfigInfo d = new QueueConfigInfo("root.d", nearEmptyCapacity); + updateInfo.getAddQueueInfo().add(d1); + updateInfo.getAddQueueInfo().add(d2); + updateInfo.getAddQueueInfo().add(d); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(4, newCSConf.getQueues("root").length); + assertEquals(2, newCSConf.getQueues("root.d").length); + assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"), + 0.01f); + assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"), + 0.01f); + } + + @Test + public void testAddWithUpdate() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Add root.d with capacity 25, reducing root.b capacity from 75 to 50. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + Map dCapacity = new HashMap<>(); + dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); + Map bCapacity = new HashMap<>(); + bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "50"); + QueueConfigInfo d = new QueueConfigInfo("root.d", dCapacity); + QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity); + updateInfo.getAddQueueInfo().add(d); + updateInfo.getUpdateQueueInfo().add(b); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(4, newCSConf.getQueues("root").length); + assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f); + assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); + } + + @Test + public void testRemoveQueue() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + stopQueue("root.a.a2"); + // Remove root.a.a2 + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + updateInfo.getRemoveQueueInfo().add("root.a.a2"); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(1, newCSConf.getQueues("root.a").length); + assertEquals("a1", newCSConf.getQueues("root.a")[0]); + } + + @Test + public void testRemoveParentQueue() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + stopQueue("root.c", "root.c.c1"); + // Remove root.c (parent queue) + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + updateInfo.getRemoveQueueInfo().add("root.c"); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(2, newCSConf.getQueues("root").length); + assertNull(newCSConf.getQueues("root.c")); + } + + @Test + public void testRemoveParentQueueWithCapacity() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + stopQueue("root.a", "root.a.a1", "root.a.a2"); + // Remove root.a (parent queue) with capacity 25 + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + updateInfo.getRemoveQueueInfo().add("root.a"); + + // Set root.b capacity to 100 + Map bCapacity = new HashMap<>(); + bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100"); + QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity); + updateInfo.getUpdateQueueInfo().add(b); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(2, newCSConf.getQueues("root").length); + assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), + 0.01f); + } + + @Test + public void testRemoveMultipleQueues() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + stopQueue("root.b", "root.c", "root.c.c1"); + // Remove root.b and root.c + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + updateInfo.getRemoveQueueInfo().add("root.b"); + updateInfo.getRemoveQueueInfo().add("root.c"); + Map aCapacity = new HashMap<>(); + aCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100"); + aCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "100"); + QueueConfigInfo configInfo = new QueueConfigInfo("root.a", aCapacity); + updateInfo.getUpdateQueueInfo().add(configInfo); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(1, newCSConf.getQueues("root").length); + } + + private void stopQueue(String... queuePaths) throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Set state of queues to STOPPED. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + Map stoppedParam = new HashMap<>(); + stoppedParam.put(CapacitySchedulerConfiguration.STATE, + QueueState.STOPPED.toString()); + for (String queue : queuePaths) { + QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue, stoppedParam); + updateInfo.getUpdateQueueInfo().add(stoppedInfo); + } + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + for (String queue : queuePaths) { + assertEquals(QueueState.STOPPED, newCSConf.getState(queue)); + } + } + + @Test + public void testUpdateQueue() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Update config value. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + Map updateParam = new HashMap<>(); + updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX, + "0.2"); + QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam); + updateInfo.getUpdateQueueInfo().add(aUpdateInfo); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + assertEquals(CapacitySchedulerConfiguration + .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, + cs.getConfiguration() + .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), + 0.001f); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = cs.getConfiguration(); + assertEquals(0.2f, newCSConf + .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), 0.001f); + + // Remove config. Config value should be reverted to default. + updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX, + null); + aUpdateInfo = new QueueConfigInfo("root.a", updateParam); + updateInfo.getUpdateQueueInfo().clear(); + updateInfo.getUpdateQueueInfo().add(aUpdateInfo); + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + newCSConf = cs.getConfiguration(); + assertEquals(CapacitySchedulerConfiguration + .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf + .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), + 0.001f); + } + + @Test + public void testUpdateQueueCapacity() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Update root.a and root.b capacity to 50. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + Map updateParam = new HashMap<>(); + updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50"); + QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam); + QueueConfigInfo bUpdateInfo = new QueueConfigInfo("root.b", updateParam); + updateInfo.getUpdateQueueInfo().add(aUpdateInfo); + updateInfo.getUpdateQueueInfo().add(bUpdateInfo); + + response = + r.path("ws").path("v1").path("cluster") + .path("queues").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f); + assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); + } + + @Override + @After + public void tearDown() throws Exception { + if (rm != null) { + rm.stop(); + } + CONF_FILE.delete(); + if (!OLD_CONF_FILE.renameTo(CONF_FILE)) { + throw new RuntimeException("Failed to re-copy old configuration file"); + } + super.tearDown(); + } + + @SuppressWarnings("rawtypes") + private String toJson(Object nsli, Class klass) throws Exception { + StringWriter sw = new StringWriter(); + JSONJAXBContext ctx = new JSONJAXBContext(klass); + JSONMarshaller jm = ctx.createJSONMarshaller(); + jm.marshallToJSON(nsli, sw); + return sw.toString(); + } +} From a4e62530469e4c3d5b339a06adeac2146fc15fa5 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 22 May 2017 13:38:31 -0700 Subject: [PATCH 05/16] YARN-5949. Add pluggable configuration ACL policy interface and implementation. (Jonathan Hung via wangda) Change-Id: Ib98e82ff753bede21fcab2e6ca9ec1e7a5a2008f --- .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../src/main/resources/yarn-default.xml | 11 ++ .../ConfigurationMutationACLPolicy.java | 47 ++++++ ...ConfigurationMutationACLPolicyFactory.java | 49 ++++++ ...DefaultConfigurationMutationACLPolicy.java | 45 +++++ .../scheduler/MutableConfScheduler.java | 19 ++- .../MutableConfigurationProvider.java | 8 +- .../scheduler/capacity/CapacityScheduler.java | 6 +- .../conf/MutableCSConfigurationProvider.java | 151 ++++++++++++++++- ...ueAdminConfigurationMutationACLPolicy.java | 96 +++++++++++ .../resourcemanager/webapp/RMWebServices.java | 131 +-------------- .../TestConfigurationMutationACLPolicies.java | 154 ++++++++++++++++++ .../TestMutableCSConfigurationProvider.java | 40 +++-- 13 files changed, 610 insertions(+), 150 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 92b9697b3bf..b603135cb8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -680,6 +680,9 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_CONFIGURATION_STORE = MEMORY_CONFIGURATION_STORE; + public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS = + YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class"; + public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX + "authorization-provider"; private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 73f4717b0eb..e124c5b14a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3379,4 +3379,15 @@ memory + + + The class to use for configuration mutation ACL policy if using a mutable + configuration provider. Controls whether a mutation request is allowed. + The DefaultConfigurationMutationACLPolicy checks if the requestor is a + YARN admin. + + yarn.scheduler.configuration.mutation.acl-policy.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java new file mode 100644 index 00000000000..724487b615e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; + +/** + * Interface for determining whether configuration mutations are allowed. + */ +public interface ConfigurationMutationACLPolicy { + + /** + * Initialize ACL policy with configuration and RMContext. + * @param conf Configuration to initialize with. + * @param rmContext rmContext + */ + void init(Configuration conf, RMContext rmContext); + + /** + * Check if mutation is allowed. + * @param user User issuing the request + * @param confUpdate configurations to be updated + * @return whether provided mutation is allowed or not + */ + boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo + confUpdate); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java new file mode 100644 index 00000000000..28987857c0b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * Factory class for creating instances of + * {@link ConfigurationMutationACLPolicy}. + */ +public final class ConfigurationMutationACLPolicyFactory { + + private static final Log LOG = LogFactory.getLog( + ConfigurationMutationACLPolicyFactory.class); + + private ConfigurationMutationACLPolicyFactory() { + // Unused. + } + + public static ConfigurationMutationACLPolicy getPolicy(Configuration conf) { + Class policyClass = + conf.getClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + DefaultConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + LOG.info("Using ConfigurationMutationACLPolicy implementation - " + + policyClass); + return ReflectionUtils.newInstance(policyClass, conf); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java new file mode 100644 index 00000000000..680c3b85f7b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; + +/** + * Default configuration mutation ACL policy. Checks if user is YARN admin. + */ +public class DefaultConfigurationMutationACLPolicy implements + ConfigurationMutationACLPolicy { + + private YarnAuthorizationProvider authorizer; + + @Override + public void init(Configuration conf, RMContext rmContext) { + authorizer = YarnAuthorizationProvider.getInstance(conf); + } + + @Override + public boolean isMutationAllowed(UserGroupInformation user, + QueueConfigsUpdateInfo confUpdate) { + return authorizer.isAdmin(user); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 35e36e1c23b..93a935e33f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -17,10 +17,11 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; import java.io.IOException; -import java.util.Map; /** * Interface for a scheduler that supports changing configuration at runtime. @@ -31,10 +32,22 @@ public interface MutableConfScheduler extends ResourceScheduler { /** * Update the scheduler's configuration. * @param user Caller of this update - * @param confUpdate key-value map of the configuration update + * @param confUpdate configuration update * @throws IOException if update is invalid */ void updateConfiguration(UserGroupInformation user, - Map confUpdate) throws IOException; + QueueConfigsUpdateInfo confUpdate) throws IOException; + /** + * Get the scheduler configuration. + * @return the scheduler configuration + */ + Configuration getConfiguration(); + + /** + * Get queue object based on queue name. + * @param queueName the queue name + * @return the queue object + */ + Queue getQueue(String queueName); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 889c3bc1f03..f04c128f6be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; + import java.io.IOException; -import java.util.Map; /** * Interface for allowing changing scheduler configurations. @@ -32,7 +34,7 @@ public interface MutableConfigurationProvider { * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized */ - void mutateConfiguration(String user, Map confUpdate) - throws IOException; + void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo + confUpdate) throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 7c5839bf310..4f89f7e9d48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -649,6 +650,7 @@ public class CapacityScheduler extends preemptionManager.refreshQueues(null, this.getRootQueue()); } + @Override public CSQueue getQueue(String queueName) { if (queueName == null) { return null; @@ -2615,10 +2617,10 @@ public class CapacityScheduler extends @Override public void updateConfiguration(UserGroupInformation user, - Map confUpdate) throws IOException { + QueueConfigsUpdateInfo confUpdate) throws IOException { if (csConfProvider instanceof MutableConfigurationProvider) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( - user.getShortUserName(), confUpdate); + user, confUpdate); } else { throw new UnsupportedOperationException("Configured CS configuration " + "provider does not support updating configuration."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index ea1b3c070f0..8b879b072bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -18,14 +18,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import com.google.common.base.Joiner; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -38,6 +51,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, private Configuration schedConf; private YarnConfigurationStore confStore; + private ConfigurationMutationACLPolicy aclMutationPolicy; private RMContext rmContext; private Configuration conf; @@ -68,6 +82,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, schedConf.set(kv.getKey(), kv.getValue()); } confStore.initialize(config, schedConf); + this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory + .getPolicy(config); + aclMutationPolicy.init(config, rmContext); this.conf = config; } @@ -80,12 +97,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } @Override - public void mutateConfiguration(String user, - Map confUpdate) throws IOException { + public void mutateConfiguration(UserGroupInformation user, + QueueConfigsUpdateInfo confUpdate) throws IOException { + if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { + throw new AccessControlException("User is not admin of all modified" + + " queues."); + } Configuration oldConf = new Configuration(schedConf); - LogMutation log = new LogMutation(confUpdate, user); + Map kvUpdate = constructKeyValueConfUpdate(confUpdate); + LogMutation log = new LogMutation(kvUpdate, user.getShortUserName()); long id = confStore.logMutation(log); - for (Map.Entry kv : confUpdate.entrySet()) { + for (Map.Entry kv : kvUpdate.entrySet()) { if (kv.getValue() == null) { schedConf.unset(kv.getKey()); } else { @@ -101,4 +123,125 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } confStore.confirmMutation(id, true); } + + + private Map constructKeyValueConfUpdate( + QueueConfigsUpdateInfo mutationInfo) throws IOException { + CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); + CapacitySchedulerConfiguration proposedConf = + new CapacitySchedulerConfiguration(cs.getConfiguration(), false); + Map confUpdate = new HashMap<>(); + for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) { + removeQueue(queueToRemove, proposedConf, confUpdate); + } + for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) { + addQueue(addQueueInfo, proposedConf, confUpdate); + } + for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) { + updateQueue(updateQueueInfo, proposedConf, confUpdate); + } + return confUpdate; + } + + private void removeQueue( + String queueToRemove, CapacitySchedulerConfiguration proposedConf, + Map confUpdate) throws IOException { + if (queueToRemove == null) { + return; + } else { + CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); + String queueName = queueToRemove.substring( + queueToRemove.lastIndexOf('.') + 1); + CSQueue queue = cs.getQueue(queueName); + if (queue == null || + !queue.getQueuePath().equals(queueToRemove)) { + throw new IOException("Queue " + queueToRemove + " not found"); + } else if (queueToRemove.lastIndexOf('.') == -1) { + throw new IOException("Can't remove queue " + queueToRemove); + } + String parentQueuePath = queueToRemove.substring(0, queueToRemove + .lastIndexOf('.')); + String[] siblingQueues = proposedConf.getQueues(parentQueuePath); + List newSiblingQueues = new ArrayList<>(); + for (String siblingQueue : siblingQueues) { + if (!siblingQueue.equals(queueName)) { + newSiblingQueues.add(siblingQueue); + } + } + proposedConf.setQueues(parentQueuePath, newSiblingQueues + .toArray(new String[0])); + String queuesConfig = CapacitySchedulerConfiguration.PREFIX + + parentQueuePath + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.QUEUES; + if (newSiblingQueues.size() == 0) { + confUpdate.put(queuesConfig, null); + } else { + confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues)); + } + for (Map.Entry confRemove : proposedConf.getValByRegex( + ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*") + .entrySet()) { + proposedConf.unset(confRemove.getKey()); + confUpdate.put(confRemove.getKey(), null); + } + } + } + + private void addQueue( + QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf, + Map confUpdate) throws IOException { + if (addInfo == null) { + return; + } else { + CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); + String queuePath = addInfo.getQueue(); + String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1); + if (cs.getQueue(queueName) != null) { + throw new IOException("Can't add existing queue " + queuePath); + } else if (queuePath.lastIndexOf('.') == -1) { + throw new IOException("Can't add invalid queue " + queuePath); + } + String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); + String[] siblings = proposedConf.getQueues(parentQueue); + List siblingQueues = siblings == null ? new ArrayList<>() : + new ArrayList<>(Arrays.asList(siblings)); + siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1)); + proposedConf.setQueues(parentQueue, + siblingQueues.toArray(new String[0])); + confUpdate.put(CapacitySchedulerConfiguration.PREFIX + + parentQueue + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.QUEUES, + Joiner.on(',').join(siblingQueues)); + String keyPrefix = CapacitySchedulerConfiguration.PREFIX + + queuePath + CapacitySchedulerConfiguration.DOT; + for (Map.Entry kv : addInfo.getParams().entrySet()) { + if (kv.getValue() == null) { + proposedConf.unset(keyPrefix + kv.getKey()); + } else { + proposedConf.set(keyPrefix + kv.getKey(), kv.getValue()); + } + confUpdate.put(keyPrefix + kv.getKey(), kv.getValue()); + } + } + } + + private void updateQueue(QueueConfigInfo updateInfo, + CapacitySchedulerConfiguration proposedConf, + Map confUpdate) { + if (updateInfo == null) { + return; + } else { + String queuePath = updateInfo.getQueue(); + String keyPrefix = CapacitySchedulerConfiguration.PREFIX + + queuePath + CapacitySchedulerConfiguration.DOT; + for (Map.Entry kv : updateInfo.getParams().entrySet()) { + if (kv.getValue() == null) { + proposedConf.unset(keyPrefix + kv.getKey()); + } else { + proposedConf.set(keyPrefix + kv.getKey(), kv.getValue()); + } + confUpdate.put(keyPrefix + kv.getKey(), kv.getValue()); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java new file mode 100644 index 00000000000..1f94c1c5d07 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * A configuration mutation ACL policy which checks that user has admin + * privileges on all queues they are changing. + */ +public class QueueAdminConfigurationMutationACLPolicy implements + ConfigurationMutationACLPolicy { + + private RMContext rmContext; + + @Override + public void init(Configuration conf, RMContext context) { + this.rmContext = context; + } + + @Override + public boolean isMutationAllowed(UserGroupInformation user, + QueueConfigsUpdateInfo confUpdate) { + Set queues = new HashSet<>(); + for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) { + queues.add(addQueueInfo.getQueue()); + } + for (String removeQueue : confUpdate.getRemoveQueueInfo()) { + queues.add(removeQueue); + } + for (QueueConfigInfo updateQueueInfo : confUpdate.getUpdateQueueInfo()) { + queues.add(updateQueueInfo.getQueue()); + } + for (String queuePath : queues) { + String queueName = queuePath.lastIndexOf('.') != -1 ? + queuePath.substring(queuePath.lastIndexOf('.') + 1) : queuePath; + QueueInfo queueInfo = null; + try { + queueInfo = rmContext.getScheduler() + .getQueueInfo(queueName, false, false); + } catch (IOException e) { + // Queue is not found, do nothing. + } + String parentPath = queuePath; + // TODO: handle global config change. + while (queueInfo == null) { + // We are adding a queue (whose parent we are possibly also adding). + // Check ACL of lowest parent queue which already exists. + parentPath = parentPath.substring(0, parentPath.lastIndexOf('.')); + String parentName = parentPath.lastIndexOf('.') != -1 ? + parentPath.substring(parentPath.lastIndexOf('.') + 1) : parentPath; + try { + queueInfo = rmContext.getScheduler() + .getQueueInfo(parentName, false, false); + } catch (IOException e) { + // Queue is not found, do nothing. + } + } + Queue queue = ((MutableConfScheduler) rmContext.getScheduler()) + .getQueue(queueInfo.getQueueName()); + if (queue != null && !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, user)) { + return false; + } + } + return true; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 215e5117cd4..d95465b62d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -142,7 +142,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; @@ -2484,10 +2483,8 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { callerUGI.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws IOException, YarnException { - Map confUpdate = - constructKeyValueConfUpdate(mutationInfo); - ((CapacityScheduler) scheduler).updateConfiguration(callerUGI, - confUpdate); + ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI, + mutationInfo); return null; } }); @@ -2499,129 +2496,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { "successfully applied.").build(); } else { return Response.status(Status.BAD_REQUEST) - .entity("Configuration change only supported by CapacityScheduler.") + .entity("Configuration change only supported by " + + "MutableConfScheduler.") .build(); } } - - private Map constructKeyValueConfUpdate( - QueueConfigsUpdateInfo mutationInfo) throws IOException { - CapacitySchedulerConfiguration currentConf = - ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - CapacitySchedulerConfiguration proposedConf = - new CapacitySchedulerConfiguration(currentConf, false); - Map confUpdate = new HashMap<>(); - for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) { - removeQueue(queueToRemove, proposedConf, confUpdate); - } - for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) { - addQueue(addQueueInfo, proposedConf, confUpdate); - } - for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) { - updateQueue(updateQueueInfo, proposedConf, confUpdate); - } - return confUpdate; - } - - private void removeQueue( - String queueToRemove, CapacitySchedulerConfiguration proposedConf, - Map confUpdate) throws IOException { - if (queueToRemove == null) { - return; - } else { - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - String queueName = queueToRemove.substring( - queueToRemove.lastIndexOf('.') + 1); - CSQueue queue = cs.getQueue(queueName); - if (queue == null || - !queue.getQueuePath().equals(queueToRemove)) { - throw new IOException("Queue " + queueToRemove + " not found"); - } else if (queueToRemove.lastIndexOf('.') == -1) { - throw new IOException("Can't remove queue " + queueToRemove); - } - String parentQueuePath = queueToRemove.substring(0, queueToRemove - .lastIndexOf('.')); - String[] siblingQueues = proposedConf.getQueues(parentQueuePath); - List newSiblingQueues = new ArrayList<>(); - for (String siblingQueue : siblingQueues) { - if (!siblingQueue.equals(queueName)) { - newSiblingQueues.add(siblingQueue); - } - } - proposedConf.setQueues(parentQueuePath, newSiblingQueues - .toArray(new String[0])); - String queuesConfig = CapacitySchedulerConfiguration.PREFIX + - parentQueuePath + CapacitySchedulerConfiguration.DOT + - CapacitySchedulerConfiguration.QUEUES; - if (newSiblingQueues.size() == 0) { - confUpdate.put(queuesConfig, null); - } else { - confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues)); - } - for (Map.Entry confRemove : proposedConf.getValByRegex( - ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*") - .entrySet()) { - proposedConf.unset(confRemove.getKey()); - confUpdate.put(confRemove.getKey(), null); - } - } - } - - private void addQueue( - QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf, - Map confUpdate) throws IOException { - if (addInfo == null) { - return; - } else { - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - String queuePath = addInfo.getQueue(); - String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1); - if (cs.getQueue(queueName) != null) { - throw new IOException("Can't add existing queue " + queuePath); - } else if (queuePath.lastIndexOf('.') == -1) { - throw new IOException("Can't add invalid queue " + queuePath); - } - String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); - String[] siblings = proposedConf.getQueues(parentQueue); - List siblingQueues = siblings == null ? new ArrayList<>() : - new ArrayList<>(Arrays.asList(siblings)); - siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1)); - proposedConf.setQueues(parentQueue, - siblingQueues.toArray(new String[0])); - confUpdate.put(CapacitySchedulerConfiguration.PREFIX + - parentQueue + CapacitySchedulerConfiguration.DOT + - CapacitySchedulerConfiguration.QUEUES, - Joiner.on(',').join(siblingQueues)); - String keyPrefix = CapacitySchedulerConfiguration.PREFIX + - queuePath + CapacitySchedulerConfiguration.DOT; - for (Map.Entry kv : addInfo.getParams().entrySet()) { - if (kv.getValue() == null) { - proposedConf.unset(keyPrefix + kv.getKey()); - } else { - proposedConf.set(keyPrefix + kv.getKey(), kv.getValue()); - } - confUpdate.put(keyPrefix + kv.getKey(), kv.getValue()); - } - } - } - - private void updateQueue(QueueConfigInfo updateInfo, - CapacitySchedulerConfiguration proposedConf, - Map confUpdate) { - if (updateInfo == null) { - return; - } else { - String queuePath = updateInfo.getQueue(); - String keyPrefix = CapacitySchedulerConfiguration.PREFIX + - queuePath + CapacitySchedulerConfiguration.DOT; - for (Map.Entry kv : updateInfo.getParams().entrySet()) { - if (kv.getValue() == null) { - proposedConf.unset(keyPrefix + kv.getKey()); - } else { - proposedConf.set(keyPrefix + kv.getKey(), kv.getValue()); - } - confUpdate.put(keyPrefix + kv.getKey(), kv.getValue()); - } - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java new file mode 100644 index 00000000000..4016dcffde2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestConfigurationMutationACLPolicies { + + private ConfigurationMutationACLPolicy policy; + private RMContext rmContext; + private MutableConfScheduler scheduler; + + private static final UserGroupInformation GOOD_USER = UserGroupInformation + .createUserForTesting("goodUser", new String[] {}); + private static final UserGroupInformation BAD_USER = UserGroupInformation + .createUserForTesting("badUser", new String[] {}); + private static final Map EMPTY_MAP = + Collections.emptyMap(); + + @Before + public void setUp() throws IOException { + rmContext = mock(RMContext.class); + scheduler = mock(MutableConfScheduler.class); + when(rmContext.getScheduler()).thenReturn(scheduler); + mockQueue("a", scheduler); + mockQueue("b", scheduler); + mockQueue("b1", scheduler); + } + + private void mockQueue(String queueName, MutableConfScheduler scheduler) + throws IOException { + QueueInfo queueInfo = QueueInfo.newInstance(queueName, 0, 0, 0, null, null, + null, null, null, null, false); + when(scheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean())) + .thenReturn(queueInfo); + Queue queue = mock(Queue.class); + when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(GOOD_USER))) + .thenReturn(true); + when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(BAD_USER))) + .thenReturn(false); + when(scheduler.getQueue(eq(queueName))).thenReturn(queue); + } + @Test + public void testDefaultPolicy() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName()); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + DefaultConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + assertTrue(policy.isMutationAllowed(GOOD_USER, null)); + assertFalse(policy.isMutationAllowed(BAD_USER, null)); + } + + @Test + public void testQueueAdminBasedPolicy() { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + QueueAdminConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP); + updateInfo.getUpdateQueueInfo().add(configInfo); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); + } + + @Test + public void testQueueAdminPolicyAddQueue() { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + QueueAdminConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + // Add root.b.b1. Should check ACL of root.b queue. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP); + updateInfo.getAddQueueInfo().add(configInfo); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); + } + + @Test + public void testQueueAdminPolicyAddNestedQueue() { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + QueueAdminConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + // Add root.b.b1.b11. Should check ACL of root.b queue. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP); + updateInfo.getAddQueueInfo().add(configInfo); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); + } + + @Test + public void testQueueAdminPolicyRemoveQueue() { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + QueueAdminConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + // Remove root.b.b1. + QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + updateInfo.getRemoveQueueInfo().add("root.b.b1"); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 254da318934..13229b10860 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -19,8 +19,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; import org.junit.Before; import org.junit.Test; @@ -43,22 +47,34 @@ public class TestMutableCSConfigurationProvider { private MutableCSConfigurationProvider confProvider; private RMContext rmContext; - private Map goodUpdate; - private Map badUpdate; + private QueueConfigsUpdateInfo goodUpdate; + private QueueConfigsUpdateInfo badUpdate; private CapacityScheduler cs; - private static final String TEST_USER = "testUser"; + private static final UserGroupInformation TEST_USER = UserGroupInformation + .createUserForTesting("testUser", new String[] {}); @Before public void setUp() { cs = mock(CapacityScheduler.class); rmContext = mock(RMContext.class); when(rmContext.getScheduler()).thenReturn(cs); + when(cs.getConfiguration()).thenReturn( + new CapacitySchedulerConfiguration()); confProvider = new MutableCSConfigurationProvider(rmContext); - goodUpdate = new HashMap<>(); - goodUpdate.put("goodKey", "goodVal"); - badUpdate = new HashMap<>(); - badUpdate.put("badKey", "badVal"); + goodUpdate = new QueueConfigsUpdateInfo(); + Map goodUpdateMap = new HashMap<>(); + goodUpdateMap.put("goodKey", "goodVal"); + QueueConfigInfo goodUpdateInfo = new + QueueConfigInfo("root.a", goodUpdateMap); + goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo); + + badUpdate = new QueueConfigsUpdateInfo(); + Map badUpdateMap = new HashMap<>(); + badUpdateMap.put("badKey", "badVal"); + QueueConfigInfo badUpdateInfo = new + QueueConfigInfo("root.a", badUpdateMap); + badUpdate.getUpdateQueueInfo().add(badUpdateInfo); } @Test @@ -66,15 +82,16 @@ public class TestMutableCSConfigurationProvider { Configuration conf = new Configuration(); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) - .get("goodKey")); + .get("yarn.scheduler.capacity.root.a.goodKey")); doNothing().when(cs).reinitialize(any(Configuration.class), any(RMContext.class)); confProvider.mutateConfiguration(TEST_USER, goodUpdate); assertEquals("goodVal", confProvider.loadConfiguration(conf) - .get("goodKey")); + .get("yarn.scheduler.capacity.root.a.goodKey")); - assertNull(confProvider.loadConfiguration(conf).get("badKey")); + assertNull(confProvider.loadConfiguration(conf).get( + "yarn.scheduler.capacity.root.a.badKey")); doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class), any(RMContext.class)); try { @@ -82,6 +99,7 @@ public class TestMutableCSConfigurationProvider { } catch (IOException e) { // Expected exception. } - assertNull(confProvider.loadConfiguration(conf).get("badKey")); + assertNull(confProvider.loadConfiguration(conf).get( + "yarn.scheduler.capacity.root.a.badKey")); } } From e566fd8b58a6570593b21597aa5294d7b37cb388 Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 5 Jun 2017 16:30:38 -0700 Subject: [PATCH 06/16] YARN-6575. Support global configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong) --- .../ConfigurationMutationACLPolicy.java | 4 +- ...DefaultConfigurationMutationACLPolicy.java | 4 +- .../scheduler/MutableConfScheduler.java | 4 +- .../MutableConfigurationProvider.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 4 +- .../conf/MutableCSConfigurationProvider.java | 10 +- ...ueAdminConfigurationMutationACLPolicy.java | 22 +++- .../resourcemanager/webapp/RMWebServices.java | 4 +- ...dateInfo.java => SchedConfUpdateInfo.java} | 21 ++-- .../TestConfigurationMutationACLPolicies.java | 28 ++++- .../TestMutableCSConfigurationProvider.java | 10 +- ...estRMWebServicesConfigurationMutation.java | 101 ++++++++++++------ 12 files changed, 151 insertions(+), 65 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/{QueueConfigsUpdateInfo.java => SchedConfUpdateInfo.java} (77%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java index 724487b615e..3a388feb612 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java @@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; /** * Interface for determining whether configuration mutations are allowed. @@ -41,7 +41,7 @@ public interface ConfigurationMutationACLPolicy { * @param confUpdate configurations to be updated * @return whether provided mutation is allowed or not */ - boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo + boolean isMutationAllowed(UserGroupInformation user, SchedConfUpdateInfo confUpdate); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java index 680c3b85f7b..6648668d9e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java @@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; /** * Default configuration mutation ACL policy. Checks if user is YARN admin. @@ -39,7 +39,7 @@ public class DefaultConfigurationMutationACLPolicy implements @Override public boolean isMutationAllowed(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) { + SchedConfUpdateInfo confUpdate) { return authorizer.isAdmin(user); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 93a935e33f6..027d9441b57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -36,7 +36,7 @@ public interface MutableConfScheduler extends ResourceScheduler { * @throws IOException if update is invalid */ void updateConfiguration(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) throws IOException; + SchedConfUpdateInfo confUpdate) throws IOException; /** * Get the scheduler configuration. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index f04c128f6be..6b8306c80ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -34,7 +34,7 @@ public interface MutableConfigurationProvider { * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized */ - void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo + void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo confUpdate) throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 4f89f7e9d48..4d367e0da15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -137,7 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -2617,7 +2617,7 @@ public class CapacityScheduler extends @Override public void updateConfiguration(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException { if (csConfProvider instanceof MutableConfigurationProvider) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( user, confUpdate); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 8b879b072bf..eb9726078f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; import java.util.ArrayList; @@ -98,7 +98,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, @Override public void mutateConfiguration(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException { if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { throw new AccessControlException("User is not admin of all modified" + " queues."); @@ -126,7 +126,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, private Map constructKeyValueConfUpdate( - QueueConfigsUpdateInfo mutationInfo) throws IOException { + SchedConfUpdateInfo mutationInfo) throws IOException { CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); CapacitySchedulerConfiguration proposedConf = new CapacitySchedulerConfiguration(cs.getConfiguration(), false); @@ -140,6 +140,10 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) { updateQueue(updateQueueInfo, proposedConf, confUpdate); } + for (Map.Entry global : mutationInfo.getGlobalParams() + .entrySet()) { + confUpdate.put(global.getKey(), global.getValue()); + } return confUpdate; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java index 1f94c1c5d07..0a82d50cfe5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java @@ -22,15 +22,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; import java.util.HashSet; +import java.util.Map; import java.util.Set; /** @@ -40,16 +42,29 @@ import java.util.Set; public class QueueAdminConfigurationMutationACLPolicy implements ConfigurationMutationACLPolicy { + private Configuration conf; private RMContext rmContext; + private YarnAuthorizationProvider authorizer; @Override - public void init(Configuration conf, RMContext context) { + public void init(Configuration config, RMContext context) { + this.conf = config; this.rmContext = context; + this.authorizer = YarnAuthorizationProvider.getInstance(conf); } @Override public boolean isMutationAllowed(UserGroupInformation user, - QueueConfigsUpdateInfo confUpdate) { + SchedConfUpdateInfo confUpdate) { + // If there are global config changes, check if user is admin. + Map globalParams = confUpdate.getGlobalParams(); + if (globalParams != null && globalParams.size() != 0) { + if (!authorizer.isAdmin(user)) { + return false; + } + } + + // Check if user is admin of all modified queues. Set queues = new HashSet<>(); for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) { queues.add(addQueueInfo.getQueue()); @@ -71,7 +86,6 @@ public class QueueAdminConfigurationMutationACLPolicy implements // Queue is not found, do nothing. } String parentPath = queuePath; - // TODO: handle global config change. while (queueInfo == null) { // We are adding a queue (whose parent we are possibly also adding). // Check ACL of lowest parent queue which already exists. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d95465b62d9..b97e2125e38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2459,11 +2459,11 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @PUT - @Path("/queues") + @Path("/sched-conf") @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) - public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, @Context HttpServletRequest hsr) throws AuthorizationException, InterruptedException { init(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java similarity index 77% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java index 644ec90f279..b7c585e733c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java @@ -19,30 +19,34 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; +import java.util.HashMap; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; /** * Information for making scheduler configuration changes (supports adding, - * removing, or updating a queue). + * removing, or updating a queue, as well as global scheduler conf changes). */ @XmlRootElement(name = "schedConf") @XmlAccessorType(XmlAccessType.FIELD) -public class QueueConfigsUpdateInfo { +public class SchedConfUpdateInfo { - @XmlElement(name = "add") + @XmlElement(name = "add-queue") private ArrayList addQueueInfo = new ArrayList<>(); - @XmlElement(name = "remove") + @XmlElement(name = "remove-queue") private ArrayList removeQueueInfo = new ArrayList<>(); - @XmlElement(name = "update") + @XmlElement(name = "update-queue") private ArrayList updateQueueInfo = new ArrayList<>(); - public QueueConfigsUpdateInfo() { + private HashMap global = new HashMap<>(); + + public SchedConfUpdateInfo() { // JAXB needs this } @@ -57,4 +61,9 @@ public class QueueConfigsUpdateInfo { public ArrayList getUpdateQueueInfo() { return updateQueueInfo; } + + @XmlElementWrapper(name = "global-updates") + public HashMap getGlobalParams() { + return global; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java index 4016dcffde2..0f5a3d82c54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; @@ -77,6 +77,7 @@ public class TestConfigurationMutationACLPolicies { .thenReturn(false); when(scheduler.getQueue(eq(queueName))).thenReturn(queue); } + @Test public void testDefaultPolicy() { Configuration conf = new Configuration(); @@ -98,7 +99,7 @@ public class TestConfigurationMutationACLPolicies { ConfigurationMutationACLPolicy.class); policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP); updateInfo.getUpdateQueueInfo().add(configInfo); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); @@ -114,7 +115,7 @@ public class TestConfigurationMutationACLPolicies { policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); // Add root.b.b1. Should check ACL of root.b queue. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP); updateInfo.getAddQueueInfo().add(configInfo); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); @@ -130,7 +131,7 @@ public class TestConfigurationMutationACLPolicies { policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); // Add root.b.b1.b11. Should check ACL of root.b queue. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP); updateInfo.getAddQueueInfo().add(configInfo); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); @@ -146,9 +147,26 @@ public class TestConfigurationMutationACLPolicies { policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); policy.init(conf, rmContext); // Remove root.b.b1. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.b.b1"); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); } + + @Test + public void testQueueAdminPolicyGlobal() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName()); + conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS, + QueueAdminConfigurationMutationACLPolicy.class, + ConfigurationMutationACLPolicy.class); + policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf); + policy.init(conf, rmContext); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertTrue(policy.isMutationAllowed(BAD_USER, updateInfo)); + updateInfo.getGlobalParams().put("globalKey", "globalValue"); + assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); + assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 13229b10860..32167815a65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; @@ -47,8 +47,8 @@ public class TestMutableCSConfigurationProvider { private MutableCSConfigurationProvider confProvider; private RMContext rmContext; - private QueueConfigsUpdateInfo goodUpdate; - private QueueConfigsUpdateInfo badUpdate; + private SchedConfUpdateInfo goodUpdate; + private SchedConfUpdateInfo badUpdate; private CapacityScheduler cs; private static final UserGroupInformation TEST_USER = UserGroupInformation @@ -62,14 +62,14 @@ public class TestMutableCSConfigurationProvider { when(cs.getConfiguration()).thenReturn( new CapacitySchedulerConfiguration()); confProvider = new MutableCSConfigurationProvider(rmContext); - goodUpdate = new QueueConfigsUpdateInfo(); + goodUpdate = new SchedConfUpdateInfo(); Map goodUpdateMap = new HashMap<>(); goodUpdateMap.put("goodKey", "goodVal"); QueueConfigInfo goodUpdateInfo = new QueueConfigInfo("root.a", goodUpdateMap); goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo); - badUpdate = new QueueConfigsUpdateInfo(); + badUpdate = new SchedConfUpdateInfo(); Map badUpdateMap = new HashMap<>(); badUpdateMap.put("badKey", "badVal"); QueueConfigInfo badUpdateInfo = new diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index d149055a452..5fbe36fd62c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -162,7 +162,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Add parent queue root.d with two children d1 and d2. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map d1Capacity = new HashMap<>(); d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25"); @@ -181,9 +181,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getAddQueueInfo().add(d); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -205,7 +205,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Add root.d with capacity 25, reducing root.b capacity from 75 to 50. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map dCapacity = new HashMap<>(); dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); Map bCapacity = new HashMap<>(); @@ -216,9 +216,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -238,13 +238,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.a.a2"); // Remove root.a.a2 - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a.a2"); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -263,13 +263,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.c", "root.c.c1"); // Remove root.c (parent queue) - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.c"); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -288,7 +288,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.a", "root.a.a1", "root.a.a2"); // Remove root.a (parent queue) with capacity 25 - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a"); // Set root.b capacity to 100 @@ -298,9 +298,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -320,7 +320,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { stopQueue("root.b", "root.c", "root.c.c1"); // Remove root.b and root.c - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.b"); updateInfo.getRemoveQueueInfo().add("root.c"); Map aCapacity = new HashMap<>(); @@ -330,9 +330,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(configInfo); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -348,7 +348,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Set state of queues to STOPPED. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map stoppedParam = new HashMap<>(); stoppedParam.put(CapacitySchedulerConfiguration.STATE, QueueState.STOPPED.toString()); @@ -358,9 +358,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { } response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -378,7 +378,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Update config value. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map updateParam = new HashMap<>(); updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX, "0.2"); @@ -393,9 +393,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { 0.001f); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -411,9 +411,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(aUpdateInfo); response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -431,7 +431,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { ClientResponse response; // Update root.a and root.b capacity to 50. - QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo(); + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map updateParam = new HashMap<>(); updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50"); QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam); @@ -441,9 +441,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { response = r.path("ws").path("v1").path("cluster") - .path("queues").queryParam("user.name", userName) + .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class), + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -453,6 +453,47 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); } + @Test + public void testGlobalConfChange() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Set maximum-applications to 30000. + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX + + "maximum-applications", "30000"); + + response = + r.path("ws").path("v1").path("cluster") + .path("sched-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(30000, newCSConf.getMaximumSystemApplications()); + + updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX + + "maximum-applications", null); + // Unset maximum-applications. Should be set to default. + response = + r.path("ws").path("v1").path("cluster") + .path("sched-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(updateInfo, SchedConfUpdateInfo.class), + MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(CapacitySchedulerConfiguration + .DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS, + newCSConf.getMaximumSystemApplications()); + } + @Override @After public void tearDown() throws Exception { From 916bdbd6bedbc75000bb74e16c1e2ee692a4912c Mon Sep 17 00:00:00 2001 From: Xuan Date: Fri, 7 Jul 2017 14:16:46 -0700 Subject: [PATCH 07/16] YARN-5953:Create CLI for changing YARN configurations. (Jonathan Hung via xgong) --- hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 + hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 5 + .../hadoop/yarn/client/cli/SchedConfCLI.java | 238 ++++++++++++++++++ .../yarn/client/cli/TestSchedConfCLI.java | 160 ++++++++++++ .../hadoop/yarn/webapp/dao/package-info.java | 27 ++ .../yarn/webapp/util/YarnWebServiceUtils.java | 14 ++ .../ConfigurationMutationACLPolicy.java | 2 +- ...DefaultConfigurationMutationACLPolicy.java | 2 +- .../scheduler/MutableConfScheduler.java | 2 +- .../MutableConfigurationProvider.java | 2 +- .../scheduler/capacity/CapacityScheduler.java | 2 +- .../conf/MutableCSConfigurationProvider.java | 4 +- ...ueAdminConfigurationMutationACLPolicy.java | 4 +- .../resourcemanager/webapp/RMWebServices.java | 1 + .../webapp/dao/QueueConfigInfo.java | 4 +- .../webapp/dao/SchedConfUpdateInfo.java | 18 +- .../TestConfigurationMutationACLPolicies.java | 4 +- .../TestMutableCSConfigurationProvider.java | 4 +- ...estRMWebServicesConfigurationMutation.java | 65 ++--- 19 files changed, 507 insertions(+), 54 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 52138c5065b..00635961144 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -142,6 +142,9 @@ function yarncmd_case HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router' ;; + schedconf) + HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI' + ;; scmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin' ;; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 690badf7e1e..7ec9848ae64 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -295,6 +295,11 @@ goto :eof set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof +:schedconf + set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + @rem This changes %1, %2 etc. Hence those cannot be used after calling this. :make_command_arguments if "%1" == "--config" ( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java new file mode 100644 index 00000000000..e17062e5f17 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -0,0 +1,238 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.client.cli; + +import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.Status; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * CLI for modifying scheduler configuration. + */ +@Public +@Evolving +public class SchedConfCLI extends Configured implements Tool { + + private static final String ADD_QUEUES_OPTION = "addQueues"; + private static final String REMOVE_QUEUES_OPTION = "removeQueues"; + private static final String UPDATE_QUEUES_OPTION = "updateQueues"; + private static final String GLOBAL_OPTIONS = "globalUpdates"; + private static final String HELP_CMD = "help"; + + private static final String CONF_ERR_MSG = "Specify configuration key " + + "value as confKey=confVal."; + + public SchedConfCLI() { + super(new YarnConfiguration()); + } + + public static void main(String[] args) throws Exception { + SchedConfCLI cli = new SchedConfCLI(); + int exitCode = cli.run(args); + System.exit(exitCode); + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + + opts.addOption("add", ADD_QUEUES_OPTION, true, + "Add queues with configurations"); + opts.addOption("remove", REMOVE_QUEUES_OPTION, true, + "Remove queues"); + opts.addOption("update", UPDATE_QUEUES_OPTION, true, + "Update queue configurations"); + opts.addOption("global", GLOBAL_OPTIONS, true, + "Update global scheduler configurations"); + opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); + + int exitCode = -1; + CommandLine parsedCli = null; + try { + parsedCli = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + System.err.println("Missing argument for options"); + printUsage(); + return exitCode; + } + + if (parsedCli.hasOption(HELP_CMD)) { + printUsage(); + return 0; + } + + boolean hasOption = false; + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + try { + if (parsedCli.hasOption(ADD_QUEUES_OPTION)) { + hasOption = true; + addQueues(parsedCli.getOptionValue(ADD_QUEUES_OPTION), updateInfo); + } + if (parsedCli.hasOption(REMOVE_QUEUES_OPTION)) { + hasOption = true; + removeQueues(parsedCli.getOptionValue(REMOVE_QUEUES_OPTION), + updateInfo); + } + if (parsedCli.hasOption(UPDATE_QUEUES_OPTION)) { + hasOption = true; + updateQueues(parsedCli.getOptionValue(UPDATE_QUEUES_OPTION), + updateInfo); + } + if (parsedCli.hasOption(GLOBAL_OPTIONS)) { + hasOption = true; + globalUpdates(parsedCli.getOptionValue(GLOBAL_OPTIONS), updateInfo); + } + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + return -1; + } + + if (!hasOption) { + System.err.println("Invalid Command Usage: "); + printUsage(); + return -1; + } + + Client webServiceClient = Client.create(); + WebResource webResource = webServiceClient.resource(WebAppUtils. + getRMWebAppURLWithScheme(getConf())); + ClientResponse response = webResource.path("ws").path("v1").path("cluster") + .path("sched-conf").accept(MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + if (response != null) { + if (response.getStatus() == Status.OK.getStatusCode()) { + System.out.println("Configuration changed successfully."); + return 0; + } else { + System.err.println("Configuration change unsuccessful: " + + response.getEntity(String.class)); + } + } else { + System.err.println("Configuration change unsuccessful: null response"); + } + return -1; + } + + @VisibleForTesting + void addQueues(String args, SchedConfUpdateInfo updateInfo) { + if (args == null) { + return; + } + ArrayList queueConfigInfos = new ArrayList<>(); + for (String arg : args.split(";")) { + queueConfigInfos.add(getQueueConfigInfo(arg)); + } + updateInfo.setAddQueueInfo(queueConfigInfos); + } + + @VisibleForTesting + void removeQueues(String args, SchedConfUpdateInfo updateInfo) { + if (args == null) { + return; + } + List queuesToRemove = Arrays.asList(args.split(",")); + updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove)); + } + + @VisibleForTesting + void updateQueues(String args, SchedConfUpdateInfo updateInfo) { + if (args == null) { + return; + } + ArrayList queueConfigInfos = new ArrayList<>(); + for (String arg : args.split(";")) { + queueConfigInfos.add(getQueueConfigInfo(arg)); + } + updateInfo.setUpdateQueueInfo(queueConfigInfos); + } + + @VisibleForTesting + void globalUpdates(String args, SchedConfUpdateInfo updateInfo) { + if (args == null) { + return; + } + HashMap globalUpdates = new HashMap<>(); + for (String globalUpdate : args.split(",")) { + putKeyValuePair(globalUpdates, globalUpdate); + } + updateInfo.setGlobalParams(globalUpdates); + } + + private QueueConfigInfo getQueueConfigInfo(String arg) { + String[] queueArgs = arg.split(","); + String queuePath = queueArgs[0]; + Map queueConfigs = new HashMap<>(); + for (int i = 1; i < queueArgs.length; ++i) { + putKeyValuePair(queueConfigs, queueArgs[i]); + } + return new QueueConfigInfo(queuePath, queueConfigs); + } + + private void putKeyValuePair(Map kv, String args) { + String[] argParts = args.split("="); + if (argParts.length == 1) { + if (argParts[0].isEmpty() || !args.contains("=")) { + throw new IllegalArgumentException(CONF_ERR_MSG); + } else { + // key specified, but no value e.g. "confKey=" + kv.put(argParts[0], null); + } + } else if (argParts.length > 2) { + throw new IllegalArgumentException(CONF_ERR_MSG); + } else { + if (argParts[0].isEmpty()) { + throw new IllegalArgumentException(CONF_ERR_MSG); + } + kv.put(argParts[0], argParts[1]); + } + } + + private void printUsage() { + System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1," + + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] " + + "[-remove queueRemovePath1,queueRemovePath2] " + + "[-update queueUpdatePath1,confKey1=confVal1] " + + "[-global globalConfKey1=globalConfVal1," + + "globalConfKey2=globalConfVal2]"); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java new file mode 100644 index 00000000000..d2f063920d9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -0,0 +1,160 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.client.cli; + +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Class for testing {@link SchedConfCLI}. + */ +public class TestSchedConfCLI { + + private ByteArrayOutputStream sysOutStream; + private PrintStream sysOut; + + private ByteArrayOutputStream sysErrStream; + private PrintStream sysErr; + + private SchedConfCLI cli; + + @Before + public void setUp() { + sysOutStream = new ByteArrayOutputStream(); + sysOut = new PrintStream(sysOutStream); + System.setOut(sysOut); + + sysErrStream = new ByteArrayOutputStream(); + sysErr = new PrintStream(sysErrStream); + System.setErr(sysErr); + + cli = new SchedConfCLI(); + } + + @Test(timeout = 10000) + public void testInvalidConf() throws Exception { + // conf pair with no key should be invalid + int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"}); + assertTrue("Should return an error code", exitCode != 0); + assertTrue(sysErrStream.toString().contains("Specify configuration key " + + "value as confKey=confVal.")); + exitCode = cli.run(new String[] {"-update", "root.a,=confVal"}); + assertTrue("Should return an error code", exitCode != 0); + assertTrue(sysErrStream.toString().contains("Specify configuration key " + + "value as confKey=confVal.")); + + exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"}); + assertTrue("Should return an error code", exitCode != 0); + assertTrue(sysErrStream.toString().contains("Specify configuration key " + + "value as confKey=confVal.")); + exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"}); + assertTrue("Should return an error code", exitCode != 0); + assertTrue(sysErrStream.toString().contains("Specify configuration key " + + "value as confKey=confVal.")); + } + + @Test(timeout = 10000) + public void testAddQueues() { + SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); + cli.addQueues("root.a,a1=aVal1,a2=aVal2," + + "a3=", schedUpdateInfo); + QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0); + assertEquals("root.a", addInfo.getQueue()); + Map params = addInfo.getParams(); + assertEquals(3, params.size()); + assertEquals("aVal1", params.get("a1")); + assertEquals("aVal2", params.get("a2")); + assertNull(params.get("a3")); + + schedUpdateInfo = new SchedConfUpdateInfo(); + cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo); + assertEquals(2, schedUpdateInfo.getAddQueueInfo().size()); + QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0); + assertEquals("root.b", bAddInfo.getQueue()); + Map bParams = bAddInfo.getParams(); + assertEquals(1, bParams.size()); + assertEquals("bVal1", bParams.get("b1")); + QueueConfigInfo cAddInfo = schedUpdateInfo.getAddQueueInfo().get(1); + assertEquals("root.c", cAddInfo.getQueue()); + Map cParams = cAddInfo.getParams(); + assertEquals(1, cParams.size()); + assertEquals("cVal1", cParams.get("c1")); + } + + @Test(timeout = 10000) + public void testRemoveQueues() { + SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); + cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo); + List removeInfo = schedUpdateInfo.getRemoveQueueInfo(); + assertEquals(3, removeInfo.size()); + assertEquals("root.a", removeInfo.get(0)); + assertEquals("root.b", removeInfo.get(1)); + assertEquals("root.c.c1", removeInfo.get(2)); + } + + @Test(timeout = 10000) + public void testUpdateQueues() { + SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); + cli.updateQueues("root.a,a1=aVal1,a2=aVal2," + + "a3=", schedUpdateInfo); + QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); + assertEquals("root.a", updateInfo.getQueue()); + Map params = updateInfo.getParams(); + assertEquals(3, params.size()); + assertEquals("aVal1", params.get("a1")); + assertEquals("aVal2", params.get("a2")); + assertNull(params.get("a3")); + + schedUpdateInfo = new SchedConfUpdateInfo(); + cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo); + assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size()); + QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); + assertEquals("root.b", bUpdateInfo.getQueue()); + Map bParams = bUpdateInfo.getParams(); + assertEquals(1, bParams.size()); + assertEquals("bVal1", bParams.get("b1")); + QueueConfigInfo cUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(1); + assertEquals("root.c", cUpdateInfo.getQueue()); + Map cParams = cUpdateInfo.getParams(); + assertEquals(1, cParams.size()); + assertEquals("cVal1", cParams.get("c1")); + } + + @Test(timeout = 10000) + public void testGlobalUpdate() { + SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); + cli.globalUpdates("schedKey1=schedVal1,schedKey2=schedVal2", + schedUpdateInfo); + Map globalInfo = schedUpdateInfo.getGlobalParams(); + assertEquals(2, globalInfo.size()); + assertEquals("schedVal1", globalInfo.get("schedKey1")); + assertEquals("schedVal2", globalInfo.get("schedKey2")); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java new file mode 100644 index 00000000000..aec67627729 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Data structures for scheduler configuration mutation info. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.webapp.dao; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java index 4167e21f51d..1cf1e9717e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java @@ -23,9 +23,14 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; import javax.ws.rs.core.MediaType; + +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; import org.apache.hadoop.conf.Configuration; import org.codehaus.jettison.json.JSONObject; +import java.io.StringWriter; + /** * This class contains several utility function which could be used to generate * Restful calls to RM/NM/AHS. @@ -59,4 +64,13 @@ public final class YarnWebServiceUtils { .get(ClientResponse.class); return response.getEntity(JSONObject.class); } + + @SuppressWarnings("rawtypes") + public static String toJson(Object nsli, Class klass) throws Exception { + StringWriter sw = new StringWriter(); + JSONJAXBContext ctx = new JSONJAXBContext(klass); + JSONMarshaller jm = ctx.createJSONMarshaller(); + jm.marshallToJSON(nsli, sw); + return sw.toString(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java index 3a388feb612..5bc5874ae39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java @@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * Interface for determining whether configuration mutations are allowed. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java index 6648668d9e7..1de6f6b0724 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java @@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * Default configuration mutation ACL policy. Checks if user is YARN admin. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 027d9441b57..007dc29af7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 6b8306c80ce..86be7c378a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 4d367e0da15..da395b72d16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -137,11 +137,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index eb9726078f1..670c0f9573d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; import java.util.ArrayList; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java index 0a82d50cfe5..ee53fd18b2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java @@ -27,8 +27,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; import java.util.HashSet; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index b97e2125e38..1da4e653d58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -199,6 +199,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java index b20eda6a136..d1d91c2be0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +package org.apache.hadoop.yarn.webapp.dao; import java.util.HashMap; import java.util.Map; @@ -54,4 +54,4 @@ public class QueueConfigInfo { return this.params; } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java index b7c585e733c..bb84096a545 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +package org.apache.hadoop.yarn.webapp.dao; import java.util.ArrayList; import java.util.HashMap; @@ -54,16 +54,32 @@ public class SchedConfUpdateInfo { return addQueueInfo; } + public void setAddQueueInfo(ArrayList addQueueInfo) { + this.addQueueInfo = addQueueInfo; + } + public ArrayList getRemoveQueueInfo() { return removeQueueInfo; } + public void setRemoveQueueInfo(ArrayList removeQueueInfo) { + this.removeQueueInfo = removeQueueInfo; + } + public ArrayList getUpdateQueueInfo() { return updateQueueInfo; } + public void setUpdateQueueInfo(ArrayList updateQueueInfo) { + this.updateQueueInfo = updateQueueInfo; + } + @XmlElementWrapper(name = "global-updates") public HashMap getGlobalParams() { return global; } + + public void setGlobalParams(HashMap globalInfo) { + this.global = globalInfo; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java index 0f5a3d82c54..398e90921e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 32167815a65..9104f1635e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -23,8 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 5fbe36fd62c..26ef1b7f7bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -22,8 +22,6 @@ import com.google.inject.Guice; import com.google.inject.servlet.ServletModule; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.json.JSONJAXBContext; -import com.sun.jersey.api.json.JSONMarshaller; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.conf.Configuration; @@ -35,11 +33,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -50,7 +49,6 @@ import javax.ws.rs.core.Response.Status; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.StringWriter; import java.util.HashMap; import java.util.Map; @@ -183,8 +181,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -218,8 +216,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -244,8 +242,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -269,8 +267,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -300,8 +298,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -332,8 +330,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); @@ -360,8 +358,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = @@ -395,8 +393,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = cs.getConfiguration(); @@ -413,8 +411,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); newCSConf = cs.getConfiguration(); @@ -443,8 +441,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = @@ -468,8 +466,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = @@ -483,8 +481,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { r.path("ws").path("v1").path("cluster") .path("sched-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) - .entity(toJson(updateInfo, SchedConfUpdateInfo.class), - MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); newCSConf = @@ -506,13 +504,4 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { } super.tearDown(); } - - @SuppressWarnings("rawtypes") - private String toJson(Object nsli, Class klass) throws Exception { - StringWriter sw = new StringWriter(); - JSONJAXBContext ctx = new JSONJAXBContext(klass); - JSONMarshaller jm = ctx.createJSONMarshaller(); - jm.marshallToJSON(nsli, sw); - return sw.toString(); - } } From 04f9e80bb2662c51c2fc53c2edc10c660fd71a9e Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 31 Jul 2017 11:49:05 -0700 Subject: [PATCH 08/16] YARN-5953 addendum: Move QueueConfigInfo and SchedConfUpdateInfo to package org.apache.hadoop.yarn.webapp.dao --- .../java/org/apache/hadoop/yarn}/webapp/dao/QueueConfigInfo.java | 0 .../org/apache/hadoop/yarn}/webapp/dao/SchedConfUpdateInfo.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn}/webapp/dao/QueueConfigInfo.java (100%) rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager => hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn}/webapp/dao/SchedConfUpdateInfo.java (100%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java From 74ba6ffa0b601badc68366685dcf993a0844520a Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 31 Jul 2017 16:48:40 -0700 Subject: [PATCH 09/16] YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung --- .../hadoop/yarn/conf/YarnConfiguration.java | 13 + .../src/main/resources/yarn-default.xml | 29 ++ .../MutableConfigurationProvider.java | 6 + .../scheduler/capacity/CapacityScheduler.java | 3 + .../conf/LeveldbConfigurationStore.java | 314 ++++++++++++++++++ .../conf/MutableCSConfigurationProvider.java | 38 ++- .../capacity/conf/YarnConfigurationStore.java | 14 +- .../conf/TestYarnConfigurationStore.java | 3 +- 8 files changed, 414 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b603135cb8f..ea8652d8613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration { public static final String SCHEDULER_CONFIGURATION_STORE_CLASS = YARN_PREFIX + "scheduler.configuration.store.class"; public static final String MEMORY_CONFIGURATION_STORE = "memory"; + public static final String LEVELDB_CONFIGURATION_STORE = "leveldb"; public static final String DEFAULT_CONFIGURATION_STORE = MEMORY_CONFIGURATION_STORE; + public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX + + "scheduler.configuration.leveldb-store.path"; + + public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = + YARN_PREFIX + + "scheduler.configuration.leveldb-store.compaction-interval-secs"; + public static final long + DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L; + + public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS = + YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs"; + public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000; public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS = YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e124c5b14a3..5afec1bacd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3390,4 +3390,33 @@ org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy + + + The storage path for LevelDB implementation of configuration store, + when yarn.scheduler.configuration.store.class is configured to be + "leveldb". + + yarn.scheduler.configuration.leveldb-store.path + ${hadoop.tmp.dir}/yarn/system/confstore + + + + + The compaction interval for LevelDB configuration store in secs, + when yarn.scheduler.configuration.store.class is configured to be + "leveldb". Default is one day. + + yarn.scheduler.configuration.leveldb-store.compaction-interval-secs + 86400 + + + + + The max number of configuration change log entries kept in LevelDB config + store, when yarn.scheduler.configuration.store.class is configured to be + "leveldb". Default is 1000. + + yarn.scheduler.configuration.leveldb-store.max-logs + 1000 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 86be7c378a1..1f134677da0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -28,6 +28,12 @@ import java.io.IOException; */ public interface MutableConfigurationProvider { + /** + * Apply transactions which were not committed. + * @throws IOException if recovery fails + */ + void recoverConf() throws IOException; + /** * Update the scheduler configuration with the provided key value pairs. * @param user User issuing the request diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index da395b72d16..6d2de7ed958 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -393,6 +393,9 @@ public class CapacityScheduler extends @Override public void serviceStart() throws Exception { startSchedulerThreads(); + if (this.csConfProvider instanceof MutableConfigurationProvider) { + ((MutableConfigurationProvider) csConfProvider).recoverConf(); + } super.serviceStart(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java new file mode 100644 index 00000000000..15346850c65 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -0,0 +1,314 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.fusesource.leveldbjni.JniDBFactory; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBComparator; +import org.iq80.leveldb.DBException; +import org.iq80.leveldb.DBIterator; +import org.iq80.leveldb.Options; +import org.iq80.leveldb.WriteBatch; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectInputStream; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; + +import static org.fusesource.leveldbjni.JniDBFactory.bytes; + +/** + * A LevelDB implementation of {@link YarnConfigurationStore}. + */ +public class LeveldbConfigurationStore implements YarnConfigurationStore { + + public static final Log LOG = + LogFactory.getLog(LeveldbConfigurationStore.class); + + private static final String DB_NAME = "yarn-conf-store"; + private static final String LOG_PREFIX = "log."; + private static final String LOG_COMMITTED_TXN = "committedTxn"; + + private DB db; + private long txnId = 0; + private long minTxn = 0; + private long maxLogs; + private Configuration conf; + private LinkedList pendingMutations = new LinkedList<>(); + private Timer compactionTimer; + private long compactionIntervalMsec; + + @Override + public void initialize(Configuration config, Configuration schedConf) + throws IOException { + this.conf = config; + try { + this.db = initDatabase(schedConf); + this.txnId = Long.parseLong(new String(db.get(bytes(LOG_COMMITTED_TXN)), + StandardCharsets.UTF_8)); + DBIterator itr = db.iterator(); + itr.seek(bytes(LOG_PREFIX + txnId)); + // Seek to first uncommitted log + itr.next(); + while (itr.hasNext()) { + Map.Entry entry = itr.next(); + if (!new String(entry.getKey(), StandardCharsets.UTF_8) + .startsWith(LOG_PREFIX)) { + break; + } + pendingMutations.add(deserLogMutation(entry.getValue())); + } + // Get the earliest txnId stored in logs + itr.seekToFirst(); + if (itr.hasNext()) { + Map.Entry entry = itr.next(); + byte[] key = entry.getKey(); + String logId = new String(key, StandardCharsets.UTF_8); + if (logId.startsWith(LOG_PREFIX)) { + minTxn = Long.parseLong(logId.substring(logId.indexOf('.') + 1)); + } + } + this.maxLogs = config.getLong( + YarnConfiguration.RM_SCHEDCONF_LEVELDB_MAX_LOGS, + YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS); + this.compactionIntervalMsec = config.getLong( + YarnConfiguration.RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS, + YarnConfiguration + .DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS) * 1000; + startCompactionTimer(); + } catch (Exception e) { + throw new IOException(e); + } + } + + private DB initDatabase(Configuration config) throws Exception { + Path storeRoot = createStorageDir(); + Options options = new Options(); + options.createIfMissing(false); + options.comparator(new DBComparator() { + @Override + public int compare(byte[] key1, byte[] key2) { + String key1Str = new String(key1, StandardCharsets.UTF_8); + String key2Str = new String(key2, StandardCharsets.UTF_8); + int key1Txn = Integer.MAX_VALUE; + int key2Txn = Integer.MAX_VALUE; + if (key1Str.startsWith(LOG_PREFIX)) { + key1Txn = Integer.parseInt(key1Str.substring( + key1Str.indexOf('.') + 1)); + } + if (key2Str.startsWith(LOG_PREFIX)) { + key2Txn = Integer.parseInt(key2Str.substring( + key2Str.indexOf('.') + 1)); + } + // TODO txnId could overflow, in theory + if (key1Txn == Integer.MAX_VALUE && key2Txn == Integer.MAX_VALUE) { + if (key1Str.equals(key2Str) && key1Str.equals(LOG_COMMITTED_TXN)) { + return 0; + } else if (key1Str.equals(LOG_COMMITTED_TXN)) { + return -1; + } else if (key2Str.equals(LOG_COMMITTED_TXN)) { + return 1; + } + return key1Str.compareTo(key2Str); + } + return key1Txn - key2Txn; + } + + @Override + public String name() { + return "logComparator"; + } + + public byte[] findShortestSeparator(byte[] start, byte[] limit) { + return start; + } + + public byte[] findShortSuccessor(byte[] key) { + return key; + } + }); + LOG.info("Using conf database at " + storeRoot); + File dbfile = new File(storeRoot.toString()); + try { + db = JniDBFactory.factory.open(dbfile, options); + } catch (NativeDB.DBException e) { + if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { + LOG.info("Creating conf database at " + dbfile); + options.createIfMissing(true); + try { + db = JniDBFactory.factory.open(dbfile, options); + // Write the initial scheduler configuration + WriteBatch initBatch = db.createWriteBatch(); + for (Map.Entry kv : config) { + initBatch.put(bytes(kv.getKey()), bytes(kv.getValue())); + } + initBatch.put(bytes(LOG_COMMITTED_TXN), bytes("0")); + db.write(initBatch); + } catch (DBException dbErr) { + throw new IOException(dbErr.getMessage(), dbErr); + } + } else { + throw e; + } + } + return db; + } + + private Path createStorageDir() throws IOException { + Path root = getStorageDir(); + FileSystem fs = FileSystem.getLocal(conf); + fs.mkdirs(root, new FsPermission((short) 0700)); + return root; + } + + private Path getStorageDir() throws IOException { + String storePath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_PATH); + if (storePath == null) { + throw new IOException("No store location directory configured in " + + YarnConfiguration.RM_SCHEDCONF_STORE_PATH); + } + return new Path(storePath, DB_NAME); + } + + @Override + public synchronized long logMutation(LogMutation logMutation) + throws IOException { + logMutation.setId(++txnId); + WriteBatch logBatch = db.createWriteBatch(); + logBatch.put(bytes(LOG_PREFIX + txnId), serLogMutation(logMutation)); + if (txnId - minTxn >= maxLogs) { + logBatch.delete(bytes(LOG_PREFIX + minTxn)); + minTxn++; + } + db.write(logBatch); + pendingMutations.add(logMutation); + return txnId; + } + + @Override + public synchronized boolean confirmMutation(long id, boolean isValid) + throws IOException { + WriteBatch updateBatch = db.createWriteBatch(); + if (isValid) { + LogMutation mutation = deserLogMutation(db.get(bytes(LOG_PREFIX + id))); + for (Map.Entry changes : + mutation.getUpdates().entrySet()) { + if (changes.getValue() == null || changes.getValue().isEmpty()) { + updateBatch.delete(bytes(changes.getKey())); + } else { + updateBatch.put(bytes(changes.getKey()), bytes(changes.getValue())); + } + } + } + updateBatch.put(bytes(LOG_COMMITTED_TXN), bytes(String.valueOf(id))); + db.write(updateBatch); + // Assumes logMutation and confirmMutation are done in the same + // synchronized method. For example, + // {@link MutableCSConfigurationProvider#mutateConfiguration( + // UserGroupInformation user, SchedConfUpdateInfo confUpdate)} + pendingMutations.removeFirst(); + return true; + } + + private byte[] serLogMutation(LogMutation mutation) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ObjectOutput oos = new ObjectOutputStream(baos)) { + oos.writeObject(mutation); + oos.flush(); + return baos.toByteArray(); + } + } + private LogMutation deserLogMutation(byte[] mutation) throws IOException { + try (ObjectInput input = new ObjectInputStream( + new ByteArrayInputStream(mutation))) { + return (LogMutation) input.readObject(); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + } + + @Override + public synchronized Configuration retrieve() { + DBIterator itr = db.iterator(); + itr.seek(bytes(LOG_COMMITTED_TXN)); + Configuration config = new Configuration(false); + itr.next(); + while (itr.hasNext()) { + Map.Entry entry = itr.next(); + config.set(new String(entry.getKey(), StandardCharsets.UTF_8), + new String(entry.getValue(), StandardCharsets.UTF_8)); + } + return config; + } + + @Override + public List getPendingMutations() { + return pendingMutations; + } + + @Override + public List getConfirmedConfHistory(long fromId) { + return null; // unimplemented + } + + // TODO below was taken from LeveldbRMStateStore, it can probably be + // refactored + private void startCompactionTimer() { + if (compactionIntervalMsec > 0) { + compactionTimer = new Timer( + this.getClass().getSimpleName() + " compaction timer", true); + compactionTimer.schedule(new CompactionTimerTask(), + compactionIntervalMsec, compactionIntervalMsec); + } + } + + private class CompactionTimerTask extends TimerTask { + @Override + public void run() { + long start = Time.monotonicNow(); + LOG.info("Starting full compaction cycle"); + try { + db.compactRange(null, null); + } catch (DBException e) { + LOG.error("Error compacting database", e); + } + long duration = Time.monotonicNow() - start; + LOG.info("Full compaction cycle completed in " + duration + " msec"); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 670c0f9573d..9ccc146f19a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import com.google.common.base.Joiner; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -49,6 +51,9 @@ import java.util.Map; public class MutableCSConfigurationProvider implements CSConfigurationProvider, MutableConfigurationProvider { + public static final Log LOG = + LogFactory.getLog(MutableCSConfigurationProvider.class); + private Configuration schedConf; private YarnConfigurationStore confStore; private ConfigurationMutationACLPolicy aclMutationPolicy; @@ -68,6 +73,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, case YarnConfiguration.MEMORY_CONFIGURATION_STORE: this.confStore = new InMemoryConfigurationStore(); break; + case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: + this.confStore = new LeveldbConfigurationStore(); + break; default: this.confStore = YarnConfigurationStoreFactory.getStore(config); break; @@ -82,6 +90,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, schedConf.set(kv.getKey(), kv.getValue()); } confStore.initialize(config, schedConf); + // After initializing confStore, the store may already have an existing + // configuration. Use this one. + schedConf = confStore.retrieve(); this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory .getPolicy(config); aclMutationPolicy.init(config, rmContext); @@ -97,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } @Override - public void mutateConfiguration(UserGroupInformation user, + public synchronized void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo confUpdate) throws IOException { if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { throw new AccessControlException("User is not admin of all modified" + @@ -124,6 +135,31 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, confStore.confirmMutation(id, true); } + @Override + public void recoverConf() throws IOException { + List uncommittedLogs = confStore.getPendingMutations(); + Configuration oldConf = new Configuration(schedConf); + for (LogMutation mutation : uncommittedLogs) { + for (Map.Entry kv : mutation.getUpdates().entrySet()) { + if (kv.getValue() == null) { + schedConf.unset(kv.getKey()); + } else { + schedConf.set(kv.getKey(), kv.getValue()); + } + } + try { + rmContext.getScheduler().reinitialize(conf, rmContext); + } catch (IOException e) { + schedConf = oldConf; + confStore.confirmMutation(mutation.getId(), false); + LOG.info("Configuration mutation " + mutation.getId() + + " was rejected", e); + continue; + } + confStore.confirmMutation(mutation.getId(), true); + LOG.info("Configuration mutation " + mutation.getId()+ " was accepted"); + } + } private Map constructKeyValueConfUpdate( SchedConfUpdateInfo mutationInfo) throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 22c0ef8f5ef..065c877eeb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import java.io.IOException; +import java.io.Serializable; import java.util.List; import java.util.Map; @@ -43,7 +45,7 @@ public interface YarnConfigurationStore { * LogMutation encapsulates the fields needed for configuration mutation * audit logging and recovery. */ - class LogMutation { + class LogMutation implements Serializable { private Map updates; private String user; private long id; @@ -106,16 +108,19 @@ public interface YarnConfigurationStore { * Initialize the configuration store. * @param conf configuration to initialize store with * @param schedConf Initial key-value configuration to persist + * @throws IOException if initialization fails */ - void initialize(Configuration conf, Configuration schedConf); + void initialize(Configuration conf, Configuration schedConf) + throws IOException; /** * Logs the configuration change to backing store. Generates an id associated * with this mutation, sets it in {@code logMutation}, and returns it. * @param logMutation configuration change to be persisted in write ahead log * @return id which configuration store associates with this mutation + * @throws IOException if logging fails */ - long logMutation(LogMutation logMutation); + long logMutation(LogMutation logMutation) throws IOException; /** * Should be called after {@code logMutation}. Gets the pending mutation @@ -130,8 +135,9 @@ public interface YarnConfigurationStore { * @param isValid if true, update persisted configuration with mutation * associated with {@code id}. * @return true on success + * @throws IOException if mutation confirmation fails */ - boolean confirmMutation(long id, boolean isValid); + boolean confirmMutation(long id, boolean isValid) throws IOException; /** * Retrieve the persisted configuration. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java index dff4e774700..631ce657e84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar import org.junit.Before; import org.junit.Test; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -43,7 +44,7 @@ public class TestYarnConfigurationStore { } @Test - public void testInMemoryConfigurationStore() { + public void testInMemoryConfigurationStore() throws IOException { confStore = new InMemoryConfigurationStore(); confStore.initialize(new Configuration(), schedConf); assertEquals("val1", confStore.retrieve().get("key1")); From 4c8b208adb8c7639628676387e275856cf0842c9 Mon Sep 17 00:00:00 2001 From: Xuan Date: Tue, 1 Aug 2017 08:48:04 -0700 Subject: [PATCH 10/16] YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung --- .../server/resourcemanager/AdminService.java | 12 +++++++++- .../scheduler/MutableConfScheduler.java | 12 ++++++++-- .../MutableConfigurationProvider.java | 4 +++- .../scheduler/capacity/CapacityScheduler.java | 9 ++++++-- .../conf/MutableCSConfigurationProvider.java | 11 ++++----- .../resourcemanager/TestRMAdminService.java | 23 +++++++++++++++++++ .../TestMutableCSConfigurationProvider.java | 14 ++++++----- 7 files changed, 67 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 3457ae329bf..fd9e849f3c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol; @@ -92,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import com.google.common.annotations.VisibleForTesting; @@ -384,6 +387,12 @@ public class AdminService extends CompositeService implements RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + throw new IOException("Scheduler configuration is mutable. " + + operation + " is not allowed in this scenario."); + } refreshQueues(); RMAuditLogger.logSuccess(user.getShortUserName(), operation, "AdminService"); @@ -393,7 +402,8 @@ public class AdminService extends CompositeService implements } } - private void refreshQueues() throws IOException, YarnException { + @Private + public void refreshQueues() throws IOException, YarnException { rm.getRMContext().getScheduler().reinitialize(getConfig(), this.rm.getRMContext()); // refresh the reservation system diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 007dc29af7d..313bf6aee1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -33,10 +34,11 @@ public interface MutableConfScheduler extends ResourceScheduler { * Update the scheduler's configuration. * @param user Caller of this update * @param confUpdate configuration update - * @throws IOException if update is invalid + * @throws IOException if scheduler could not be reinitialized + * @throws YarnException if reservation system could not be reinitialized */ void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException; + SchedConfUpdateInfo confUpdate) throws IOException, YarnException; /** * Get the scheduler configuration. @@ -50,4 +52,10 @@ public interface MutableConfScheduler extends ResourceScheduler { * @return the queue object */ Queue getQueue(String queueName); + + /** + * Return whether the scheduler configuration is mutable. + * @return whether scheduler configuration is mutable or not. + */ + boolean isConfigurationMutable(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 1f134677da0..9baf1ad1ee2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -39,8 +40,9 @@ public interface MutableConfigurationProvider { * @param user User issuing the request * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized + * @throws YarnException if reservation system could not be reinitialized */ void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo - confUpdate) throws IOException; + confUpdate) throws IOException, YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 6d2de7ed958..51ee6a76e89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2620,8 +2620,8 @@ public class CapacityScheduler extends @Override public void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException { - if (csConfProvider instanceof MutableConfigurationProvider) { + SchedConfUpdateInfo confUpdate) throws IOException, YarnException { + if (isConfigurationMutable()) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( user, confUpdate); } else { @@ -2629,4 +2629,9 @@ public class CapacityScheduler extends "provider does not support updating configuration."); } } + + @Override + public boolean isConfigurationMutable() { + return csConfProvider instanceof MutableConfigurationProvider; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 9ccc146f19a..d03b2e23d27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory; @@ -58,7 +59,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, private YarnConfigurationStore confStore; private ConfigurationMutationACLPolicy aclMutationPolicy; private RMContext rmContext; - private Configuration conf; public MutableCSConfigurationProvider(RMContext rmContext) { this.rmContext = rmContext; @@ -96,7 +96,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory .getPolicy(config); aclMutationPolicy.init(config, rmContext); - this.conf = config; } @Override @@ -109,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, @Override public synchronized void mutateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException, YarnException { if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { throw new AccessControlException("User is not admin of all modified" + " queues."); @@ -126,8 +125,8 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } } try { - rmContext.getScheduler().reinitialize(conf, rmContext); - } catch (IOException e) { + rmContext.getRMAdminService().refreshQueues(); + } catch (IOException | YarnException e) { schedConf = oldConf; confStore.confirmMutation(id, false); throw e; @@ -148,7 +147,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } } try { - rmContext.getScheduler().reinitialize(conf, rmContext); + rmContext.getScheduler().reinitialize(schedConf, rmContext); } catch (IOException e) { schedConf = oldConf; confStore.confirmMutation(mutation.getId(), false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 588f16deefc..620ca58a77c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -197,6 +197,29 @@ public class TestRMAdminService { Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + @Test + public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() { + configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + fail("Expected exception while calling refreshQueues when scheduler" + + " configuration is mutable."); + } catch (Exception ex) { + assertTrue(ex.getMessage().endsWith("Scheduler configuration is " + + "mutable. refreshQueues is not allowed in this scenario.")); + } + } + @Test public void testAdminRefreshNodesWithoutConfiguration() throws IOException, YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 9104f1635e0..635a184e8ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -34,7 +36,6 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -50,6 +51,7 @@ public class TestMutableCSConfigurationProvider { private SchedConfUpdateInfo goodUpdate; private SchedConfUpdateInfo badUpdate; private CapacityScheduler cs; + private AdminService adminService; private static final UserGroupInformation TEST_USER = UserGroupInformation .createUserForTesting("testUser", new String[] {}); @@ -61,6 +63,8 @@ public class TestMutableCSConfigurationProvider { when(rmContext.getScheduler()).thenReturn(cs); when(cs.getConfiguration()).thenReturn( new CapacitySchedulerConfiguration()); + adminService = mock(AdminService.class); + when(rmContext.getRMAdminService()).thenReturn(adminService); confProvider = new MutableCSConfigurationProvider(rmContext); goodUpdate = new SchedConfUpdateInfo(); Map goodUpdateMap = new HashMap<>(); @@ -78,22 +82,20 @@ public class TestMutableCSConfigurationProvider { } @Test - public void testInMemoryBackedProvider() throws IOException { + public void testInMemoryBackedProvider() throws IOException, YarnException { Configuration conf = new Configuration(); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); - doNothing().when(cs).reinitialize(any(Configuration.class), - any(RMContext.class)); + doNothing().when(adminService).refreshQueues(); confProvider.mutateConfiguration(TEST_USER, goodUpdate); assertEquals("goodVal", confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); assertNull(confProvider.loadConfiguration(conf).get( "yarn.scheduler.capacity.root.a.badKey")); - doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class), - any(RMContext.class)); + doThrow(new IOException()).when(adminService).refreshQueues(); try { confProvider.mutateConfiguration(TEST_USER, badUpdate); } catch (IOException e) { From 4d8abd84f40a6124e7b05b34c14b4035309623ef Mon Sep 17 00:00:00 2001 From: Xuan Date: Wed, 23 Aug 2017 11:11:41 -0700 Subject: [PATCH 11/16] YARN-7024: Fix issues on recovery in LevelDB store. Contributed by Jonathan Hung --- .../scheduler/capacity/conf/InMemoryConfigurationStore.java | 2 +- .../scheduler/capacity/conf/LeveldbConfigurationStore.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java index b97be1bd4c8..c63734dfac8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -79,7 +79,7 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore { @Override public synchronized List getPendingMutations() { - return pendingMutations; + return new LinkedList<>(pendingMutations); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index 15346850c65..1280fab931b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -65,6 +65,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { private static final String LOG_COMMITTED_TXN = "committedTxn"; private DB db; + // Txnid for the last transaction logged to the store. private long txnId = 0; private long minTxn = 0; private long maxLogs; @@ -92,6 +93,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { break; } pendingMutations.add(deserLogMutation(entry.getValue())); + txnId++; } // Get the earliest txnId stored in logs itr.seekToFirst(); @@ -278,7 +280,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { @Override public List getPendingMutations() { - return pendingMutations; + return new LinkedList<>(pendingMutations); } @Override From ff39c0de206a4fce1f0e8a416357a7a8261f8634 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 18 Sep 2017 09:53:42 -0700 Subject: [PATCH 12/16] YARN-6840. Implement zookeeper based store for scheduler configuration updates. (Jonathan Hung via wangda) Change-Id: I9debea674fe8c7e4109d4ca136965a1ea4c48bcc --- .../hadoop/yarn/conf/YarnConfiguration.java | 14 +- .../src/main/resources/yarn-default.xml | 15 +- .../server/resourcemanager/AdminService.java | 18 +- .../resourcemanager/ResourceManager.java | 24 +- .../RMStateVersionIncompatibleException.java | 2 +- .../recovery/ZKRMStateStore.java | 5 +- .../scheduler/MutableConfScheduler.java | 22 +- .../MutableConfigurationProvider.java | 36 +- .../scheduler/capacity/CapacityScheduler.java | 24 +- .../conf/InMemoryConfigurationStore.java | 73 ++-- .../conf/LeveldbConfigurationStore.java | 168 +++++----- .../conf/MutableCSConfigurationProvider.java | 150 ++++----- .../capacity/conf/YarnConfigurationStore.java | 132 ++++---- .../capacity/conf/ZKConfigurationStore.java | 235 +++++++++++++ .../resourcemanager/webapp/RMWebServices.java | 26 +- .../conf/ConfigurationStoreBaseTest.java | 90 +++++ .../conf/TestInMemoryConfigurationStore.java | 30 ++ .../TestMutableCSConfigurationProvider.java | 18 +- .../conf/TestYarnConfigurationStore.java | 71 ---- .../conf/TestZKConfigurationStore.java | 312 ++++++++++++++++++ 20 files changed, 1040 insertions(+), 425 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ea8652d8613..8809a7cb1e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -678,6 +678,7 @@ public class YarnConfiguration extends Configuration { YARN_PREFIX + "scheduler.configuration.store.class"; public static final String MEMORY_CONFIGURATION_STORE = "memory"; public static final String LEVELDB_CONFIGURATION_STORE = "leveldb"; + public static final String ZK_CONFIGURATION_STORE = "zk"; public static final String DEFAULT_CONFIGURATION_STORE = MEMORY_CONFIGURATION_STORE; public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX @@ -689,9 +690,16 @@ public class YarnConfiguration extends Configuration { public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L; - public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS = - YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs"; - public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000; + public static final String RM_SCHEDCONF_MAX_LOGS = + YARN_PREFIX + "scheduler.configuration.store.max-logs"; + public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000; + public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000; + + /** Parent znode path under which ZKConfigurationStore will create znodes. */ + public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX + + "scheduler.configuration.zk-store.parent-path"; + public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH = + "/confstore"; public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS = YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 5afec1bacd8..4516edfe33e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3412,11 +3412,20 @@ - The max number of configuration change log entries kept in LevelDB config + The max number of configuration change log entries kept in config store, when yarn.scheduler.configuration.store.class is configured to be - "leveldb". Default is 1000. + "leveldb" or "zk". Default is 1000 for either. - yarn.scheduler.configuration.leveldb-store.max-logs + yarn.scheduler.configuration.store.max-logs 1000 + + + + ZK root node path for configuration store when using zookeeper-based + configuration store. + + yarn.scheduler.configuration.zk-store.parent-path + /confstore + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index fd9e849f3c4..6c0a8541223 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -387,9 +387,7 @@ public class AdminService extends CompositeService implements RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { - ResourceScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof MutableConfScheduler - && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + if (isSchedulerMutable()) { throw new IOException("Scheduler configuration is mutable. " + operation + " is not allowed in this scenario."); } @@ -413,6 +411,12 @@ public class AdminService extends CompositeService implements } } + private boolean isSchedulerMutable() { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + return (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()); + } + @Override public RefreshNodesResponse refreshNodes(RefreshNodesRequest request) throws YarnException, StandbyException { @@ -721,6 +725,14 @@ public class AdminService extends CompositeService implements void refreshAll() throws ServiceFailedException { try { checkAcls("refreshAll"); + if (isSchedulerMutable()) { + try { + ((MutableConfScheduler) rm.getRMContext().getScheduler()) + .getMutableConfProvider().reloadConfigurationFromStore(); + } catch (Exception e) { + throw new IOException("Failed to refresh configuration:", e); + } + } refreshQueues(); refreshNodes(); refreshSuperUserGroupsConfiguration(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index fd95ee5c2ce..d9312f268c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -351,7 +351,7 @@ public class ResourceManager extends CompositeService implements Recoverable { conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED); if (curatorEnabled) { - this.zkManager = createAndStartZKManager(conf); + this.zkManager = getAndStartZKManager(conf); elector = new CuratorBasedElectorService(this); } else { elector = new ActiveStandbyElectorBasedElectorService(this); @@ -360,13 +360,16 @@ public class ResourceManager extends CompositeService implements Recoverable { } /** - * Create and ZooKeeper Curator manager. + * Get ZooKeeper Curator manager, creating and starting if not exists. * @param config Configuration for the ZooKeeper curator. - * @return New ZooKeeper Curator manager. + * @return ZooKeeper Curator manager. * @throws IOException If it cannot create the manager. */ - public ZKCuratorManager createAndStartZKManager(Configuration config) - throws IOException { + public synchronized ZKCuratorManager getAndStartZKManager(Configuration + config) throws IOException { + if (this.zkManager != null) { + return zkManager; + } ZKCuratorManager manager = new ZKCuratorManager(config); // Get authentication @@ -386,15 +389,8 @@ public class ResourceManager extends CompositeService implements Recoverable { } manager.start(authInfos); - return manager; - } - - /** - * Get the ZooKeeper Curator manager. - * @return ZooKeeper Curator manager. - */ - public ZKCuratorManager getZKManager() { - return this.zkManager; + this.zkManager = manager; + return zkManager; } public CuratorFramework getCurator() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java index 135868f1a57..d5fce36efba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java @@ -22,7 +22,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; /** * This exception is thrown by ResourceManager if it's loading an incompatible - * version of state from state store on recovery. + * version of storage on recovery. */ public class RMStateVersionIncompatibleException extends YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index ac67dcd9a86..5bff77f3416 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -327,10 +327,7 @@ public class ZKRMStateStore extends RMStateStore { amrmTokenSecretManagerRoot = getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT); reservationRoot = getNodePath(zkRootNodePath, RESERVATION_SYSTEM_ROOT); - zkManager = resourceManager.getZKManager(); - if (zkManager == null) { - zkManager = resourceManager.createAndStartZKManager(conf); - } + zkManager = resourceManager.getAndStartZKManager(conf); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 313bf6aee1c..6f677fbfb3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -18,11 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; - -import java.io.IOException; /** * Interface for a scheduler that supports changing configuration at runtime. @@ -30,16 +25,6 @@ import java.io.IOException; */ public interface MutableConfScheduler extends ResourceScheduler { - /** - * Update the scheduler's configuration. - * @param user Caller of this update - * @param confUpdate configuration update - * @throws IOException if scheduler could not be reinitialized - * @throws YarnException if reservation system could not be reinitialized - */ - void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException, YarnException; - /** * Get the scheduler configuration. * @return the scheduler configuration @@ -58,4 +43,11 @@ public interface MutableConfScheduler extends ResourceScheduler { * @return whether scheduler configuration is mutable or not. */ boolean isConfigurationMutable(); + + /** + * Get scheduler's configuration provider, so other classes can directly + * call mutation APIs on configuration provider. + * @return scheduler's configuration provider + */ + MutableConfigurationProvider getMutableConfProvider(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 9baf1ad1ee2..f8e8814377a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,30 +19,40 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; -import java.io.IOException; - /** * Interface for allowing changing scheduler configurations. */ public interface MutableConfigurationProvider { /** - * Apply transactions which were not committed. - * @throws IOException if recovery fails + * Get the acl mutation policy for this configuration provider. + * @return The acl mutation policy. */ - void recoverConf() throws IOException; + ConfigurationMutationACLPolicy getAclMutationPolicy(); /** - * Update the scheduler configuration with the provided key value pairs. - * @param user User issuing the request - * @param confUpdate Key-value pairs for configurations to be updated. - * @throws IOException if scheduler could not be reinitialized - * @throws YarnException if reservation system could not be reinitialized + * Called when a new ResourceManager is starting/becomes active. Ensures + * configuration is up-to-date. + * @throws Exception if configuration could not be refreshed from store */ - void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo - confUpdate) throws IOException, YarnException; + void reloadConfigurationFromStore() throws Exception; + /** + * Log user's requested configuration mutation, and applies it in-memory. + * @param user User who requested the change + * @param confUpdate User's requested configuration change + * @throws Exception if logging the mutation fails + */ + void logAndApplyMutation(UserGroupInformation user, SchedConfUpdateInfo + confUpdate) throws Exception; + + /** + * Confirm last logged mutation. + * @param isValid if the last logged mutation is applied to scheduler + * properly. + * @throws Exception if confirming mutation fails + */ + void confirmPendingMutation(boolean isValid) throws Exception; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 51ee6a76e89..16b27c100dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -141,7 +141,6 @@ import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -393,9 +392,6 @@ public class CapacityScheduler extends @Override public void serviceStart() throws Exception { startSchedulerThreads(); - if (this.csConfProvider instanceof MutableConfigurationProvider) { - ((MutableConfigurationProvider) csConfProvider).recoverConf(); - } super.serviceStart(); } @@ -2618,20 +2614,16 @@ public class CapacityScheduler extends return ((LeafQueue) queue).getMaximumApplicationLifetime(); } - @Override - public void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException, YarnException { - if (isConfigurationMutable()) { - ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( - user, confUpdate); - } else { - throw new UnsupportedOperationException("Configured CS configuration " + - "provider does not support updating configuration."); - } - } - @Override public boolean isConfigurationMutable() { return csConfProvider instanceof MutableConfigurationProvider; } + + @Override + public MutableConfigurationProvider getMutableConfProvider() { + if (isConfigurationMutable()) { + return (MutableConfigurationProvider) csConfProvider; + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java index c63734dfac8..d69c23690c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java @@ -19,8 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.records.Version; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -28,48 +29,35 @@ import java.util.Map; * A default implementation of {@link YarnConfigurationStore}. Doesn't offer * persistent configuration storage, just stores the configuration in memory. */ -public class InMemoryConfigurationStore implements YarnConfigurationStore { +public class InMemoryConfigurationStore extends YarnConfigurationStore { private Configuration schedConf; - private LinkedList pendingMutations; - private long pendingId; + private LogMutation pendingMutation; @Override - public void initialize(Configuration conf, Configuration schedConf) { + public void initialize(Configuration conf, Configuration schedConf, + RMContext rmContext) { this.schedConf = schedConf; - this.pendingMutations = new LinkedList<>(); - this.pendingId = 0; } @Override - public synchronized long logMutation(LogMutation logMutation) { - logMutation.setId(++pendingId); - pendingMutations.add(logMutation); - return pendingId; + public void logMutation(LogMutation logMutation) { + pendingMutation = logMutation; } @Override - public synchronized boolean confirmMutation(long id, boolean isValid) { - LogMutation mutation = pendingMutations.poll(); - // If confirmMutation is called out of order, discard mutations until id - // is reached. - while (mutation != null) { - if (mutation.getId() == id) { - if (isValid) { - Map mutations = mutation.getUpdates(); - for (Map.Entry kv : mutations.entrySet()) { - if (kv.getValue() == null) { - schedConf.unset(kv.getKey()); - } else { - schedConf.set(kv.getKey(), kv.getValue()); - } - } + public void confirmMutation(boolean isValid) { + if (isValid) { + for (Map.Entry kv : pendingMutation.getUpdates() + .entrySet()) { + if (kv.getValue() == null) { + schedConf.unset(kv.getKey()); + } else { + schedConf.set(kv.getKey(), kv.getValue()); } - return true; } - mutation = pendingMutations.poll(); } - return false; + pendingMutation = null; } @Override @@ -77,14 +65,31 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore { return schedConf; } - @Override - public synchronized List getPendingMutations() { - return new LinkedList<>(pendingMutations); - } - @Override public List getConfirmedConfHistory(long fromId) { // Unimplemented. return null; } + + @Override + public Version getConfStoreVersion() throws Exception { + // Does nothing. + return null; + } + + @Override + public void storeVersion() throws Exception { + // Does nothing. + } + + @Override + public Version getCurrentVersion() { + // Does nothing. + return null; + } + + @Override + public void checkVersion() { + // Does nothing. (Version is always compatible since it's in memory) + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index 1280fab931b..1b0eb9f1a9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -26,6 +26,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos; +import org.apache.hadoop.yarn.server.records.Version; +import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.fusesource.leveldbjni.JniDBFactory; import org.fusesource.leveldbjni.internal.NativeDB; import org.iq80.leveldb.DB; @@ -55,58 +59,32 @@ import static org.fusesource.leveldbjni.JniDBFactory.bytes; /** * A LevelDB implementation of {@link YarnConfigurationStore}. */ -public class LeveldbConfigurationStore implements YarnConfigurationStore { +public class LeveldbConfigurationStore extends YarnConfigurationStore { public static final Log LOG = LogFactory.getLog(LeveldbConfigurationStore.class); private static final String DB_NAME = "yarn-conf-store"; - private static final String LOG_PREFIX = "log."; - private static final String LOG_COMMITTED_TXN = "committedTxn"; + private static final String LOG_KEY = "log"; + private static final String VERSION_KEY = "version"; private DB db; - // Txnid for the last transaction logged to the store. - private long txnId = 0; - private long minTxn = 0; private long maxLogs; private Configuration conf; - private LinkedList pendingMutations = new LinkedList<>(); + private LogMutation pendingMutation; + private static final Version CURRENT_VERSION_INFO = Version + .newInstance(0, 1); private Timer compactionTimer; private long compactionIntervalMsec; @Override - public void initialize(Configuration config, Configuration schedConf) - throws IOException { + public void initialize(Configuration config, Configuration schedConf, + RMContext rmContext) throws IOException { this.conf = config; try { this.db = initDatabase(schedConf); - this.txnId = Long.parseLong(new String(db.get(bytes(LOG_COMMITTED_TXN)), - StandardCharsets.UTF_8)); - DBIterator itr = db.iterator(); - itr.seek(bytes(LOG_PREFIX + txnId)); - // Seek to first uncommitted log - itr.next(); - while (itr.hasNext()) { - Map.Entry entry = itr.next(); - if (!new String(entry.getKey(), StandardCharsets.UTF_8) - .startsWith(LOG_PREFIX)) { - break; - } - pendingMutations.add(deserLogMutation(entry.getValue())); - txnId++; - } - // Get the earliest txnId stored in logs - itr.seekToFirst(); - if (itr.hasNext()) { - Map.Entry entry = itr.next(); - byte[] key = entry.getKey(); - String logId = new String(key, StandardCharsets.UTF_8); - if (logId.startsWith(LOG_PREFIX)) { - minTxn = Long.parseLong(logId.substring(logId.indexOf('.') + 1)); - } - } this.maxLogs = config.getLong( - YarnConfiguration.RM_SCHEDCONF_LEVELDB_MAX_LOGS, + YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS); this.compactionIntervalMsec = config.getLong( YarnConfiguration.RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS, @@ -127,33 +105,23 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { public int compare(byte[] key1, byte[] key2) { String key1Str = new String(key1, StandardCharsets.UTF_8); String key2Str = new String(key2, StandardCharsets.UTF_8); - int key1Txn = Integer.MAX_VALUE; - int key2Txn = Integer.MAX_VALUE; - if (key1Str.startsWith(LOG_PREFIX)) { - key1Txn = Integer.parseInt(key1Str.substring( - key1Str.indexOf('.') + 1)); + if (key1Str.equals(key2Str)) { + return 0; + } else if (key1Str.equals(VERSION_KEY)) { + return -1; + } else if (key2Str.equals(VERSION_KEY)) { + return 1; + } else if (key1Str.equals(LOG_KEY)) { + return -1; + } else if (key2Str.equals(LOG_KEY)) { + return 1; } - if (key2Str.startsWith(LOG_PREFIX)) { - key2Txn = Integer.parseInt(key2Str.substring( - key2Str.indexOf('.') + 1)); - } - // TODO txnId could overflow, in theory - if (key1Txn == Integer.MAX_VALUE && key2Txn == Integer.MAX_VALUE) { - if (key1Str.equals(key2Str) && key1Str.equals(LOG_COMMITTED_TXN)) { - return 0; - } else if (key1Str.equals(LOG_COMMITTED_TXN)) { - return -1; - } else if (key2Str.equals(LOG_COMMITTED_TXN)) { - return 1; - } - return key1Str.compareTo(key2Str); - } - return key1Txn - key2Txn; + return key1Str.compareTo(key2Str); } @Override public String name() { - return "logComparator"; + return "keyComparator"; } public byte[] findShortestSeparator(byte[] start, byte[] limit) { @@ -164,6 +132,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { return key; } }); + LOG.info("Using conf database at " + storeRoot); File dbfile = new File(storeRoot.toString()); try { @@ -179,7 +148,6 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { for (Map.Entry kv : config) { initBatch.put(bytes(kv.getKey()), bytes(kv.getValue())); } - initBatch.put(bytes(LOG_COMMITTED_TXN), bytes("0")); db.write(initBatch); } catch (DBException dbErr) { throw new IOException(dbErr.getMessage(), dbErr); @@ -208,28 +176,22 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { } @Override - public synchronized long logMutation(LogMutation logMutation) - throws IOException { - logMutation.setId(++txnId); - WriteBatch logBatch = db.createWriteBatch(); - logBatch.put(bytes(LOG_PREFIX + txnId), serLogMutation(logMutation)); - if (txnId - minTxn >= maxLogs) { - logBatch.delete(bytes(LOG_PREFIX + minTxn)); - minTxn++; + public void logMutation(LogMutation logMutation) throws IOException { + LinkedList logs = deserLogMutations(db.get(bytes(LOG_KEY))); + logs.add(logMutation); + if (logs.size() > maxLogs) { + logs.removeFirst(); } - db.write(logBatch); - pendingMutations.add(logMutation); - return txnId; + db.put(bytes(LOG_KEY), serLogMutations(logs)); + pendingMutation = logMutation; } @Override - public synchronized boolean confirmMutation(long id, boolean isValid) - throws IOException { + public void confirmMutation(boolean isValid) throws IOException { WriteBatch updateBatch = db.createWriteBatch(); if (isValid) { - LogMutation mutation = deserLogMutation(db.get(bytes(LOG_PREFIX + id))); for (Map.Entry changes : - mutation.getUpdates().entrySet()) { + pendingMutation.getUpdates().entrySet()) { if (changes.getValue() == null || changes.getValue().isEmpty()) { updateBatch.delete(bytes(changes.getKey())); } else { @@ -237,28 +199,24 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { } } } - updateBatch.put(bytes(LOG_COMMITTED_TXN), bytes(String.valueOf(id))); db.write(updateBatch); - // Assumes logMutation and confirmMutation are done in the same - // synchronized method. For example, - // {@link MutableCSConfigurationProvider#mutateConfiguration( - // UserGroupInformation user, SchedConfUpdateInfo confUpdate)} - pendingMutations.removeFirst(); - return true; + pendingMutation = null; } - private byte[] serLogMutation(LogMutation mutation) throws IOException { + private byte[] serLogMutations(LinkedList mutations) throws + IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ObjectOutput oos = new ObjectOutputStream(baos)) { - oos.writeObject(mutation); + oos.writeObject(mutations); oos.flush(); return baos.toByteArray(); } } - private LogMutation deserLogMutation(byte[] mutation) throws IOException { + private LinkedList deserLogMutations(byte[] mutations) throws + IOException { try (ObjectInput input = new ObjectInputStream( - new ByteArrayInputStream(mutation))) { - return (LogMutation) input.readObject(); + new ByteArrayInputStream(mutations))) { + return (LinkedList) input.readObject(); } catch (ClassNotFoundException e) { throw new IOException(e); } @@ -267,7 +225,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { @Override public synchronized Configuration retrieve() { DBIterator itr = db.iterator(); - itr.seek(bytes(LOG_COMMITTED_TXN)); + itr.seek(bytes(LOG_KEY)); Configuration config = new Configuration(false); itr.next(); while (itr.hasNext()) { @@ -278,11 +236,6 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { return config; } - @Override - public List getPendingMutations() { - return new LinkedList<>(pendingMutations); - } - @Override public List getConfirmedConfHistory(long fromId) { return null; // unimplemented @@ -299,6 +252,39 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore { } } + // TODO: following is taken from LeveldbRMStateStore + @Override + public Version getConfStoreVersion() throws Exception { + Version version = null; + try { + byte[] data = db.get(bytes(VERSION_KEY)); + if (data != null) { + version = new VersionPBImpl(YarnServerCommonProtos.VersionProto + .parseFrom(data)); + } + } catch (DBException e) { + throw new IOException(e); + } + return version; + } + + @Override + public void storeVersion() throws Exception { + String key = VERSION_KEY; + byte[] data = ((VersionPBImpl) CURRENT_VERSION_INFO).getProto() + .toByteArray(); + try { + db.put(bytes(key), data); + } catch (DBException e) { + throw new IOException(e); + } + } + + @Override + public Version getCurrentVersion() { + return CURRENT_VERSION_INFO; + } + private class CompactionTimerTask extends TimerTask { @Override public void run() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index d03b2e23d27..70d18403b51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -18,20 +18,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; @@ -56,6 +53,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, LogFactory.getLog(MutableCSConfigurationProvider.class); private Configuration schedConf; + private Configuration oldConf; private YarnConfigurationStore confStore; private ConfigurationMutationACLPolicy aclMutationPolicy; private RMContext rmContext; @@ -76,6 +74,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: this.confStore = new LeveldbConfigurationStore(); break; + case YarnConfiguration.ZK_CONFIGURATION_STORE: + this.confStore = new ZKConfigurationStore(); + break; default: this.confStore = YarnConfigurationStoreFactory.getStore(config); break; @@ -89,7 +90,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, for (Map.Entry kv : initialSchedConf) { schedConf.set(kv.getKey(), kv.getValue()); } - confStore.initialize(config, schedConf); + try { + confStore.initialize(config, schedConf, rmContext); + } catch (Exception e) { + throw new IOException(e); + } // After initializing confStore, the store may already have an existing // configuration. Use this one. schedConf = confStore.retrieve(); @@ -98,6 +103,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, aclMutationPolicy.init(config, rmContext); } + @VisibleForTesting + public YarnConfigurationStore getConfStore() { + return confStore; + } + @Override public CapacitySchedulerConfiguration loadConfiguration(Configuration configuration) throws IOException { @@ -107,16 +117,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } @Override - public synchronized void mutateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException, YarnException { - if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { - throw new AccessControlException("User is not admin of all modified" + - " queues."); - } - Configuration oldConf = new Configuration(schedConf); + public ConfigurationMutationACLPolicy getAclMutationPolicy() { + return aclMutationPolicy; + } + + @Override + public void logAndApplyMutation(UserGroupInformation user, + SchedConfUpdateInfo confUpdate) throws Exception { + oldConf = new Configuration(schedConf); Map kvUpdate = constructKeyValueConfUpdate(confUpdate); LogMutation log = new LogMutation(kvUpdate, user.getShortUserName()); - long id = confStore.logMutation(log); + confStore.logMutation(log); for (Map.Entry kv : kvUpdate.entrySet()) { if (kv.getValue() == null) { schedConf.unset(kv.getKey()); @@ -124,47 +135,33 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, schedConf.set(kv.getKey(), kv.getValue()); } } - try { - rmContext.getRMAdminService().refreshQueues(); - } catch (IOException | YarnException e) { - schedConf = oldConf; - confStore.confirmMutation(id, false); - throw e; - } - confStore.confirmMutation(id, true); } @Override - public void recoverConf() throws IOException { - List uncommittedLogs = confStore.getPendingMutations(); - Configuration oldConf = new Configuration(schedConf); - for (LogMutation mutation : uncommittedLogs) { - for (Map.Entry kv : mutation.getUpdates().entrySet()) { - if (kv.getValue() == null) { - schedConf.unset(kv.getKey()); - } else { - schedConf.set(kv.getKey(), kv.getValue()); - } - } - try { - rmContext.getScheduler().reinitialize(schedConf, rmContext); - } catch (IOException e) { - schedConf = oldConf; - confStore.confirmMutation(mutation.getId(), false); - LOG.info("Configuration mutation " + mutation.getId() - + " was rejected", e); - continue; - } - confStore.confirmMutation(mutation.getId(), true); - LOG.info("Configuration mutation " + mutation.getId()+ " was accepted"); + public void confirmPendingMutation(boolean isValid) throws Exception { + confStore.confirmMutation(isValid); + if (!isValid) { + schedConf = oldConf; } } + @Override + public void reloadConfigurationFromStore() throws Exception { + schedConf = confStore.retrieve(); + } + + private List getSiblingQueues(String queuePath, Configuration conf) { + String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); + String childQueuesKey = CapacitySchedulerConfiguration.PREFIX + + parentQueue + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.QUEUES; + return new ArrayList<>(conf.getStringCollection(childQueuesKey)); + } + private Map constructKeyValueConfUpdate( SchedConfUpdateInfo mutationInfo) throws IOException { - CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); CapacitySchedulerConfiguration proposedConf = - new CapacitySchedulerConfiguration(cs.getConfiguration(), false); + new CapacitySchedulerConfiguration(schedConf, false); Map confUpdate = new HashMap<>(); for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) { removeQueue(queueToRemove, proposedConf, confUpdate); @@ -188,40 +185,35 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, if (queueToRemove == null) { return; } else { - CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); String queueName = queueToRemove.substring( queueToRemove.lastIndexOf('.') + 1); - CSQueue queue = cs.getQueue(queueName); - if (queue == null || - !queue.getQueuePath().equals(queueToRemove)) { - throw new IOException("Queue " + queueToRemove + " not found"); - } else if (queueToRemove.lastIndexOf('.') == -1) { + if (queueToRemove.lastIndexOf('.') == -1) { throw new IOException("Can't remove queue " + queueToRemove); - } - String parentQueuePath = queueToRemove.substring(0, queueToRemove - .lastIndexOf('.')); - String[] siblingQueues = proposedConf.getQueues(parentQueuePath); - List newSiblingQueues = new ArrayList<>(); - for (String siblingQueue : siblingQueues) { - if (!siblingQueue.equals(queueName)) { - newSiblingQueues.add(siblingQueue); - } - } - proposedConf.setQueues(parentQueuePath, newSiblingQueues - .toArray(new String[0])); - String queuesConfig = CapacitySchedulerConfiguration.PREFIX - + parentQueuePath + CapacitySchedulerConfiguration.DOT - + CapacitySchedulerConfiguration.QUEUES; - if (newSiblingQueues.size() == 0) { - confUpdate.put(queuesConfig, null); } else { - confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues)); - } - for (Map.Entry confRemove : proposedConf.getValByRegex( - ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*") - .entrySet()) { - proposedConf.unset(confRemove.getKey()); - confUpdate.put(confRemove.getKey(), null); + List siblingQueues = getSiblingQueues(queueToRemove, + proposedConf); + if (!siblingQueues.contains(queueName)) { + throw new IOException("Queue " + queueToRemove + " not found"); + } + siblingQueues.remove(queueName); + String parentQueuePath = queueToRemove.substring(0, queueToRemove + .lastIndexOf('.')); + proposedConf.setQueues(parentQueuePath, siblingQueues.toArray( + new String[0])); + String queuesConfig = CapacitySchedulerConfiguration.PREFIX + + parentQueuePath + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.QUEUES; + if (siblingQueues.size() == 0) { + confUpdate.put(queuesConfig, null); + } else { + confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues)); + } + for (Map.Entry confRemove : proposedConf.getValByRegex( + ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*") + .entrySet()) { + proposedConf.unset(confRemove.getKey()); + confUpdate.put(confRemove.getKey(), null); + } } } } @@ -232,13 +224,13 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, if (addInfo == null) { return; } else { - CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler(); String queuePath = addInfo.getQueue(); String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1); - if (cs.getQueue(queueName) != null) { - throw new IOException("Can't add existing queue " + queuePath); - } else if (queuePath.lastIndexOf('.') == -1) { + if (queuePath.lastIndexOf('.') == -1) { throw new IOException("Can't add invalid queue " + queuePath); + } else if (getSiblingQueues(queuePath, proposedConf).contains( + queueName)) { + throw new IOException("Can't add existing queue " + queuePath); } String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); String[] siblings = proposedConf.getQueues(parentQueue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 065c877eeb7..13565356ee0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -18,7 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.records.Version; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateVersionIncompatibleException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import java.io.IOException; @@ -39,36 +44,26 @@ import java.util.Map; * {@code getPendingMutations}, and replay/confirm them via * {@code confirmMutation} as in the normal case. */ -public interface YarnConfigurationStore { +public abstract class YarnConfigurationStore { + public static final Log LOG = + LogFactory.getLog(YarnConfigurationStore.class); /** * LogMutation encapsulates the fields needed for configuration mutation * audit logging and recovery. */ - class LogMutation implements Serializable { + static class LogMutation implements Serializable { private Map updates; private String user; - private long id; /** - * Create log mutation prior to logging. + * Create log mutation. * @param updates key-value configuration updates * @param user user who requested configuration change */ - public LogMutation(Map updates, String user) { - this(updates, user, 0); - } - - /** - * Create log mutation for recovery. - * @param updates key-value configuration updates - * @param user user who requested configuration change - * @param id transaction id of configuration change - */ - LogMutation(Map updates, String user, long id) { + LogMutation(Map updates, String user) { this.updates = updates; this.user = user; - this.id = id; } /** @@ -86,75 +81,92 @@ public interface YarnConfigurationStore { public String getUser() { return user; } - - /** - * Get transaction id of this configuration change. - * @return transaction id - */ - public long getId() { - return id; - } - - /** - * Set transaction id of this configuration change. - * @param id transaction id - */ - public void setId(long id) { - this.id = id; - } } /** - * Initialize the configuration store. + * Initialize the configuration store, with schedConf as the initial + * scheduler configuration. If a persisted store already exists, use the + * scheduler configuration stored there, and ignore schedConf. * @param conf configuration to initialize store with - * @param schedConf Initial key-value configuration to persist + * @param schedConf Initial key-value scheduler configuration to persist. + * @param rmContext RMContext for this configuration store * @throws IOException if initialization fails */ - void initialize(Configuration conf, Configuration schedConf) - throws IOException; + public abstract void initialize(Configuration conf, Configuration schedConf, + RMContext rmContext) throws Exception; /** - * Logs the configuration change to backing store. Generates an id associated - * with this mutation, sets it in {@code logMutation}, and returns it. + * Logs the configuration change to backing store. * @param logMutation configuration change to be persisted in write ahead log - * @return id which configuration store associates with this mutation * @throws IOException if logging fails */ - long logMutation(LogMutation logMutation) throws IOException; + public abstract void logMutation(LogMutation logMutation) throws Exception; /** * Should be called after {@code logMutation}. Gets the pending mutation - * associated with {@code id} and marks the mutation as persisted (no longer - * pending). If isValid is true, merge the mutation with the persisted + * last logged by {@code logMutation} and marks the mutation as persisted (no + * longer pending). If isValid is true, merge the mutation with the persisted * configuration. - * - * If {@code confirmMutation} is called with ids in a different order than - * was returned by {@code logMutation}, the result is implementation - * dependent. - * @param id id of mutation to be confirmed - * @param isValid if true, update persisted configuration with mutation - * associated with {@code id}. - * @return true on success - * @throws IOException if mutation confirmation fails + * @param isValid if true, update persisted configuration with pending + * mutation. + * @throws Exception if mutation confirmation fails */ - boolean confirmMutation(long id, boolean isValid) throws IOException; + public abstract void confirmMutation(boolean isValid) throws Exception; /** * Retrieve the persisted configuration. * @return configuration as key-value */ - Configuration retrieve(); - - /** - * Get the list of pending mutations, in the order they were logged. - * @return list of mutations - */ - List getPendingMutations(); + public abstract Configuration retrieve(); /** * Get a list of confirmed configuration mutations starting from a given id. * @param fromId id from which to start getting mutations, inclusive * @return list of configuration mutations */ - List getConfirmedConfHistory(long fromId); + public abstract List getConfirmedConfHistory(long fromId); + + /** + * Get schema version of persisted conf store, for detecting compatibility + * issues when changing conf store schema. + * @return Schema version currently used by the persisted configuration store. + * @throws Exception On version fetch failure + */ + protected abstract Version getConfStoreVersion() throws Exception; + + /** + * Persist the hard-coded schema version to the conf store. + * @throws Exception On storage failure + */ + protected abstract void storeVersion() throws Exception; + + /** + * Get the hard-coded schema version, for comparison against the schema + * version currently persisted. + * @return Current hard-coded schema version + */ + protected abstract Version getCurrentVersion(); + + public void checkVersion() throws Exception { + // TODO this was taken from RMStateStore. Should probably refactor + Version loadedVersion = getConfStoreVersion(); + LOG.info("Loaded configuration store version info " + loadedVersion); + if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) { + return; + } + // if there is no version info, treat it as CURRENT_VERSION_INFO; + if (loadedVersion == null) { + loadedVersion = getCurrentVersion(); + } + if (loadedVersion.isCompatibleTo(getCurrentVersion())) { + LOG.info("Storing configuration store version info " + + getCurrentVersion()); + storeVersion(); + } else { + throw new RMStateVersionIncompatibleException( + "Expecting configuration store version " + getCurrentVersion() + + ", but loading version " + loadedVersion); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java new file mode 100644 index 00000000000..a0bba8c440d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.curator.ZKCuratorManager; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos; +import org.apache.hadoop.yarn.server.records.Version; +import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.ACL; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * A Zookeeper-based implementation of {@link YarnConfigurationStore}. + */ +public class ZKConfigurationStore extends YarnConfigurationStore { + + public static final Log LOG = + LogFactory.getLog(ZKConfigurationStore.class); + + private long maxLogs; + + @VisibleForTesting + protected static final Version CURRENT_VERSION_INFO = Version + .newInstance(0, 1); + private Configuration conf; + private LogMutation pendingMutation; + + private String znodeParentPath; + + private static final String ZK_VERSION_PATH = "VERSION"; + private static final String LOGS_PATH = "LOGS"; + private static final String CONF_STORE_PATH = "CONF_STORE"; + private static final String FENCING_PATH = "FENCING"; + + private String zkVersionPath; + private String logsPath; + private String confStorePath; + private String fencingNodePath; + + @VisibleForTesting + protected ZKCuratorManager zkManager; + private List zkAcl; + + @Override + public void initialize(Configuration config, Configuration schedConf, + RMContext rmContext) throws Exception { + this.conf = config; + this.maxLogs = conf.getLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, + YarnConfiguration.DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS); + this.znodeParentPath = + conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH, + YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH); + this.zkManager = rmContext.getResourceManager().getAndStartZKManager(conf); + this.zkAcl = ZKCuratorManager.getZKAcls(conf); + + this.zkVersionPath = getNodePath(znodeParentPath, ZK_VERSION_PATH); + this.logsPath = getNodePath(znodeParentPath, LOGS_PATH); + this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH); + this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH); + + zkManager.createRootDirRecursively(znodeParentPath); + zkManager.delete(fencingNodePath); + + if (!zkManager.exists(logsPath)) { + zkManager.create(logsPath); + zkManager.setData(logsPath, + serializeObject(new LinkedList()), -1); + } + + if (!zkManager.exists(confStorePath)) { + zkManager.create(confStorePath); + HashMap mapSchedConf = new HashMap<>(); + for (Map.Entry entry : schedConf) { + mapSchedConf.put(entry.getKey(), entry.getValue()); + } + zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1); + } + } + + @VisibleForTesting + protected LinkedList getLogs() throws Exception { + return (LinkedList) + deserializeObject(zkManager.getData(logsPath)); + } + + // TODO: following version-related code is taken from ZKRMStateStore + @Override + public Version getCurrentVersion() { + return CURRENT_VERSION_INFO; + } + + @Override + public Version getConfStoreVersion() throws Exception { + if (zkManager.exists(zkVersionPath)) { + byte[] data = zkManager.getData(zkVersionPath); + return new VersionPBImpl(YarnServerCommonProtos.VersionProto + .parseFrom(data)); + } + + return null; + } + + @Override + public synchronized void storeVersion() throws Exception { + byte[] data = + ((VersionPBImpl) CURRENT_VERSION_INFO).getProto().toByteArray(); + + if (zkManager.exists(zkVersionPath)) { + zkManager.safeSetData(zkVersionPath, data, -1, zkAcl, fencingNodePath); + } else { + zkManager.safeCreate(zkVersionPath, data, zkAcl, CreateMode.PERSISTENT, + zkAcl, fencingNodePath); + } + } + + @Override + public void logMutation(LogMutation logMutation) throws Exception { + byte[] storedLogs = zkManager.getData(logsPath); + LinkedList logs = new LinkedList<>(); + if (storedLogs != null) { + logs = (LinkedList) deserializeObject(storedLogs); + } + logs.add(logMutation); + if (logs.size() > maxLogs) { + logs.remove(logs.removeFirst()); + } + zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl, + fencingNodePath); + pendingMutation = logMutation; + } + + @Override + public void confirmMutation(boolean isValid) + throws Exception { + if (isValid) { + Configuration storedConfigs = retrieve(); + Map mapConf = new HashMap<>(); + for (Map.Entry storedConf : storedConfigs) { + mapConf.put(storedConf.getKey(), storedConf.getValue()); + } + for (Map.Entry confChange : + pendingMutation.getUpdates().entrySet()) { + if (confChange.getValue() == null || confChange.getValue().isEmpty()) { + mapConf.remove(confChange.getKey()); + } else { + mapConf.put(confChange.getKey(), confChange.getValue()); + } + } + zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1, + zkAcl, fencingNodePath); + } + pendingMutation = null; + } + + @Override + public synchronized Configuration retrieve() { + byte[] serializedSchedConf; + try { + serializedSchedConf = zkManager.getData(confStorePath); + } catch (Exception e) { + LOG.error("Failed to retrieve configuration from zookeeper store", e); + return null; + } + try { + Map map = + (HashMap) deserializeObject(serializedSchedConf); + Configuration c = new Configuration(); + for (Map.Entry e : map.entrySet()) { + c.set(e.getKey(), e.getValue()); + } + return c; + } catch (Exception e) { + LOG.error("Exception while deserializing scheduler configuration " + + "from store", e); + } + return null; + } + + @Override + public List getConfirmedConfHistory(long fromId) { + return null; // unimplemented + } + + private static String getNodePath(String root, String nodeName) { + return ZKCuratorManager.getNodePath(root, nodeName); + } + + private static byte[] serializeObject(Object o) throws Exception { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos);) { + oos.writeObject(o); + oos.flush(); + baos.flush(); + return baos.toByteArray(); + } + } + + private static Object deserializeObject(byte[] bytes) throws Exception { + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais);) { + return ois.readObject(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 1da4e653d58..d264c107d8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -136,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -2464,7 +2465,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) - public Response updateSchedulerConfiguration(SchedConfUpdateInfo + public synchronized Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, @Context HttpServletRequest hsr) throws AuthorizationException, InterruptedException { init(); @@ -2479,17 +2480,32 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } ResourceScheduler scheduler = rm.getResourceScheduler(); - if (scheduler instanceof MutableConfScheduler) { + if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler) + scheduler).isConfigurationMutable()) { try { callerUGI.doAs(new PrivilegedExceptionAction() { @Override - public Void run() throws IOException, YarnException { - ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI, - mutationInfo); + public Void run() throws Exception { + MutableConfigurationProvider provider = ((MutableConfScheduler) + scheduler).getMutableConfProvider(); + if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI, + mutationInfo)) { + throw new org.apache.hadoop.security.AccessControlException("User" + + " is not admin of all modified queues."); + } + provider.logAndApplyMutation(callerUGI, mutationInfo); + try { + rm.getRMContext().getRMAdminService().refreshQueues(); + } catch (IOException | YarnException e) { + provider.confirmPendingMutation(false); + throw e; + } + provider.confirmPendingMutation(true); return null; } }); } catch (IOException e) { + LOG.error("Exception thrown when modifying configuration.", e); return Response.status(Status.BAD_REQUEST).entity(e.getMessage()) .build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java new file mode 100644 index 00000000000..bbe95700f18 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Base class for {@link YarnConfigurationStore} implementations. + */ +public abstract class ConfigurationStoreBaseTest { + + protected YarnConfigurationStore confStore = createConfStore(); + + protected abstract YarnConfigurationStore createConfStore(); + + protected Configuration conf; + protected Configuration schedConf; + protected RMContext rmContext; + + protected static final String TEST_USER = "testUser"; + + @Before + public void setUp() throws Exception { + this.conf = new Configuration(); + this.schedConf = new Configuration(false); + } + + @Test + public void testConfigurationUpdate() throws Exception { + schedConf.set("key1", "val1"); + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val1", confStore.retrieve().get("key1")); + + Map update1 = new HashMap<>(); + update1.put("keyUpdate1", "valUpdate1"); + YarnConfigurationStore.LogMutation mutation1 = + new YarnConfigurationStore.LogMutation(update1, TEST_USER); + confStore.logMutation(mutation1); + confStore.confirmMutation(true); + assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1")); + + Map update2 = new HashMap<>(); + update2.put("keyUpdate2", "valUpdate2"); + YarnConfigurationStore.LogMutation mutation2 = + new YarnConfigurationStore.LogMutation(update2, TEST_USER); + confStore.logMutation(mutation2); + confStore.confirmMutation(false); + assertNull("Configuration should not be updated", + confStore.retrieve().get("keyUpdate2")); + } + + @Test + public void testNullConfigurationUpdate() throws Exception { + schedConf.set("key", "val"); + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + + Map update = new HashMap<>(); + update.put("key", null); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update, TEST_USER); + confStore.logMutation(mutation); + confStore.confirmMutation(true); + assertNull(confStore.retrieve().get("key")); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java new file mode 100644 index 00000000000..c40d16a27cb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +/** + * Tests {@link InMemoryConfigurationStore}. + */ +public class TestInMemoryConfigurationStore extends ConfigurationStoreBaseTest { + + @Override + protected YarnConfigurationStore createConfStore() { + return new InMemoryConfigurationStore(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 635a184e8ff..9b080cd1e2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -30,14 +29,11 @@ import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; import org.junit.Test; -import java.io.IOException; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -82,25 +78,21 @@ public class TestMutableCSConfigurationProvider { } @Test - public void testInMemoryBackedProvider() throws IOException, YarnException { + public void testInMemoryBackedProvider() throws Exception { Configuration conf = new Configuration(); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); - doNothing().when(adminService).refreshQueues(); - confProvider.mutateConfiguration(TEST_USER, goodUpdate); + confProvider.logAndApplyMutation(TEST_USER, goodUpdate); + confProvider.confirmPendingMutation(true); assertEquals("goodVal", confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); assertNull(confProvider.loadConfiguration(conf).get( "yarn.scheduler.capacity.root.a.badKey")); - doThrow(new IOException()).when(adminService).refreshQueues(); - try { - confProvider.mutateConfiguration(TEST_USER, badUpdate); - } catch (IOException e) { - // Expected exception. - } + confProvider.logAndApplyMutation(TEST_USER, badUpdate); + confProvider.confirmPendingMutation(false); assertNull(confProvider.loadConfiguration(conf).get( "yarn.scheduler.capacity.root.a.badKey")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java deleted file mode 100644 index 631ce657e84..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class TestYarnConfigurationStore { - - private YarnConfigurationStore confStore; - private Configuration schedConf; - - private static final String testUser = "testUser"; - - @Before - public void setUp() { - schedConf = new Configuration(false); - schedConf.set("key1", "val1"); - } - - @Test - public void testInMemoryConfigurationStore() throws IOException { - confStore = new InMemoryConfigurationStore(); - confStore.initialize(new Configuration(), schedConf); - assertEquals("val1", confStore.retrieve().get("key1")); - - Map update1 = new HashMap<>(); - update1.put("keyUpdate1", "valUpdate1"); - LogMutation mutation1 = new LogMutation(update1, testUser); - long id = confStore.logMutation(mutation1); - assertEquals(1, confStore.getPendingMutations().size()); - confStore.confirmMutation(id, true); - assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1")); - assertEquals(0, confStore.getPendingMutations().size()); - - Map update2 = new HashMap<>(); - update2.put("keyUpdate2", "valUpdate2"); - LogMutation mutation2 = new LogMutation(update2, testUser); - id = confStore.logMutation(mutation2); - assertEquals(1, confStore.getPendingMutations().size()); - confStore.confirmMutation(id, false); - assertNull("Configuration should not be updated", - confStore.retrieve().get("keyUpdate2")); - assertEquals(0, confStore.getPendingMutations().size()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java new file mode 100644 index 00000000000..3cfa8da8047 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -0,0 +1,312 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.curator.test.TestingServer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.conf.HAUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tests {@link ZKConfigurationStore}. + */ +public class TestZKConfigurationStore extends ConfigurationStoreBaseTest { + + public static final Log LOG = + LogFactory.getLog(TestZKConfigurationStore.class); + + private static final int ZK_TIMEOUT_MS = 10000; + private TestingServer curatorTestingServer; + private CuratorFramework curatorFramework; + private ResourceManager rm; + + public static TestingServer setupCuratorServer() throws Exception { + TestingServer curatorTestingServer = new TestingServer(); + curatorTestingServer.start(); + return curatorTestingServer; + } + + public static CuratorFramework setupCuratorFramework( + TestingServer curatorTestingServer) throws Exception { + CuratorFramework curatorFramework = CuratorFrameworkFactory.builder() + .connectString(curatorTestingServer.getConnectString()) + .retryPolicy(new RetryNTimes(100, 100)) + .build(); + curatorFramework.start(); + return curatorFramework; + } + + @Before + public void setUp() throws Exception { + super.setUp(); + curatorTestingServer = setupCuratorServer(); + curatorFramework = setupCuratorFramework(curatorTestingServer); + + conf.set(CommonConfigurationKeys.ZK_ADDRESS, + curatorTestingServer.getConnectString()); + rm = new MockRM(conf); + rm.start(); + rmContext = rm.getRMContext(); + } + + @After + public void cleanup() throws IOException { + rm.stop(); + curatorFramework.close(); + curatorTestingServer.stop(); + } + + @Test + public void testVersioning() throws Exception { + confStore.initialize(conf, schedConf, rmContext); + assertNull(confStore.getConfStoreVersion()); + confStore.checkVersion(); + assertEquals(ZKConfigurationStore.CURRENT_VERSION_INFO, + confStore.getConfStoreVersion()); + } + + @Test + public void testPersistConfiguration() throws Exception { + schedConf.set("key", "val"); + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + + // Create a new configuration store, and check for old configuration + confStore = createConfStore(); + schedConf.set("key", "badVal"); + // Should ignore passed-in scheduler configuration. + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + } + + + @Test + public void testPersistUpdatedConfiguration() throws Exception { + confStore.initialize(conf, schedConf, rmContext); + assertNull(confStore.retrieve().get("key")); + + Map update = new HashMap<>(); + update.put("key", "val"); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update, TEST_USER); + confStore.logMutation(mutation); + confStore.confirmMutation(true); + assertEquals("val", confStore.retrieve().get("key")); + + // Create a new configuration store, and check for updated configuration + confStore = createConfStore(); + schedConf.set("key", "badVal"); + // Should ignore passed-in scheduler configuration. + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + } + + @Test + public void testMaxLogs() throws Exception { + conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2); + confStore.initialize(conf, schedConf, rmContext); + LinkedList logs = + ((ZKConfigurationStore) confStore).getLogs(); + assertEquals(0, logs.size()); + + Map update1 = new HashMap<>(); + update1.put("key1", "val1"); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update1, TEST_USER); + confStore.logMutation(mutation); + logs = ((ZKConfigurationStore) confStore).getLogs(); + assertEquals(1, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + confStore.confirmMutation(true); + assertEquals(1, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + + Map update2 = new HashMap<>(); + update2.put("key2", "val2"); + mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER); + confStore.logMutation(mutation); + logs = ((ZKConfigurationStore) confStore).getLogs(); + assertEquals(2, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + assertEquals("val2", logs.get(1).getUpdates().get("key2")); + confStore.confirmMutation(true); + assertEquals(2, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + assertEquals("val2", logs.get(1).getUpdates().get("key2")); + + // Next update should purge first update from logs. + Map update3 = new HashMap<>(); + update3.put("key3", "val3"); + mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER); + confStore.logMutation(mutation); + logs = ((ZKConfigurationStore) confStore).getLogs(); + assertEquals(2, logs.size()); + assertEquals("val2", logs.get(0).getUpdates().get("key2")); + assertEquals("val3", logs.get(1).getUpdates().get("key3")); + confStore.confirmMutation(true); + assertEquals(2, logs.size()); + assertEquals("val2", logs.get(0).getUpdates().get("key2")); + assertEquals("val3", logs.get(1).getUpdates().get("key3")); + } + + public Configuration createRMHAConf(String rmIds, String rmId, + int adminPort) { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.set(YarnConfiguration.RM_HA_IDS, rmIds); + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); + conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.ZK_CONFIGURATION_STORE); + conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); + conf.set(YarnConfiguration.RM_ZK_ADDRESS, + curatorTestingServer.getConnectString()); + conf.set(YarnConfiguration.RM_HA_ID, rmId); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0"); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); + for (String rpcAddress : + YarnConfiguration.getServiceAddressConfKeys(conf)) { + for (String id : HAUtil.getRMHAIds(conf)) { + conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0"); + } + } + conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId), + "localhost:" + adminPort); + return conf; + } + + /** + * When failing over, new active RM should read from current state of store, + * including any updates when the new active RM was in standby. + * @throws Exception + */ + @Test + public void testFailoverReadsFromUpdatedStore() throws Exception { + HAServiceProtocol.StateChangeRequestInfo req = + new HAServiceProtocol.StateChangeRequestInfo( + HAServiceProtocol.RequestSource.REQUEST_BY_USER); + + Configuration conf1 = createRMHAConf("rm1,rm2", "rm1", 1234); + ResourceManager rm1 = new MockRM(conf1); + rm1.start(); + rm1.getRMContext().getRMAdminService().transitionToActive(req); + assertEquals("RM with ZKStore didn't start", + Service.STATE.STARTED, rm1.getServiceState()); + assertEquals("RM should be Active", + HAServiceProtocol.HAServiceState.ACTIVE, + rm1.getRMContext().getRMAdminService().getServiceStatus().getState()); + assertNull(((MutableConfScheduler) rm1.getResourceScheduler()) + .getConfiguration().get("key")); + + Configuration conf2 = createRMHAConf("rm1,rm2", "rm2", 5678); + ResourceManager rm2 = new MockRM(conf2); + rm2.start(); + assertEquals("RM should be Standby", + HAServiceProtocol.HAServiceState.STANDBY, + rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); + + // Update configuration on RM1 + SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo(); + schedConfUpdateInfo.getGlobalParams().put("key", "val"); + MutableConfigurationProvider confProvider = ((MutableConfScheduler) + rm1.getResourceScheduler()).getMutableConfProvider(); + UserGroupInformation user = UserGroupInformation + .createUserForTesting(TEST_USER, new String[0]); + confProvider.logAndApplyMutation(user, schedConfUpdateInfo); + rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext()); + assertEquals("val", ((MutableConfScheduler) rm1.getResourceScheduler()) + .getConfiguration().get("key")); + confProvider.confirmPendingMutation(true); + assertEquals("val", ((MutableCSConfigurationProvider) confProvider) + .getConfStore().retrieve().get("key")); + // Next update is not persisted, it should not be recovered + schedConfUpdateInfo.getGlobalParams().put("key", "badVal"); + confProvider.logAndApplyMutation(user, schedConfUpdateInfo); + + // Start RM2 and verifies it starts with updated configuration + rm2.getRMContext().getRMAdminService().transitionToActive(req); + assertEquals("RM with ZKStore didn't start", + Service.STATE.STARTED, rm2.getServiceState()); + assertEquals("RM should be Active", + HAServiceProtocol.HAServiceState.ACTIVE, + rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); + + for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) { + if (HAServiceProtocol.HAServiceState.ACTIVE == + rm1.getRMContext().getRMAdminService().getServiceStatus() + .getState()) { + Thread.sleep(100); + } + } + assertEquals("RM should have been fenced", + HAServiceProtocol.HAServiceState.STANDBY, + rm1.getRMContext().getRMAdminService().getServiceStatus().getState()); + assertEquals("RM should be Active", + HAServiceProtocol.HAServiceState.ACTIVE, + rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); + + assertEquals("val", ((MutableCSConfigurationProvider) ( + (CapacityScheduler) rm2.getResourceScheduler()) + .getMutableConfProvider()).getConfStore().retrieve().get("key")); + assertEquals("val", ((MutableConfScheduler) rm2.getResourceScheduler()) + .getConfiguration().get("key")); + // Transition to standby will set RM's HA status and then reinitialize in + // a separate thread. Despite asserting for STANDBY state, it's + // possible for reinitialization to be unfinished. Wait here for it to + // finish, otherwise closing rm1 will close zkManager and the unfinished + // reinitialization will throw an exception. + Thread.sleep(10000); + rm1.close(); + rm2.close(); + } + + @Override + public YarnConfigurationStore createConfStore() { + return new ZKConfigurationStore(); + } +} From d6622daaa3c95a5040d06f91dc0c793bafa70f95 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Fri, 22 Sep 2017 10:18:27 -0700 Subject: [PATCH 13/16] YARN-7046. Add closing logic to configuration store --- .../MutableConfigurationProvider.java | 8 + .../scheduler/capacity/CapacityScheduler.java | 3 + .../conf/LeveldbConfigurationStore.java | 44 +++- .../conf/MutableCSConfigurationProvider.java | 6 + .../capacity/conf/YarnConfigurationStore.java | 6 + .../conf/ConfigurationStoreBaseTest.java | 2 + .../conf/TestLeveldbConfigurationStore.java | 213 ++++++++++++++++++ 7 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index f8e8814377a..2b9b25ac090 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import java.io.IOException; + /** * Interface for allowing changing scheduler configurations. */ @@ -55,4 +57,10 @@ public interface MutableConfigurationProvider { * @throws Exception if confirming mutation fails */ void confirmPendingMutation(boolean isValid) throws Exception; + + /** + * Closes the configuration provider, releasing any required resources. + * @throws IOException on failure to close + */ + void close() throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 16b27c100dd..de951795eb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -411,6 +411,9 @@ public class CapacityScheduler extends writeLock.unlock(); } + if (isConfigurationMutable()) { + ((MutableConfigurationProvider) csConfProvider).close(); + } super.serviceStop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index 1b0eb9f1a9c..21de7a26d61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -72,7 +73,8 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { private long maxLogs; private Configuration conf; private LogMutation pendingMutation; - private static final Version CURRENT_VERSION_INFO = Version + @VisibleForTesting + protected static final Version CURRENT_VERSION_INFO = Version .newInstance(0, 1); private Timer compactionTimer; private long compactionIntervalMsec; @@ -82,7 +84,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { RMContext rmContext) throws IOException { this.conf = config; try { - this.db = initDatabase(schedConf); + initDatabase(schedConf); this.maxLogs = config.getLong( YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS); @@ -96,7 +98,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { } } - private DB initDatabase(Configuration config) throws Exception { + private void initDatabase(Configuration config) throws Exception { Path storeRoot = createStorageDir(); Options options = new Options(); options.createIfMissing(false); @@ -108,13 +110,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { if (key1Str.equals(key2Str)) { return 0; } else if (key1Str.equals(VERSION_KEY)) { - return -1; + return 1; } else if (key2Str.equals(VERSION_KEY)) { - return 1; - } else if (key1Str.equals(LOG_KEY)) { return -1; - } else if (key2Str.equals(LOG_KEY)) { + } else if (key1Str.equals(LOG_KEY)) { return 1; + } else if (key2Str.equals(LOG_KEY)) { + return -1; } return key1Str.compareTo(key2Str); } @@ -156,7 +158,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { throw e; } } - return db; } private Path createStorageDir() throws IOException { @@ -175,6 +176,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { return new Path(storePath, DB_NAME); } + @Override + public void close() throws IOException { + if (db != null) { + db.close(); + } + } + @Override public void logMutation(LogMutation logMutation) throws IOException { LinkedList logs = deserLogMutations(db.get(bytes(LOG_KEY))); @@ -212,8 +220,12 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { return baos.toByteArray(); } } + private LinkedList deserLogMutations(byte[] mutations) throws IOException { + if (mutations == null) { + return new LinkedList<>(); + } try (ObjectInput input = new ObjectInputStream( new ByteArrayInputStream(mutations))) { return (LinkedList) input.readObject(); @@ -225,13 +237,16 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { @Override public synchronized Configuration retrieve() { DBIterator itr = db.iterator(); - itr.seek(bytes(LOG_KEY)); + itr.seekToFirst(); Configuration config = new Configuration(false); - itr.next(); while (itr.hasNext()) { Map.Entry entry = itr.next(); - config.set(new String(entry.getKey(), StandardCharsets.UTF_8), - new String(entry.getValue(), StandardCharsets.UTF_8)); + String key = new String(entry.getKey(), StandardCharsets.UTF_8); + String value = new String(entry.getValue(), StandardCharsets.UTF_8); + if (key.equals(LOG_KEY) || key.equals(VERSION_KEY)) { + break; + } + config.set(key, value); } return config; } @@ -268,6 +283,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore { return version; } + @VisibleForTesting + protected LinkedList getLogs() throws Exception { + return deserLogMutations(db.get(bytes(LOG_KEY))); + } + @Override public void storeVersion() throws Exception { String key = VERSION_KEY; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 70d18403b51..ccadf76f950 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -92,6 +92,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } try { confStore.initialize(config, schedConf, rmContext); + confStore.checkVersion(); } catch (Exception e) { throw new IOException(e); } @@ -103,6 +104,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, aclMutationPolicy.init(config, rmContext); } + @Override + public void close() throws IOException { + confStore.close(); + } + @VisibleForTesting public YarnConfigurationStore getConfStore() { return confStore; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 13565356ee0..7fb52fcccd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -95,6 +95,12 @@ public abstract class YarnConfigurationStore { public abstract void initialize(Configuration conf, Configuration schedConf, RMContext rmContext) throws Exception; + /** + * Closes the configuration store, releasing any required resources. + * @throws IOException on failure to close + */ + public void close() throws IOException {} + /** * Logs the configuration change to backing store. * @param logMutation configuration change to be persisted in write ahead log diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java index bbe95700f18..8f3bc710837 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java @@ -71,6 +71,7 @@ public abstract class ConfigurationStoreBaseTest { confStore.confirmMutation(false); assertNull("Configuration should not be updated", confStore.retrieve().get("keyUpdate2")); + confStore.close(); } @Test @@ -86,5 +87,6 @@ public abstract class ConfigurationStoreBaseTest { confStore.logMutation(mutation); confStore.confirmMutation(true); assertNull(confStore.retrieve().get("key")); + confStore.close(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java new file mode 100644 index 00000000000..779208a3fb2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tests {@link LeveldbConfigurationStore}. + */ +public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest { + + public static final Log LOG = + LogFactory.getLog(TestLeveldbConfigurationStore.class); + private static final File TEST_DIR = new File( + System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), + TestLeveldbConfigurationStore.class.getName()); + + private ResourceManager rm; + + @Before + public void setUp() throws Exception { + super.setUp(); + FileUtil.fullyDelete(TEST_DIR); + conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.LEVELDB_CONFIGURATION_STORE); + conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString()); + } + + @Test + public void testVersioning() throws Exception { + confStore.initialize(conf, schedConf, rmContext); + assertNull(confStore.getConfStoreVersion()); + confStore.checkVersion(); + assertEquals(LeveldbConfigurationStore.CURRENT_VERSION_INFO, + confStore.getConfStoreVersion()); + confStore.close(); + } + + @Test + public void testPersistConfiguration() throws Exception { + schedConf.set("key", "val"); + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + confStore.close(); + + // Create a new configuration store, and check for old configuration + confStore = createConfStore(); + schedConf.set("key", "badVal"); + // Should ignore passed-in scheduler configuration. + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + confStore.close(); + } + + @Test + public void testPersistUpdatedConfiguration() throws Exception { + confStore.initialize(conf, schedConf, rmContext); + assertNull(confStore.retrieve().get("key")); + + Map update = new HashMap<>(); + update.put("key", "val"); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update, TEST_USER); + confStore.logMutation(mutation); + confStore.confirmMutation(true); + assertEquals("val", confStore.retrieve().get("key")); + confStore.close(); + + // Create a new configuration store, and check for updated configuration + confStore = createConfStore(); + schedConf.set("key", "badVal"); + // Should ignore passed-in scheduler configuration. + confStore.initialize(conf, schedConf, rmContext); + assertEquals("val", confStore.retrieve().get("key")); + confStore.close(); + } + + @Test + public void testMaxLogs() throws Exception { + conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2); + confStore.initialize(conf, schedConf, rmContext); + LinkedList logs = + ((LeveldbConfigurationStore) confStore).getLogs(); + assertEquals(0, logs.size()); + + Map update1 = new HashMap<>(); + update1.put("key1", "val1"); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update1, TEST_USER); + confStore.logMutation(mutation); + logs = ((LeveldbConfigurationStore) confStore).getLogs(); + assertEquals(1, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + confStore.confirmMutation(true); + assertEquals(1, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + + Map update2 = new HashMap<>(); + update2.put("key2", "val2"); + mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER); + confStore.logMutation(mutation); + logs = ((LeveldbConfigurationStore) confStore).getLogs(); + assertEquals(2, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + assertEquals("val2", logs.get(1).getUpdates().get("key2")); + confStore.confirmMutation(true); + assertEquals(2, logs.size()); + assertEquals("val1", logs.get(0).getUpdates().get("key1")); + assertEquals("val2", logs.get(1).getUpdates().get("key2")); + + // Next update should purge first update from logs. + Map update3 = new HashMap<>(); + update3.put("key3", "val3"); + mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER); + confStore.logMutation(mutation); + logs = ((LeveldbConfigurationStore) confStore).getLogs(); + assertEquals(2, logs.size()); + assertEquals("val2", logs.get(0).getUpdates().get("key2")); + assertEquals("val3", logs.get(1).getUpdates().get("key3")); + confStore.confirmMutation(true); + assertEquals(2, logs.size()); + assertEquals("val2", logs.get(0).getUpdates().get("key2")); + assertEquals("val3", logs.get(1).getUpdates().get("key3")); + confStore.close(); + } + + /** + * When restarting, RM should read from current state of store, including + * any updates from the previous RM instance. + * @throws Exception + */ + @Test + public void testRestartReadsFromUpdatedStore() throws Exception { + ResourceManager rm1 = new MockRM(conf); + rm1.start(); + assertNull(((MutableConfScheduler) rm1.getResourceScheduler()) + .getConfiguration().get("key")); + + // Update configuration on RM + SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo(); + schedConfUpdateInfo.getGlobalParams().put("key", "val"); + MutableConfigurationProvider confProvider = ((MutableConfScheduler) + rm1.getResourceScheduler()).getMutableConfProvider(); + UserGroupInformation user = UserGroupInformation + .createUserForTesting(TEST_USER, new String[0]); + confProvider.logAndApplyMutation(user, schedConfUpdateInfo); + rm1.getResourceScheduler().reinitialize(conf, rm1.getRMContext()); + assertEquals("val", ((MutableConfScheduler) rm1.getResourceScheduler()) + .getConfiguration().get("key")); + confProvider.confirmPendingMutation(true); + assertEquals("val", ((MutableCSConfigurationProvider) confProvider) + .getConfStore().retrieve().get("key")); + // Next update is not persisted, it should not be recovered + schedConfUpdateInfo.getGlobalParams().put("key", "badVal"); + confProvider.logAndApplyMutation(user, schedConfUpdateInfo); + rm1.close(); + + // Start RM2 and verifies it starts with updated configuration + ResourceManager rm2 = new MockRM(conf); + rm2.start(); + assertEquals("val", ((MutableCSConfigurationProvider) ( + (CapacityScheduler) rm2.getResourceScheduler()) + .getMutableConfProvider()).getConfStore().retrieve().get("key")); + assertEquals("val", ((MutableConfScheduler) rm2.getResourceScheduler()) + .getConfiguration().get("key")); + rm2.close(); + } + + @Override + public YarnConfigurationStore createConfStore() { + return new LeveldbConfigurationStore(); + } +} From a3075de9ab2fc9a97debaab6fc838ea07b1cc77e Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Mon, 25 Sep 2017 17:47:32 -0700 Subject: [PATCH 14/16] YARN-7238. Documentation for API based scheduler configuration management --- .../src/site/markdown/CapacityScheduler.md | 24 ++- .../src/site/markdown/ResourceManagerRest.md | 186 ++++++++++++++++++ .../src/site/markdown/YarnCommands.md | 13 ++ 3 files changed, 221 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 6bb848900ee..d70f891b0ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -295,9 +295,29 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach Changing Queue Configuration ---------------------------- -Changing queue properties and adding new queues is very simple. You need to edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*. +Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. + +### Changing queue configuration via file + + To edit by file, you need to edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*. $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues -**Note:** Queues cannot be *deleted*, only addition of new queues is supported - the updated queue configuration should be a valid one i.e. queue-capacity at each *level* should be equal to 100%. +### Changing queue configuration via API + + Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. + + | Property | Description | + |:---- |:---- | + | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. | + | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). | + | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. | + | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. | + | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. | + | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. | + | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. | + + **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file. + + See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 82f80f37878..025e9a7dcd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -4430,3 +4430,189 @@ Response Body: 90 ``` + +Scheduler Configuration Mutation API +-------------------------------- + +The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy. + + +### URI + + * http://rm-http-address:port/ws/v1/cluster/sched-conf + +### HTTP Operations Supported + + * PUT + +### Elements of the *schedConf* object + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| update-queue | object | A queue whose configurations should be updated | +| add-queue | object | A queue to add to the scheduler along with this queue's configurations | +| remove-queue | string | Full path name of a queue to remove | +| global-updates | map | Map of key value pairs to update scheduler's global configuration | + +### PUT Request Examples + +Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response. + +**Updating queue configuration(s)** + +Request for updating queue configurations. + +*Elements of the* update-queue *object* + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| queueName | string | Full path name of the queue to update | +| params | map | A map of key value configuration pairs to update for this queue | + +Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10. + +HTTP Request: + +```xml + Accept: application/xml + PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + Content-Type: application/xml + + + + root.default + + + maximum-applications + 100 + + + minimum-user-limit-percent + 10 + + + + +``` + + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Transfer-Encoding: chunked + + +**Adding a queue** + +Request for adding queues/updating queue configurations. + +*Elements of the* add-queue *object* + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| queueName | string | Full path name of the queue to add | +| params | map | A map of key value configuration pairs to set for this queue | + +Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.) + +HTTP Request: + +```xml + Accept: application/xml + PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + Content-Type: application/xml + + + + root.a + + + capacity + 10 + + + maximum-capacity + 10 + + + + + root.default + + + capacity + 90 + + + maximum-capacity + 90 + + + + +``` + + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Transfer-Encoding: chunked + +**Removing queues** + +Request for removing queues from the queue hierarchy. + +Assuming we are using the capacity scheduler and the current queue configuration is three queues *root.default*, *root.a*, and *root.b*, this example removes both *root.a* and *root.b*. (More complex examples include removing a parent queue and its children.) + +**Note:** Queues must be put into `STOPPED` state before they are deleted. Any updated queue configuration should be a valid one i.e. queue-capacity at each *level* should be equal to 100%. + + +HTTP Request: + +```xml + Accept: application/xml + PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + Content-Type: application/xml + + + root.a + root.b + +``` + + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Transfer-Encoding: chunked + +**Updating global scheduler configurations** + +Request for updating global scheduler configurations. Assuming we are using the capacity scheduler, this example enables queue mappings. For global configuration updates, the full configuration key must be specified. + +HTTP Request: + +```xml + Accept: application/xml + PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + Content-Type: application/xml + + + + + yarn.scheduler.capacity.queue-mappings-override.enable + true + + + +``` + + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Transfer-Encoding: chunked diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 5f430ec4374..2fc65bc0da2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -237,6 +237,19 @@ Usage: Runs ResourceManager admin client +### schedconf + +Usage: `yarn schedconf [options]` + +| COMMAND\_OPTIONS | Description | +|:---- |:---- | +| -add | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. | +| -remove | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. | +| -update | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. | +| -global | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. | + +Updates scheduler configuration + ### scmadmin Usage: `yarn scmadmin [options] ` From 09c5dfe937f0570cd9494b34d210df2d5f0737a7 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Mon, 25 Sep 2017 17:52:40 -0700 Subject: [PATCH 15/16] YARN-7251. Misc changes to YARN-5734 --- hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 33 ++++++++++++++- .../hadoop/yarn/client/cli/SchedConfCLI.java | 42 +++++++++++++------ .../yarn/client/cli/TestSchedConfCLI.java | 20 ++++----- .../yarn/webapp/dao/QueueConfigInfo.java | 2 +- .../yarn/webapp/dao/SchedConfUpdateInfo.java | 2 +- .../src/main/resources/yarn-default.xml | 15 ++++--- .../scheduler/capacity/CapacityScheduler.java | 12 +++--- .../CapacitySchedulerConfiguration.java | 12 ------ .../conf/MutableCSConfigurationProvider.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 2 +- .../resourcemanager/TestRMAdminService.java | 4 +- .../conf/TestLeveldbConfigurationStore.java | 2 - .../TestMutableCSConfigurationProvider.java | 3 ++ .../conf/TestZKConfigurationStore.java | 2 - ...estRMWebServicesConfigurationMutation.java | 28 ++++++------- .../src/site/markdown/CapacityScheduler.md | 11 ++--- .../src/site/markdown/ResourceManagerRest.md | 40 +++++++++--------- .../src/site/markdown/YarnCommands.md | 12 +++--- 20 files changed, 146 insertions(+), 104 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 00635961144..0f19989efc7 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -47,6 +47,7 @@ function hadoop_usage hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager" hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" + hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" @@ -142,7 +143,7 @@ function yarncmd_case HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router' ;; - schedconf) + schedulerconf) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI' ;; scmadmin) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 7ec9848ae64..fed3d90a7c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -295,7 +295,7 @@ goto :eof set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof -:schedconf +:schedulerconf set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof @@ -345,6 +345,7 @@ goto :eof @echo node prints node report(s) @echo queue prints queue information @echo logs dump container logs + @echo schedulerconf updates scheduler configuration @echo classpath prints the class path needed to get the @echo Hadoop jar and the required libraries @echo daemonlog get/set the log level for each daemon diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8809a7cb1e7..ac8ff12b8a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -674,33 +674,64 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; + @Private + @Unstable public static final String SCHEDULER_CONFIGURATION_STORE_CLASS = YARN_PREFIX + "scheduler.configuration.store.class"; + @Private + @Unstable + public static final String FILE_CONFIGURATION_STORE = "file"; + @Private + @Unstable public static final String MEMORY_CONFIGURATION_STORE = "memory"; + @Private + @Unstable public static final String LEVELDB_CONFIGURATION_STORE = "leveldb"; + @Private + @Unstable public static final String ZK_CONFIGURATION_STORE = "zk"; + @Private + @Unstable public static final String DEFAULT_CONFIGURATION_STORE = - MEMORY_CONFIGURATION_STORE; + FILE_CONFIGURATION_STORE; + @Private + @Unstable public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX + "scheduler.configuration.leveldb-store.path"; + @Private + @Unstable public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = YARN_PREFIX + "scheduler.configuration.leveldb-store.compaction-interval-secs"; + @Private + @Unstable public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L; + @Private + @Unstable public static final String RM_SCHEDCONF_MAX_LOGS = YARN_PREFIX + "scheduler.configuration.store.max-logs"; + @Private + @Unstable public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000; + @Private + @Unstable public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000; /** Parent znode path under which ZKConfigurationStore will create znodes. */ + @Private + @Unstable public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX + "scheduler.configuration.zk-store.parent-path"; + @Private + @Unstable public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH = "/confstore"; + @Private + @Unstable public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS = YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index e17062e5f17..11bfdd7ba14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -27,7 +27,7 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.MissingArgumentException; import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -48,7 +48,7 @@ import java.util.Map; * CLI for modifying scheduler configuration. */ @Public -@Evolving +@Unstable public class SchedConfCLI extends Configured implements Tool { private static final String ADD_QUEUES_OPTION = "addQueues"; @@ -135,7 +135,7 @@ public class SchedConfCLI extends Configured implements Tool { WebResource webResource = webServiceClient.resource(WebAppUtils. getRMWebAppURLWithScheme(getConf())); ClientResponse response = webResource.path("ws").path("v1").path("cluster") - .path("sched-conf").accept(MediaType.APPLICATION_JSON) + .path("scheduler-conf").accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); @@ -170,7 +170,7 @@ public class SchedConfCLI extends Configured implements Tool { if (args == null) { return; } - List queuesToRemove = Arrays.asList(args.split(",")); + List queuesToRemove = Arrays.asList(args.split(";")); updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove)); } @@ -199,11 +199,14 @@ public class SchedConfCLI extends Configured implements Tool { } private QueueConfigInfo getQueueConfigInfo(String arg) { - String[] queueArgs = arg.split(","); - String queuePath = queueArgs[0]; + String[] args = arg.split(":"); + String queuePath = args[0]; Map queueConfigs = new HashMap<>(); - for (int i = 1; i < queueArgs.length; ++i) { - putKeyValuePair(queueConfigs, queueArgs[i]); + if (args.length > 1) { + String[] queueArgs = args[1].split(","); + for (int i = 0; i < queueArgs.length; ++i) { + putKeyValuePair(queueConfigs, queueArgs[i]); + } } return new QueueConfigInfo(queuePath, queueConfigs); } @@ -228,11 +231,24 @@ public class SchedConfCLI extends Configured implements Tool { } private void printUsage() { - System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1," - + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] " - + "[-remove queueRemovePath1,queueRemovePath2] " - + "[-update queueUpdatePath1,confKey1=confVal1] " + System.out.println("yarn schedulerconf [-add " + + "\"queueAddPath1:confKey1=confVal1,confKey2=confVal2;" + + "queueAddPath2:confKey3=confVal3\"] " + + "[-remove \"queueRemovePath1;queueRemovePath2\"] " + + "[-update \"queueUpdatePath1:confKey1=confVal1\"] " + "[-global globalConfKey1=globalConfVal1," - + "globalConfKey2=globalConfVal2]"); + + "globalConfKey2=globalConfVal2]\n" + + "Example (adding queues): yarn schedulerconf -add " + + "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0," + + "maximum-capacity=0\"\n" + + "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;" + + "root.a.a2\"\n" + + "Example (updating queues): yarn schedulerconf -update \"root.a.a1" + + ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75," + + "maximum-capacity=75\"\n" + + "Example (global scheduler update): yarn schedulerconf " + + "-global yarn.scheduler.capacity.maximum-applications=10000\n" + + "Note: This is an alpha feature, the syntax/options are subject to " + + "change, please run at your own risk."); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index d2f063920d9..5364e83019c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -61,20 +61,20 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testInvalidConf() throws Exception { // conf pair with no key should be invalid - int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"}); + int exitCode = cli.run(new String[] {"-add", "root.a:=confVal"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); - exitCode = cli.run(new String[] {"-update", "root.a,=confVal"}); + exitCode = cli.run(new String[] {"-update", "root.a:=confVal"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); - exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"}); + exitCode = cli.run(new String[] {"-add", "root.a:confKey=confVal=conf"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); - exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"}); + exitCode = cli.run(new String[] {"-update", "root.a:confKey=confVal=c"}); assertTrue("Should return an error code", exitCode != 0); assertTrue(sysErrStream.toString().contains("Specify configuration key " + "value as confKey=confVal.")); @@ -83,8 +83,7 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testAddQueues() { SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); - cli.addQueues("root.a,a1=aVal1,a2=aVal2," + - "a3=", schedUpdateInfo); + cli.addQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo); QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0); assertEquals("root.a", addInfo.getQueue()); Map params = addInfo.getParams(); @@ -94,7 +93,7 @@ public class TestSchedConfCLI { assertNull(params.get("a3")); schedUpdateInfo = new SchedConfUpdateInfo(); - cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo); + cli.addQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo); assertEquals(2, schedUpdateInfo.getAddQueueInfo().size()); QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0); assertEquals("root.b", bAddInfo.getQueue()); @@ -111,7 +110,7 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testRemoveQueues() { SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); - cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo); + cli.removeQueues("root.a;root.b;root.c.c1", schedUpdateInfo); List removeInfo = schedUpdateInfo.getRemoveQueueInfo(); assertEquals(3, removeInfo.size()); assertEquals("root.a", removeInfo.get(0)); @@ -122,8 +121,7 @@ public class TestSchedConfCLI { @Test(timeout = 10000) public void testUpdateQueues() { SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); - cli.updateQueues("root.a,a1=aVal1,a2=aVal2," + - "a3=", schedUpdateInfo); + cli.updateQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo); QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); assertEquals("root.a", updateInfo.getQueue()); Map params = updateInfo.getParams(); @@ -133,7 +131,7 @@ public class TestSchedConfCLI { assertNull(params.get("a3")); schedUpdateInfo = new SchedConfUpdateInfo(); - cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo); + cli.updateQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo); assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size()); QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); assertEquals("root.b", bUpdateInfo.getQueue()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java index d1d91c2be0c..6d4e0cfa755 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java @@ -34,7 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement; @XmlAccessorType(XmlAccessType.FIELD) public class QueueConfigInfo { - @XmlElement(name = "queueName") + @XmlElement(name = "queue-name") private String queue; private HashMap params = new HashMap<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java index bb84096a545..45462919ed1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java @@ -31,7 +31,7 @@ import javax.xml.bind.annotation.XmlRootElement; * Information for making scheduler configuration changes (supports adding, * removing, or updating a queue, as well as global scheduler conf changes). */ -@XmlRootElement(name = "schedConf") +@XmlRootElement(name = "sched-conf") @XmlAccessorType(XmlAccessType.FIELD) public class SchedConfUpdateInfo { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4516edfe33e..421bccfc0ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3369,14 +3369,17 @@ - The type of configuration store to use for storing scheduler - configurations, if using a mutable configuration provider. - Keywords such as "memory" map to certain configuration store - implementations. If keyword is not found, try to load this - value as a class. + The type of configuration store to use for scheduler configurations. + Default is "file", which uses file based capacity-scheduler.xml to + retrieve and change scheduler configuration. To enable API based + scheduler configuration, use either "memory" (in memory storage, no + persistence across restarts), "leveldb" (leveldb based storage), or + "zk" (zookeeper based storage). API based configuration is only useful + when using a scheduler which supports mutable configuration. Currently + only capacity scheduler supports this. yarn.scheduler.configuration.store.class - memory + file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index de951795eb0..d91aa55a487 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -296,18 +296,20 @@ public class CapacityScheduler extends try { writeLock.lock(); String confProviderStr = configuration.get( - CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER); + YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.DEFAULT_CONFIGURATION_STORE); switch (confProviderStr) { - case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER: + case YarnConfiguration.FILE_CONFIGURATION_STORE: this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext); break; - case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER: + case YarnConfiguration.MEMORY_CONFIGURATION_STORE: + case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: + case YarnConfiguration.ZK_CONFIGURATION_STORE: this.csConfProvider = new MutableCSConfigurationProvider(rmContext); break; default: - throw new IOException("Invalid CS configuration provider: " + + throw new IOException("Invalid configuration store class: " + confProviderStr); } this.csConfProvider.init(configuration); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 40cb893b6d3..3a519ecf5f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -315,18 +315,6 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur @Private public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; - - public static final String CS_CONF_PROVIDER = PREFIX - + "configuration.provider"; - - @Private - public static final String FILE_CS_CONF_PROVIDER = "file"; - - @Private - public static final String STORE_CS_CONF_PROVIDER = "store"; - - @Private - public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER; AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index ccadf76f950..40a19a42791 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -66,7 +66,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, public void init(Configuration config) throws IOException { String store = config.get( YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, - YarnConfiguration.DEFAULT_CONFIGURATION_STORE); + YarnConfiguration.MEMORY_CONFIGURATION_STORE); switch (store) { case YarnConfiguration.MEMORY_CONFIGURATION_STORE: this.confStore = new InMemoryConfigurationStore(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d264c107d8a..e6a0cae55bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2461,7 +2461,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } @PUT - @Path("/sched-conf") + @Path("/scheduler-conf") @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 620ca58a77c..03fc0813335 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -199,8 +199,8 @@ public class TestRMAdminService { @Test public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() { - configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + configuration.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); try { rm = new MockRM(configuration); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java index 779208a3fb2..324cbee0cf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java @@ -59,8 +59,6 @@ public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest { public void setUp() throws Exception { super.setUp(); FileUtil.fullyDelete(TEST_DIR); - conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, YarnConfiguration.LEVELDB_CONFIGURATION_STORE); conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 9b080cd1e2d..5d43ebbf2ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -80,6 +81,8 @@ public class TestMutableCSConfigurationProvider { @Test public void testInMemoryBackedProvider() throws Exception { Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index 3cfa8da8047..355f7418038 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -199,8 +199,6 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest { conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, rmIds); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); - conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, YarnConfiguration.ZK_CONFIGURATION_STORE); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 26ef1b7f7bd..3d28f128173 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -88,8 +88,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, - CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName); try { if (CONF_FILE.exists()) { @@ -179,7 +179,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getAddQueueInfo().add(d); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -214,7 +214,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -240,7 +240,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getRemoveQueueInfo().add("root.a.a2"); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -265,7 +265,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getRemoveQueueInfo().add("root.c"); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -296,7 +296,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -328,7 +328,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(configInfo); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -356,7 +356,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { } response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -391,7 +391,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { 0.001f); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -409,7 +409,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { updateInfo.getUpdateQueueInfo().add(aUpdateInfo); response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -439,7 +439,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -464,7 +464,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) @@ -479,7 +479,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { // Unset maximum-applications. Should be set to default. response = r.path("ws").path("v1").path("cluster") - .path("sched-conf").queryParam("user.name", userName) + .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index d70f891b0ff..17a6ab9152b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -295,7 +295,7 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach Changing Queue Configuration ---------------------------- -Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. +Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. This behavior can be changed via `yarn.scheduler.configuration.store.class` in yarn-site.xml. Possible values are *file*, which allows modifying properties via file; *memory*, which allows modifying properties via API, but does not persist changes across restart; *leveldb*, which allows modifying properties via API and stores changes in leveldb backing store; and *zk*, which allows modifying properties via API and stores changes in zookeeper backing store. The default value is *file*. ### Changing queue configuration via file @@ -308,16 +308,17 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. + **Note:** This feature is in alpha phase and is subject to change. + | Property | Description | |:---- |:---- | - | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. | - | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). | + | `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). | | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. | | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. | | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. | | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. | | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. | - **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file. + **Note:** When enabling scheduler configuration mutations via `yarn.scheduler.configuration.store.class`, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file. - See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line. + See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedulerconf) for examples on how to change scheduler configuration via command line. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 025e9a7dcd9..f3a19079624 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -4436,16 +4436,18 @@ Scheduler Configuration Mutation API The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy. +Please note that this feature is currently in the alpha stage and is subject to change. + ### URI - * http://rm-http-address:port/ws/v1/cluster/sched-conf + * http://rm-http-address:port/ws/v1/cluster/scheduler-conf ### HTTP Operations Supported * PUT -### Elements of the *schedConf* object +### Elements of the *sched-conf* object | Item | Data Type | Description | |:---- |:---- |:---- | @@ -4466,7 +4468,7 @@ Request for updating queue configurations. | Item | Data Type | Description | |:---- |:---- |:---- | -| queueName | string | Full path name of the queue to update | +| queue-name | string | Full path name of the queue to update | | params | map | A map of key value configuration pairs to update for this queue | Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10. @@ -4475,12 +4477,12 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml - + - root.default + root.default maximum-applications @@ -4492,7 +4494,7 @@ HTTP Request: - + ``` @@ -4511,7 +4513,7 @@ Request for adding queues/updating queue configurations. | Item | Data Type | Description | |:---- |:---- |:---- | -| queueName | string | Full path name of the queue to add | +| queue-name | string | Full path name of the queue to add | | params | map | A map of key value configuration pairs to set for this queue | Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.) @@ -4520,12 +4522,12 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml - + - root.a + root.a capacity @@ -4538,7 +4540,7 @@ HTTP Request: - root.default + root.default capacity @@ -4550,7 +4552,7 @@ HTTP Request: - + ``` @@ -4573,13 +4575,13 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml - + root.a root.b - + ``` @@ -4597,17 +4599,17 @@ HTTP Request: ```xml Accept: application/xml - PUT http://rm-http-address:port/ws/v1/cluster/sched-conf + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf Content-Type: application/xml - + yarn.scheduler.capacity.queue-mappings-override.enable true - + ``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 2fc65bc0da2..fa33b8bc158 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -237,18 +237,18 @@ Usage: Runs ResourceManager admin client -### schedconf +### schedulerconf -Usage: `yarn schedconf [options]` +Usage: `yarn schedulerconf [options]` | COMMAND\_OPTIONS | Description | |:---- |:---- | -| -add | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. | -| -remove | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. | -| -update | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. | +| -add <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. | +| -remove <"queuePath1;queuePath2"> | Semicolon separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. | +| -update <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. | | -global | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. | -Updates scheduler configuration +Updates scheduler configuration. Note, this feature is in alpha phase and is subject to change. ### scmadmin From 1d36b53ab6d9bb1d9144101e424c24371343c5bf Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Tue, 26 Sep 2017 11:41:05 -0700 Subject: [PATCH 16/16] YARN-7252. Removing queue then failing over results in exception --- .../capacity/CapacitySchedulerContext.java | 6 ++ .../CapacitySchedulerQueueManager.java | 11 +- .../conf/TestZKConfigurationStore.java | 102 ++++++++++++++++++ 3 files changed, 117 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java index 9aeaec679b0..7c918a53620 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java @@ -88,4 +88,10 @@ public interface CapacitySchedulerContext { * @return Max Cluster level App priority. */ Priority getMaxClusterLevelAppPriority(); + + /** + * Returns if configuration is mutable. + * @return if configuration is mutable + */ + boolean isConfigurationMutable(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 1ceb6fb3ea1..48c289f0cde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueState; @@ -170,8 +171,14 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager< CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); - // Ensure queue hiearchy in the new XML file is proper. - validateQueueHierarchy(queues, newQueues); + // When failing over, if using configuration store, don't validate queue + // hierarchy since queues can be removed without being STOPPED. + if (!csContext.isConfigurationMutable() || + csContext.getRMContext().getHAServiceState() + != HAServiceProtocol.HAServiceState.STANDBY) { + // Ensure queue hiearchy in the new XML file is proper. + validateQueueHierarchy(queues, newQueues); + } // Add new queues and delete OldQeueus only after validation. updateQueues(queues, newQueues); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index 355f7418038..0cf5e6fa7e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -38,18 +38,21 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfSchedu import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Tests {@link ZKConfigurationStore}. @@ -303,6 +306,105 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest { rm2.close(); } + /** + * When failing over, if RM1 stopped and removed a queue that RM2 has in + * memory, failing over to RM2 should not throw an exception. + * @throws Exception + */ + @Test + public void testFailoverAfterRemoveQueue() throws Exception { + HAServiceProtocol.StateChangeRequestInfo req = + new HAServiceProtocol.StateChangeRequestInfo( + HAServiceProtocol.RequestSource.REQUEST_BY_USER); + + Configuration conf1 = createRMHAConf("rm1,rm2", "rm1", 1234); + ResourceManager rm1 = new MockRM(conf1); + rm1.start(); + rm1.getRMContext().getRMAdminService().transitionToActive(req); + assertEquals("RM with ZKStore didn't start", + Service.STATE.STARTED, rm1.getServiceState()); + assertEquals("RM should be Active", + HAServiceProtocol.HAServiceState.ACTIVE, + rm1.getRMContext().getRMAdminService().getServiceStatus().getState()); + + Configuration conf2 = createRMHAConf("rm1,rm2", "rm2", 5678); + ResourceManager rm2 = new MockRM(conf2); + rm2.start(); + assertEquals("RM should be Standby", + HAServiceProtocol.HAServiceState.STANDBY, + rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); + + UserGroupInformation user = UserGroupInformation + .createUserForTesting(TEST_USER, new String[0]); + MutableConfigurationProvider confProvider = ((MutableConfScheduler) + rm1.getResourceScheduler()).getMutableConfProvider(); + // Add root.a + SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo(); + Map addParams = new HashMap<>(); + addParams.put("capacity", "100"); + QueueConfigInfo addInfo = new QueueConfigInfo("root.a", addParams); + schedConfUpdateInfo.getAddQueueInfo().add(addInfo); + // Stop root.default + Map stopParams = new HashMap<>(); + stopParams.put("state", "STOPPED"); + stopParams.put("capacity", "0"); + QueueConfigInfo stopInfo = new QueueConfigInfo("root.default", stopParams); + schedConfUpdateInfo.getUpdateQueueInfo().add(stopInfo); + confProvider.logAndApplyMutation(user, schedConfUpdateInfo); + rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext()); + confProvider.confirmPendingMutation(true); + assertTrue(Arrays.asList(((MutableConfScheduler) rm1.getResourceScheduler()) + .getConfiguration().get("yarn.scheduler.capacity.root.queues").split + (",")).contains("a")); + + // Remove root.default + schedConfUpdateInfo.getUpdateQueueInfo().clear(); + schedConfUpdateInfo.getAddQueueInfo().clear(); + schedConfUpdateInfo.getRemoveQueueInfo().add("root.default"); + confProvider.logAndApplyMutation(user, schedConfUpdateInfo); + rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext()); + confProvider.confirmPendingMutation(true); + assertEquals("a", ((MutableConfScheduler) rm1.getResourceScheduler()) + .getConfiguration().get("yarn.scheduler.capacity.root.queues")); + + // Start RM2 and verifies it starts with updated configuration + rm2.getRMContext().getRMAdminService().transitionToActive(req); + assertEquals("RM with ZKStore didn't start", + Service.STATE.STARTED, rm2.getServiceState()); + assertEquals("RM should be Active", + HAServiceProtocol.HAServiceState.ACTIVE, + rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); + + for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) { + if (HAServiceProtocol.HAServiceState.ACTIVE == + rm1.getRMContext().getRMAdminService().getServiceStatus() + .getState()) { + Thread.sleep(100); + } + } + assertEquals("RM should have been fenced", + HAServiceProtocol.HAServiceState.STANDBY, + rm1.getRMContext().getRMAdminService().getServiceStatus().getState()); + assertEquals("RM should be Active", + HAServiceProtocol.HAServiceState.ACTIVE, + rm2.getRMContext().getRMAdminService().getServiceStatus().getState()); + + assertEquals("a", ((MutableCSConfigurationProvider) ( + (CapacityScheduler) rm2.getResourceScheduler()) + .getMutableConfProvider()).getConfStore().retrieve() + .get("yarn.scheduler.capacity.root.queues")); + assertEquals("a", ((MutableConfScheduler) rm2.getResourceScheduler()) + .getConfiguration().get("yarn.scheduler.capacity.root.queues")); + // Transition to standby will set RM's HA status and then reinitialize in + // a separate thread. Despite asserting for STANDBY state, it's + // possible for reinitialization to be unfinished. Wait here for it to + // finish, otherwise closing rm1 will close zkManager and the unfinished + // reinitialization will throw an exception. + Thread.sleep(10000); + rm1.close(); + rm2.close(); + } + @Override public YarnConfigurationStore createConfStore() { return new ZKConfigurationStore();