From 94b512bc51ecb5f4eafcf17676a340a6033aefe4 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sun, 2 Feb 2014 15:39:02 +0000 Subject: [PATCH 01/42] HDFS-5828. BlockPlacementPolicyWithNodeGroup can place multiple replicas on the same node group when dfs.namenode.avoid.write.stale.datanode is true. (Buddy via junping_du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563640 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ .../server/blockmanagement/BlockPlacementPolicyDefault.java | 2 +- .../blockmanagement/TestReplicationPolicyWithNodeGroup.java | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5867e7bf2f8..0b9caec75b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -324,6 +324,10 @@ Release 2.4.0 - UNRELEASED HDFS-5856. DataNode.checkDiskError might throw NPE. (Josh Elser via suresh) + HDFS-5828. BlockPlacementPolicyWithNodeGroup can place multiple replicas on + the same node group when dfs.namenode.avoid.write.stale.datanode is true. + (Buddy via junping_du) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index ff1f0f4c64d..f4dc208d731 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -317,7 +317,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { // We need to additionally exclude the nodes that were added to the // result list in the successful calls to choose*() above. for (DatanodeStorageInfo resultStorage : results) { - oldExcludedNodes.add(resultStorage.getDatanodeDescriptor()); + addToExcludedNodes(resultStorage.getDatanodeDescriptor(), oldExcludedNodes); } // Set numOfReplicas, since it can get out of sync with the result list // if the NotEnoughReplicasException was thrown in chooseRandom(). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index 23209f0e28b..ffcc18cc789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -124,6 +124,8 @@ public class TestReplicationPolicyWithNodeGroup { CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, NetworkTopologyWithNodeGroup.class.getName()); + CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY, true); + File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class); CONF.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, From 88d8ea95092d8391c106532511d40c897ade5707 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sun, 2 Feb 2014 19:51:39 +0000 Subject: [PATCH 02/42] HADOOP-10085. CompositeService should allow adding services while being inited. (Steve Loughran via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563694 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/service/CompositeService.java | 5 +- .../hadoop/service}/TestCompositeService.java | 232 ++++++++++++++++-- 3 files changed, 217 insertions(+), 23 deletions(-) rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util => hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service}/TestCompositeService.java (66%) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c858d7d4266..d4db1854569 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -313,6 +313,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10320. Javadoc in InterfaceStability.java lacks final . (René Nyffenegger via cnauroth) + + HADOOP-10085. CompositeService should allow adding services while being + inited. (Steve Loughran via kasha) Release 2.3.0 - UNRELEASED diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java index 123ca52ba96..ca667e2bcf4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java @@ -19,7 +19,6 @@ package org.apache.hadoop.service; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; @@ -54,13 +53,13 @@ public class CompositeService extends AbstractService { } /** - * Get an unmodifiable list of services + * Get a cloned list of services * @return a list of child services at the time of invocation - * added services will not be picked up. */ public List getServices() { synchronized (serviceList) { - return Collections.unmodifiableList(serviceList); + return new ArrayList(serviceList); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java similarity index 66% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java index 3dbdc135ac4..f2ede7d9b65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/TestCompositeService.java @@ -16,26 +16,20 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.util; +package org.apache.hadoop.service; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service.STATE; +import org.junit.Before; +import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.service.BreakableService; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.service.ServiceStateException; -import org.apache.hadoop.service.Service.STATE; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.junit.Before; -import org.junit.Test; - public class TestCompositeService { private static final int NUM_OF_SERVICES = 5; @@ -156,7 +150,7 @@ public class TestCompositeService { try { serviceManager.start(); fail("Exception should have been thrown due to startup failure of last service"); - } catch (YarnRuntimeException e) { + } catch (ServiceTestRuntimeException e) { for (int i = 0; i < NUM_OF_SERVICES - 1; i++) { if (i >= FAILED_SERVICE_SEQ_NUMBER && STOP_ONLY_STARTED_SERVICES) { // Failed service state should be INITED @@ -197,7 +191,7 @@ public class TestCompositeService { // Stop the composite service try { serviceManager.stop(); - } catch (YarnRuntimeException e) { + } catch (ServiceTestRuntimeException e) { } assertInState(STATE.STOPPED, services); } @@ -335,7 +329,41 @@ public class TestCompositeService { testService.init(new Configuration()); assertEquals("Incorrect number of services", - 1, testService.getServices().size()); + 1, testService.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddInitedSiblingInInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.INITED)); + parent.init(new Configuration()); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddUninitedSiblingInInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.INITED)); + parent.init(new Configuration()); + try { + parent.start(); + fail("Expected an exception, got " + parent); + } catch (ServiceStateException e) { + //expected + } + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); } @Test @@ -365,6 +393,105 @@ public class TestCompositeService { 2, testService.getServices().size()); } + @Test(timeout = 1000) + public void testAddStartedChildBeforeInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService child = new BreakableService(); + child.init(new Configuration()); + child.start(); + AddSiblingService.addChildToService(parent, child); + try { + parent.init(new Configuration()); + fail("Expected an exception, got " + parent); + } catch (ServiceStateException e) { + //expected + } + parent.stop(); + } + + @Test(timeout = 1000) + public void testAddStoppedChildBeforeInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService child = new BreakableService(); + child.init(new Configuration()); + child.start(); + child.stop(); + AddSiblingService.addChildToService(parent, child); + try { + parent.init(new Configuration()); + fail("Expected an exception, got " + parent); + } catch (ServiceStateException e) { + //expected + } + parent.stop(); + } + + @Test(timeout = 1000) + public void testAddStartedSiblingInStart() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + sibling.start(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.STARTED)); + parent.init(new Configuration()); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddUninitedSiblingInStart() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.STARTED)); + parent.init(new Configuration()); + assertInState(STATE.NOTINITED, sibling); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + + @Test(timeout = 1000) + public void testAddStartedSiblingInInit() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + sibling.start(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.INITED)); + parent.init(new Configuration()); + assertInState(STATE.STARTED, sibling); + parent.start(); + assertInState(STATE.STARTED, sibling); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + assertInState(STATE.STOPPED, sibling); + } + + @Test(timeout = 1000) + public void testAddStartedSiblingInStop() throws Throwable { + CompositeService parent = new CompositeService("parent"); + BreakableService sibling = new BreakableService(); + sibling.init(new Configuration()); + sibling.start(); + parent.addService(new AddSiblingService(parent, + sibling, + STATE.STOPPED)); + parent.init(new Configuration()); + parent.start(); + parent.stop(); + assertEquals("Incorrect number of services", + 2, parent.getServices().size()); + } + public static class CompositeServiceAddingAChild extends CompositeService{ Service child; @@ -379,7 +506,18 @@ public class TestCompositeService { super.serviceInit(conf); } } - + + public static class ServiceTestRuntimeException extends RuntimeException { + public ServiceTestRuntimeException(String message) { + super(message); + } + } + + /** + * This is a composite service that keeps a count of the number of lifecycle + * events called, and can be set to throw a {@link ServiceTestRuntimeException } + * during service start or stop + */ public static class CompositeServiceImpl extends CompositeService { public static boolean isPolicyToStopOnlyStartedServices() { @@ -408,7 +546,7 @@ public class TestCompositeService { @Override protected void serviceStart() throws Exception { if (throwExceptionOnStart) { - throw new YarnRuntimeException("Fake service start exception"); + throw new ServiceTestRuntimeException("Fake service start exception"); } counter++; callSequenceNumber = counter; @@ -420,7 +558,7 @@ public class TestCompositeService { counter++; callSequenceNumber = counter; if (throwExceptionOnStop) { - throw new YarnRuntimeException("Fake service stop exception"); + throw new ServiceTestRuntimeException("Fake service stop exception"); } super.serviceStop(); } @@ -457,6 +595,9 @@ public class TestCompositeService { } + /** + * Composite service that makes the addService method public to all + */ public static class ServiceManager extends CompositeService { public void addTestService(CompositeService service) { @@ -468,4 +609,55 @@ public class TestCompositeService { } } + public static class AddSiblingService extends CompositeService { + private final CompositeService parent; + private final Service serviceToAdd; + private STATE triggerState; + + public AddSiblingService(CompositeService parent, + Service serviceToAdd, + STATE triggerState) { + super("ParentStateManipulatorService"); + this.parent = parent; + this.serviceToAdd = serviceToAdd; + this.triggerState = triggerState; + } + + /** + * Add the serviceToAdd to the parent if this service + * is in the state requested + */ + private void maybeAddSibling() { + if (getServiceState() == triggerState) { + parent.addService(serviceToAdd); + } + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + maybeAddSibling(); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + maybeAddSibling(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + maybeAddSibling(); + super.serviceStop(); + } + + /** + * Expose addService method + * @param parent parent service + * @param child child to add + */ + public static void addChildToService(CompositeService parent, Service child) { + parent.addService(child); + } + } } From ca72e11158047f3d00db9e9e61c632e757383287 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 19:07:55 +0000 Subject: [PATCH 03/42] YARN-1611. Introduced the concept of a configuration provider which can be used by ResourceManager to read configuration locally or from remote systems so as to help RM failover. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564002 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../yarn/conf/ConfigurationProvider.java | 64 +++++++ .../conf/ConfigurationProviderFactory.java | 57 ++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 13 ++ .../FileSystemBasedConfigurationProvider.java | 72 ++++++++ .../yarn/LocalConfigurationProvider.java | 48 +++++ .../src/main/resources/yarn-default.xml | 12 ++ .../server/resourcemanager/AdminService.java | 34 +++- .../scheduler/capacity/CapacityScheduler.java | 14 +- .../CapacitySchedulerConfiguration.java | 13 +- .../resourcemanager/TestRMAdminService.java | 171 ++++++++++++++++++ 11 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 84634df909c..4ef7f01c315 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -82,6 +82,10 @@ Release 2.4.0 - UNRELEASED YARN-1633. Defined user-facing entity, entity-info and event objects related to Application Timeline feature. (Zhijie Shen via vinodkv) + YARN-1611. Introduced the concept of a configuration provider which can be + used by ResourceManager to read configuration locally or from remote systems + so as to help RM failover. (Xuan Gong via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java new file mode 100644 index 00000000000..78c34d9de98 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.conf; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +/** + * Base class to implement ConfigurationProvider. + * Real ConfigurationProvider implementations need to derive from it and + * implement load methods to actually load the configuration. + */ +public abstract class ConfigurationProvider { + + public void init(Configuration conf) throws Exception { + initInternal(conf); + } + + public void close() throws Exception { + closeInternal(); + } + + /** + * Get the configuration. + * @param name The configuration file name + * @return configuration + * @throws YarnException + * @throws IOException + */ + public abstract Configuration getConfiguration(String name) + throws YarnException, IOException; + + /** + * Derived classes initialize themselves using this method. + */ + public abstract void initInternal(Configuration conf) throws Exception; + + /** + * Derived classes close themselves using this method. + */ + public abstract void closeInternal() throws Exception; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java new file mode 100644 index 00000000000..4adc72e1f11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.conf; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +@Private +@Unstable +/** + * Factory for {@link ConfigurationProvider} implementations. + */ +public class ConfigurationProviderFactory { + /** + * Creates an instance of {@link ConfigurationProvider} using given + * configuration. + * @param conf + * @return configurationProvider + */ + @SuppressWarnings("unchecked") + public static ConfigurationProvider + getConfigurationProvider(Configuration conf) { + Class defaultProviderClass; + try { + defaultProviderClass = (Class) + Class.forName( + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS); + } catch (Exception e) { + throw new YarnRuntimeException( + "Invalid default configuration provider class" + + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS, e); + } + ConfigurationProvider configurationProvider = ReflectionUtils.newInstance( + conf.getClass(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + defaultProviderClass, ConfigurationProvider.class), conf); + return configurationProvider; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 32665d79bb7..a324e97b297 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -37,6 +37,9 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; @Evolving public class YarnConfiguration extends Configuration { + @Private + public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; @@ -329,6 +332,16 @@ public class YarnConfiguration extends Configuration { public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; public static final String RM_HA_ID = RM_HA_PREFIX + "id"; + /** Store the related configuration files in File System */ + public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX + + "configuration.file-system-based-store"; + public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf"; + + public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX + + "configuration.provider-class"; + public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = + "org.apache.hadoop.yarn.LocalConfigurationProvider"; + @Private public static final List RM_SERVICES_ADDRESS_CONF_KEYS = Collections.unmodifiableList(Arrays.asList( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java new file mode 100644 index 00000000000..709f54a3529 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +public class FileSystemBasedConfigurationProvider + extends ConfigurationProvider { + + private static final Log LOG = LogFactory + .getLog(FileSystemBasedConfigurationProvider.class); + private FileSystem fs; + private Path configDir; + + @Override + public synchronized Configuration getConfiguration(String name) + throws IOException, YarnException { + Path configPath = new Path(this.configDir, name); + if (!fs.exists(configPath)) { + throw new YarnException("Can not find Configuration: " + name + " in " + + configDir); + } + Configuration conf = new Configuration(false); + conf.addResource(fs.open(configPath)); + return conf; + } + + @Override + public synchronized void initInternal(Configuration conf) throws Exception { + configDir = + new Path(conf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE, + YarnConfiguration.DEFAULT_FS_BASED_RM_CONF_STORE)); + fs = configDir.getFileSystem(conf); + if (!fs.exists(configDir)) { + fs.mkdirs(configDir); + } + } + + @Override + public synchronized void closeInternal() throws Exception { + fs.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java new file mode 100644 index 00000000000..d152c353f08 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +public class LocalConfigurationProvider extends ConfigurationProvider { + + @Override + public Configuration getConfiguration(String name) + throws IOException, YarnException { + return new Configuration(); + } + + @Override + public void initInternal(Configuration conf) throws Exception { + // Do nothing + } + + @Override + public void closeInternal() throws Exception { + // Do nothing + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c4937421f55..beb1d3005cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -588,6 +588,18 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore + + The class to use as the configuration provider. + If org.apache.hadoop.yarn.LocalConfigurationProvider is used, + the local configuration will be loaded. + If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used, + the configuration which will be loaded should be uploaded to remote File system first. + > + yarn.resourcemanager.configuration.provider-class + org.apache.hadoop.yarn.LocalConfigurationProvider + + + The hostname of the NM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 971603a90ed..c7fe0e2e6e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -47,6 +47,8 @@ import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -89,6 +91,8 @@ public class AdminService extends CompositeService implements private InetSocketAddress masterServiceAddress; private AccessControlList adminAcl; + private ConfigurationProvider configurationProvider = null; + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -109,6 +113,10 @@ public class AdminService extends CompositeService implements } } + this.configurationProvider = + ConfigurationProviderFactory.getConfigurationProvider(conf); + configurationProvider.init(conf); + masterServiceAddress = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, @@ -129,6 +137,9 @@ public class AdminService extends CompositeService implements @Override protected synchronized void serviceStop() throws Exception { stopServer(); + if (this.configurationProvider != null) { + configurationProvider.close(); + } super.serviceStop(); } @@ -295,23 +306,28 @@ public class AdminService extends CompositeService implements @Override public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshQueues"); + String argName = "refreshQueues"; + UserGroupInformation user = checkAcls(argName); if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh queues."); throwStandbyException(); } + RefreshQueuesResponse response = + recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { - rmContext.getScheduler().reinitialize(getConfig(), this.rmContext); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshQueues", + Configuration conf = + getConfiguration(YarnConfiguration.CS_CONFIGURATION_FILE); + rmContext.getScheduler().reinitialize(conf, this.rmContext); + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); - return recordFactory.newRecordInstance(RefreshQueuesResponse.class); + return response; } catch (IOException ioe) { LOG.info("Exception refreshing queues ", ioe); - RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "Exception refreshing queues"); throw RPCUtil.getRemoteException(ioe); @@ -483,5 +499,9 @@ public class AdminService extends CompositeService implements UpdateNodeResourceResponse.class); return response; } - + + private synchronized Configuration getConfiguration(String confFileName) + throws YarnException, IOException { + return this.configurationProvider.getConfiguration(confFileName); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 0197c5bf998..1b47d69fa2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -195,6 +195,7 @@ public class CapacityScheduler extends AbstractYarnScheduler private ResourceCalculator calculator; private boolean usePortForNodeName; + private boolean useLocalConfigurationProvider; public CapacityScheduler() {} @@ -261,7 +262,13 @@ public class CapacityScheduler extends AbstractYarnScheduler public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { - this.conf = new CapacitySchedulerConfiguration(conf); + this.useLocalConfigurationProvider = conf.get( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( + "org.apache.hadoop.yarn.LocalConfigurationProvider"); + this.conf = + new CapacitySchedulerConfiguration(conf, + this.useLocalConfigurationProvider); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); this.maximumAllocation = this.conf.getMaximumAllocation(); @@ -279,9 +286,10 @@ public class CapacityScheduler extends AbstractYarnScheduler "minimumAllocation=<" + getMinimumResourceCapability() + ">, " + "maximumAllocation=<" + getMaximumResourceCapability() + ">"); } else { - CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = new CapacitySchedulerConfiguration(conf); + this.conf = + new CapacitySchedulerConfiguration(conf, + this.useLocalConfigurationProvider); validateConf(this.conf); try { LOG.info("Re-initializing queues..."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 6fceabf0dec..267f819ffaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -140,10 +140,17 @@ public class CapacitySchedulerConfiguration extends Configuration { } public CapacitySchedulerConfiguration(Configuration configuration) { - super(configuration); - addResource(CS_CONFIGURATION_FILE); + this(configuration, true); } - + + public CapacitySchedulerConfiguration(Configuration configuration, + boolean useLocalConfigurationProvider) { + super(configuration); + if (useLocalConfigurationProvider) { + addResource(CS_CONFIGURATION_FILE); + } + } + private String getQueuePrefix(String queue) { String queueName = PREFIX + queue + DOT; return queueName; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java new file mode 100644 index 00000000000..d800f5067d9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import static org.junit.Assert.fail; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + +public class TestRMAdminService { + + private final Configuration configuration = new YarnConfiguration(); + private MockRM rm = null; + private FileSystem fs; + private Path workingPath; + private Path tmpDir; + + @Before + public void setup() throws IOException { + fs = FileSystem.get(configuration); + workingPath = + new Path(new File("target", this.getClass().getSimpleName() + + "-remoteDir").getAbsolutePath()); + configuration.set(YarnConfiguration.FS_BASED_RM_CONF_STORE, + workingPath.toString()); + tmpDir = new Path(new File("target", this.getClass().getSimpleName() + + "-tmpDir").getAbsolutePath()); + fs.delete(workingPath, true); + fs.delete(tmpDir, true); + fs.mkdirs(workingPath); + fs.mkdirs(tmpDir); + } + + @After + public void tearDown() throws IOException { + if (rm != null) { + rm.stop(); + } + fs.delete(workingPath, true); + fs.delete(tmpDir, true); + } + @Test + public void testAdminRefreshQueuesWithLocalConfigurationProvider() + throws IOException, YarnException { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + CapacityScheduler cs = + (CapacityScheduler) rm.getRMContext().getScheduler(); + int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + Assert.assertEquals(maxAppsBefore, cs.getConfiguration() + .getMaximumSystemApplications()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + CapacityScheduler cs = + (CapacityScheduler) rm.getRMContext().getScheduler(); + int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: capacity-scheduler.xml")); + } + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); + String csConfFile = writeConfigurationXML(csConf, + "capacity-scheduler.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(csConfFile)); + + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + + int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications(); + Assert.assertEquals(maxAppsAfter, 5000); + Assert.assertTrue(maxAppsAfter != maxAppsBefore); + } + + private String writeConfigurationXML(Configuration conf, String confXMLName) + throws IOException { + DataOutputStream output = null; + try { + final File confFile = new File(tmpDir.toString(), confXMLName); + if (confFile.exists()) { + confFile.delete(); + } + if (!confFile.createNewFile()) { + Assert.fail("Can not create " + confXMLName); + } + output = new DataOutputStream( + new FileOutputStream(confFile)); + conf.writeXml(output); + return confFile.getAbsolutePath(); + } finally { + if (output != null) { + output.close(); + } + } + } + + private void uploadToRemoteFileSystem(Path filePath) + throws IOException { + fs.copyFromLocalFile(filePath, workingPath); + } + + private void cleanRemoteDirectory() throws IOException { + if (fs.exists(workingPath)) { + for (FileStatus file : fs.listStatus(workingPath)) { + fs.delete(file.getPath(), true); + } + } + } +} From cb5e0787a6fc0b0748753b7e7c4c3fdbfd2714b2 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Mon, 3 Feb 2014 19:30:55 +0000 Subject: [PATCH 04/42] YARN-1498 addendum to fix findbugs warning git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564018 13f79535-47bb-0310-9956-ffa450edef68 --- .../resourcemanager/scheduler/SchedulerApplicationAttempt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 7e7fdb765e1..7785e56bdb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -432,7 +432,7 @@ public class SchedulerApplicationAttempt { .transferStateFromPreviousAppSchedulingInfo(appAttempt.appSchedulingInfo); } - public void move(Queue newQueue) { + public synchronized void move(Queue newQueue) { QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); String user = getUser(); From 3e7d56678c334a359049adf99018683113849fc8 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 19:53:46 +0000 Subject: [PATCH 05/42] YARN-1639. Modified RM HA configuration handling to have a way of not requiring separate configuration files for each RM. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564032 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/yarn/conf/HAUtil.java | 49 ++++++++++++++++--- .../src/main/resources/yarn-default.xml | 9 +++- .../yarn/server/resourcemanager/TestRMHA.java | 41 ++++++++++++++-- 4 files changed, 91 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4ef7f01c315..022c180b9c4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -103,6 +103,9 @@ Release 2.4.0 - UNRELEASED YARN-1617. Remove ancient comment and surround LOG.debug in AppSchedulingInfo.allocate (Sandy Ryza) + YARN-1639. Modified RM HA configuration handling to have a way of not + requiring separate configuration files for each RM. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index 3cbde93a3e9..ec2c64b5c2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -21,10 +21,13 @@ package org.apache.hadoop.yarn.conf; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import java.net.InetSocketAddress; import java.util.Collection; @InterfaceAudience.Private @@ -108,8 +111,7 @@ public class HAUtil { String errmsg = iae.getMessage(); if (confKey == null) { // Error at addSuffix - errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, - getRMHAId(conf)); + errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id); } throwBadConfigurationException(errmsg); } @@ -122,10 +124,18 @@ public class HAUtil { } private static void verifyAndSetCurrentRMHAId(Configuration conf) { - String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + String rmId = getRMHAId(conf); if (rmId == null) { - throwBadConfigurationException( - getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID)); + StringBuilder msg = new StringBuilder(); + msg.append("Can not find valid RM_HA_ID. None of "); + for (String id : conf + .getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) { + msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " "); + } + msg.append(" are matching" + + " the local address OR " + YarnConfiguration.RM_HA_ID + " is not" + + " specified in HA Configuration"); + throwBadConfigurationException(msg.toString()); } else { Collection ids = getRMHAIds(conf); if (!ids.contains(rmId)) { @@ -179,7 +189,34 @@ public class HAUtil { * @return RM Id on success */ public static String getRMHAId(Configuration conf) { - return conf.get(YarnConfiguration.RM_HA_ID); + int found = 0; + String currentRMId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + if(currentRMId == null) { + for(String rmId : getRMHAIds(conf)) { + String key = addSuffix(YarnConfiguration.RM_ADDRESS, rmId); + String addr = conf.get(key); + if (addr == null) { + continue; + } + InetSocketAddress s; + try { + s = NetUtils.createSocketAddr(addr); + } catch (Exception e) { + LOG.warn("Exception in creating socket address " + addr, e); + continue; + } + if (!s.isUnresolved() && NetUtils.isLocalAddress(s.getAddress())) { + currentRMId = rmId.trim(); + found++; + } + } + } + if (found > 1) { // Only one address must match the local address + String msg = "The HA Configuration has multiple addresses that match " + + "local node's address."; + throw new HadoopIllegalArgumentException(msg); + } + return currentRMId; } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index beb1d3005cb..78ceb64b3d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -395,7 +395,9 @@ the Active mode when prompted to. (2) The nodes in the RM ensemble are listed in yarn.resourcemanager.ha.rm-ids - (3) The id of each RM comes from yarn.resourcemanager.ha.id + (3) The id of each RM either comes from yarn.resourcemanager.ha.id + if yarn.resourcemanager.ha.id is explicitly specified or can be + figured out by matching yarn.resourcemanager.address.{id} with local address (4) The actual physical addresses come from the configs of the pattern - {rpc-config}.{id} yarn.resourcemanager.ha.enabled @@ -442,7 +444,10 @@ The id (string) of the current RM. When HA is enabled, this - is a required config. See description of yarn.resourcemanager.ha.enabled + is an optional config. The id of current RM can be set by explicitly + specifying yarn.resourcemanager.ha.id or figured out by matching + yarn.resourcemanager.address.{id} with local address + See description of yarn.resourcemanager.ha.enabled for full details on how this is used. yarn.resourcemanager.ha.id diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 9d1a46776c5..5b4f5709d75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -36,6 +36,8 @@ import org.junit.Test; import java.io.IOException; +import junit.framework.Assert; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -48,12 +50,15 @@ public class TestRMHA { private static final String STATE_ERR = "ResourceManager is in wrong HA state"; - private static final String RM1_ADDRESS = "0.0.0.0:0"; + private static final String RM1_ADDRESS = "1.1.1.1:1"; private static final String RM1_NODE_ID = "rm1"; - private static final String RM2_ADDRESS = "1.1.1.1:1"; + private static final String RM2_ADDRESS = "0.0.0.0:0"; private static final String RM2_NODE_ID = "rm2"; + private static final String RM3_ADDRESS = "2.2.2.2:2"; + private static final String RM3_NODE_ID = "rm3"; + @Before public void setUp() throws Exception { configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); @@ -61,8 +66,8 @@ public class TestRMHA { for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } - configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); } private void checkMonitorHealth() throws IOException { @@ -278,6 +283,36 @@ public class TestRMHA { rm.stop(); } + @Test + public void testHAIDLookup() { + //test implicitly lookup HA-ID + Configuration conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM2_NODE_ID); + + //test explicitly lookup HA-ID + configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM1_NODE_ID); + + //test if RM_HA_ID can not be found + configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID+ "," + RM3_NODE_ID); + configuration.unset(YarnConfiguration.RM_HA_ID); + conf = new YarnConfiguration(configuration); + try { + rm = new MockRM(conf); + rm.init(conf); + fail("Should get an exception here."); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Invalid configuration! Can not find valid RM_HA_ID.")); + } + } + @SuppressWarnings("rawtypes") class MyCountingDispatcher extends AbstractService implements Dispatcher { From eac066b13f1d2882415d6888d95d9fe87c665dcc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 20:07:01 +0000 Subject: [PATCH 06/42] YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for implementing different storage impls for storing timeline information. Contributed by Billie Rinaldi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564037 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../api/records/apptimeline/ATSEntity.java | 41 +++-- .../api/records/apptimeline/ATSEvent.java | 40 ++++- .../api/records/apptimeline/ATSEvents.java | 2 +- .../api/records/apptimeline/ATSPutErrors.java | 163 ++++++++++++++++++ .../TestApplicationTimelineRecords.java | 43 ++++- .../ApplicationTimelineReader.java | 125 ++++++++++++++ .../apptimeline/ApplicationTimelineStore.java | 29 ++++ .../ApplicationTimelineWriter.java | 43 +++++ .../apptimeline/NameValuePair.java | 59 +++++++ .../apptimeline/package-info.java | 20 +++ 11 files changed, 541 insertions(+), 28 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 022c180b9c4..b07ecd77144 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -86,6 +86,10 @@ Release 2.4.0 - UNRELEASED used by ResourceManager to read configuration locally or from remote systems so as to help RM failover. (Xuan Gong via vinodkv) + YARN-1659. Defined the ApplicationTimelineStore store as an abstraction for + implementing different storage impls for storing timeline information. + (Billie Rinaldi via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java index 1884db7ac13..6b3ea1013e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -53,10 +54,10 @@ public class ATSEntity { private String entityType; private String entityId; - private long startTime; + private Long startTime; private List events = new ArrayList(); - private Map> relatedEntities = - new HashMap>(); + private Map> relatedEntities = + new HashMap>(); private Map primaryFilters = new HashMap(); private Map otherInfo = @@ -112,7 +113,7 @@ public class ATSEntity { * @return the start time of the entity */ @XmlElement(name = "starttime") - public long getStartTime() { + public Long getStartTime() { return startTime; } @@ -122,7 +123,7 @@ public class ATSEntity { * @param startTime * the start time of the entity */ - public void setStartTime(long startTime) { + public void setStartTime(Long startTime) { this.startTime = startTime; } @@ -172,26 +173,25 @@ public class ATSEntity { * @return the related entities */ @XmlElement(name = "relatedentities") - public Map> getRelatedEntities() { + public Map> getRelatedEntities() { return relatedEntities; } /** - * Add a list of entity of the same type to the existing related entity map + * Add an entity to the existing related entity map * * @param entityType * the entity type - * @param entityIds - * a list of entity Ids + * @param entityId + * the entity Id */ - public void addRelatedEntity(String entityType, List entityIds) { - List thisRelatedEntity = relatedEntities.get(entityType); - relatedEntities.put(entityType, entityIds); + public void addRelatedEntity(String entityType, String entityId) { + List thisRelatedEntity = relatedEntities.get(entityType); if (thisRelatedEntity == null) { - relatedEntities.put(entityType, entityIds); - } else { - thisRelatedEntity.addAll(entityIds); + thisRelatedEntity = new ArrayList(); + relatedEntities.put(entityType, thisRelatedEntity); } + thisRelatedEntity.add(entityId); } /** @@ -200,11 +200,10 @@ public class ATSEntity { * @param relatedEntities * a map of related entities */ - public void addRelatedEntities( - Map> relatedEntities) { - for (Map.Entry> relatedEntity : relatedEntities - .entrySet()) { - List thisRelatedEntity = + public void addRelatedEntities(Map> relatedEntities) { + for (Entry> relatedEntity : + relatedEntities.entrySet()) { + List thisRelatedEntity = this.relatedEntities.get(relatedEntity.getKey()); if (thisRelatedEntity == null) { this.relatedEntities.put( @@ -222,7 +221,7 @@ public class ATSEntity { * a map of related entities */ public void setRelatedEntities( - Map> relatedEntities) { + Map> relatedEntities) { this.relatedEntities = relatedEntities; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java index 6477a578e6a..27bac16a69f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java @@ -39,7 +39,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; @XmlAccessorType(XmlAccessType.NONE) @Public @Unstable -public class ATSEvent { +public class ATSEvent implements Comparable { private long timestamp; private String eventType; @@ -131,4 +131,42 @@ public class ATSEvent { this.eventInfo = eventInfo; } + @Override + public int compareTo(ATSEvent other) { + if (timestamp > other.timestamp) { + return -1; + } else if (timestamp < other.timestamp) { + return 1; + } else { + return eventType.compareTo(other.eventType); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ATSEvent atsEvent = (ATSEvent) o; + + if (timestamp != atsEvent.timestamp) + return false; + if (!eventType.equals(atsEvent.eventType)) + return false; + if (eventInfo != null ? !eventInfo.equals(atsEvent.eventInfo) : + atsEvent.eventInfo != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + eventType.hashCode(); + result = 31 * result + (eventInfo != null ? eventInfo.hashCode() : 0); + return result; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java index da7fd280886..a08537d7203 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java @@ -160,7 +160,7 @@ public class ATSEvents { * @param event * a single event */ - public void addEntity(ATSEvent event) { + public void addEvent(ATSEvent event) { events.add(event); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java new file mode 100644 index 00000000000..91458e1419f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSPutErrors.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * A class that holds a list of put errors. This is the response returned + * when a list of {@link ATSEntity} objects is added to the application + * timeline. If there are errors in storing individual entity objects, + * they will be indicated in the list of errors. + */ +@XmlRootElement(name = "errors") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class ATSPutErrors { + + private List errors = new ArrayList(); + + public ATSPutErrors() { + + } + + /** + * Get a list of {@link ATSPutError} instances + * + * @return a list of {@link ATSPutError} instances + */ + @XmlElement(name = "errors") + public List getErrors() { + return errors; + } + + /** + * Add a single {@link ATSPutError} instance into the existing list + * + * @param error + * a single {@link ATSPutError} instance + */ + public void addError(ATSPutError error) { + errors.add(error); + } + + /** + * Add a list of {@link ATSPutError} instances into the existing list + * + * @param errors + * a list of {@link ATSPutError} instances + */ + public void addErrors(List errors) { + this.errors.addAll(errors); + } + + /** + * Set the list to the given list of {@link ATSPutError} instances + * + * @param errors + * a list of {@link ATSPutError} instances + */ + public void setErrors(List errors) { + this.errors.clear(); + this.errors.addAll(errors); + } + + /** + * A class that holds the error code for one entity. + */ + @XmlRootElement(name = "error") + @XmlAccessorType(XmlAccessType.NONE) + @Public + @Unstable + public static class ATSPutError { + private String entityId; + private String entityType; + private Integer errorCode; + + /** + * Get the entity Id + * + * @return the entity Id + */ + @XmlElement(name = "entity") + public String getEntityId() { + return entityId; + } + + /** + * Set the entity Id + * + * @param entityId + * the entity Id + */ + public void setEntityId(String entityId) { + this.entityId = entityId; + } + + /** + * Get the entity type + * + * @return the entity type + */ + @XmlElement(name = "entitytype") + public String getEntityType() { + return entityType; + } + + /** + * Set the entity type + * + * @param entityType + * the entity type + */ + public void setEntityType(String entityType) { + this.entityType = entityType; + } + + /** + * Get the error code + * + * @return an error code + */ + @XmlElement(name = "errorcode") + public Integer getErrorCode() { + return errorCode; + } + + /** + * Set the error code to the given error code + * + * @param errorCode + * an error code + */ + public void setErrorCode(Integer errorCode) { + this.errorCode = errorCode; + } + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java index fe79e74eb6c..f2a6d3ef461 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java @@ -18,10 +18,13 @@ package org.apache.hadoop.yarn.api.records.apptimeline; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import junit.framework.Assert; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; import org.junit.Test; public class TestApplicationTimelineRecords { @@ -42,10 +45,8 @@ public class TestApplicationTimelineRecords { event.addEventInfo("key2", "val2"); entity.addEvent(event); } - entity.addRelatedEntity( - "test ref type 1", Arrays.asList((Object) "test ref id 1")); - entity.addRelatedEntity( - "test ref type 2", Arrays.asList((Object) "test ref id 2")); + entity.addRelatedEntity("test ref type 1", "test ref id 1"); + entity.addRelatedEntity("test ref type 2", "test ref id 2"); entity.addPrimaryFilter("pkey1", "pval1"); entity.addPrimaryFilter("pkey2", "pval2"); entity.addOtherInfo("okey1", "oval1"); @@ -83,7 +84,7 @@ public class TestApplicationTimelineRecords { event.setEventType("event type " + i); event.addEventInfo("key1", "val1"); event.addEventInfo("key2", "val2"); - partEvents.addEntity(event); + partEvents.addEvent(event); } events.addEvent(partEvents); } @@ -110,4 +111,36 @@ public class TestApplicationTimelineRecords { Assert.assertEquals(2, event22.getEventInfo().size()); } + @Test + public void testATSPutErrors() { + ATSPutErrors atsPutErrors = new ATSPutErrors(); + ATSPutError error1 = new ATSPutError(); + error1.setEntityId("entity id 1"); + error1.setEntityId("entity type 1"); + error1.setErrorCode(1); + atsPutErrors.addError(error1); + List errors = new ArrayList(); + errors.add(error1); + ATSPutError error2 = new ATSPutError(); + error2.setEntityId("entity id 2"); + error2.setEntityId("entity type 2"); + error2.setErrorCode(2); + errors.add(error2); + atsPutErrors.addErrors(errors); + + Assert.assertEquals(3, atsPutErrors.getErrors().size()); + ATSPutError e = atsPutErrors.getErrors().get(0); + Assert.assertEquals(error1.getEntityId(), e.getEntityId()); + Assert.assertEquals(error1.getEntityType(), e.getEntityType()); + Assert.assertEquals(error1.getErrorCode(), e.getErrorCode()); + e = atsPutErrors.getErrors().get(1); + Assert.assertEquals(error1.getEntityId(), e.getEntityId()); + Assert.assertEquals(error1.getEntityType(), e.getEntityType()); + Assert.assertEquals(error1.getErrorCode(), e.getErrorCode()); + e = atsPutErrors.getErrors().get(2); + Assert.assertEquals(error2.getEntityId(), e.getEntityId()); + Assert.assertEquals(error2.getEntityType(), e.getEntityType()); + Assert.assertEquals(error2.getErrorCode(), e.getErrorCode()); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java new file mode 100644 index 00000000000..97a217dc98a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineReader.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Set; +import java.util.SortedSet; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; + +/** + * This interface is for retrieving application timeline information. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface ApplicationTimelineReader { + + /** + * Possible fields to retrieve for {@link #getEntities} and {@link + * #getEntity}. + */ + enum Field { + EVENTS, + RELATED_ENTITIES, + PRIMARY_FILTERS, + OTHER_INFO, + LAST_EVENT_ONLY + } + + /** + * Default limit for {@link #getEntities} and {@link #getEntityTimelines}. + */ + final long DEFAULT_LIMIT = 100; + + /** + * This method retrieves a list of entity information, {@link ATSEntity}, + * sorted by the starting timestamp for the entity, descending. + * + * @param entityType The type of entities to return (required). + * @param limit A limit on the number of entities to return. If null, + * defaults to {@link #DEFAULT_LIMIT}. + * @param windowStart The earliest start timestamp to retrieve (exclusive). + * If null, defaults to retrieving all entities until the + * limit is reached. + * @param windowEnd The latest start timestamp to retrieve (inclusive). + * If null, defaults to {@link Long#MAX_VALUE} + * @param primaryFilter Retrieves only entities that have the specified + * primary filter. If null, retrieves all entities. + * This is an indexed retrieval, and no entities that + * do not match the filter are scanned. + * @param secondaryFilters Retrieves only entities that have exact matches + * for all the specified filters in their primary + * filters or other info. This is not an indexed + * retrieval, so all entities are scanned but only + * those matching the filters are returned. + * @param fieldsToRetrieve Specifies which fields of the entity object to + * retrieve (see {@link Field}). If the set of fields + * contains {@link Field#LAST_EVENT_ONLY} and not + * {@link Field#EVENTS}, the most recent event for + * each entity is retrieved. + * @return An {@link ATSEntities} object. + */ + ATSEntities getEntities(String entityType, + Long limit, Long windowStart, Long windowEnd, + NameValuePair primaryFilter, Collection secondaryFilters, + EnumSet fieldsToRetrieve); + + /** + * This method retrieves the entity information for a given entity. + * + * @param entity The entity whose information will be retrieved. + * @param entityType The type of the entity. + * @param fieldsToRetrieve Specifies which fields of the entity object to + * retrieve (see {@link Field}). If the set of + * fields contains {@link Field#LAST_EVENT_ONLY} and + * not {@link Field#EVENTS}, the most recent event + * for each entity is retrieved. + * @return An {@link ATSEntity} object. + */ + ATSEntity getEntity(String entity, String entityType, EnumSet + fieldsToRetrieve); + + /** + * This method retrieves the events for a list of entities all of the same + * entity type. The events for each entity are sorted in order of their + * timestamps, descending. + * + * @param entityType The type of entities to retrieve events for. + * @param entityIds The entity IDs to retrieve events for. + * @param limit A limit on the number of events to return for each entity. + * If null, defaults to {@link #DEFAULT_LIMIT} events per + * entity. + * @param windowStart If not null, retrieves only events later than the + * given time (exclusive) + * @param windowEnd If not null, retrieves only events earlier than the + * given time (inclusive) + * @param eventTypes Restricts the events returned to the given types. If + * null, events of all types will be returned. + * @return An {@link ATSEvents} object. + */ + ATSEvents getEntityTimelines(String entityType, + SortedSet entityIds, Long limit, Long windowStart, + Long windowEnd, Set eventTypes); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java new file mode 100644 index 00000000000..b231418a89f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStore.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.service.Service; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface ApplicationTimelineStore extends + Service, ApplicationTimelineReader, ApplicationTimelineWriter { +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java new file mode 100644 index 00000000000..b7bd0708e43 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineWriter.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; + +/** + * This interface is for storing application timeline information. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface ApplicationTimelineWriter { + + /** + * Stores entity information to the application timeline store. Any errors + * occurring for individual put request objects will be reported in the + * response. + * + * @param data An {@link ATSEntities} object. + * @return An {@link ATSPutErrors} object. + */ + ATSPutErrors put(ATSEntities data); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java new file mode 100644 index 00000000000..66a21bbf658 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/NameValuePair.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A class holding a name and value pair, used for specifying filters in + * {@link ApplicationTimelineReader}. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class NameValuePair { + String name; + Object value; + + public NameValuePair(String name, Object value) { + this.name = name; + this.value = value; + } + + /** + * Get the name. + * @return The name. + */ + public String getName() { + + return name; + } + + /** + * Get the value. + * @return The value. + */ + public Object getValue() { + return value; + } + + @Override + public String toString() { + return "{ name: " + name + ", value: " + value + " }"; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java new file mode 100644 index 00000000000..c3aaafe6747 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/package-info.java @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; +import org.apache.hadoop.classification.InterfaceAudience; From eff1e809f284d50ee49bb9ed2acea52331838c5e Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 20:16:28 +0000 Subject: [PATCH 07/42] YARN-1668. Modified RM HA handling of admin-acls to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564043 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 4 +- .../server/resourcemanager/AdminService.java | 22 ++++++-- .../resourcemanager/TestRMAdminService.java | 56 ++++++++++++++++++- 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b07ecd77144..9a947f713cd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -110,6 +110,10 @@ Release 2.4.0 - UNRELEASED YARN-1639. Modified RM HA configuration handling to have a way of not requiring separate configuration files for each RM. (Xuan Gong via vinodkv) + YARN-1668. Modified RM HA handling of admin-acls to be available across RM + failover by making using of a remote configuration-provider. (Xuan Gong via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a324e97b297..af385f81db2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -40,8 +40,10 @@ public class YarnConfiguration extends Configuration { @Private public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + @Private + public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; - private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; static { Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index c7fe0e2e6e9..9a33b706f90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRespo import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.BlockingService; public class AdminService extends CompositeService implements @@ -407,14 +408,22 @@ public class AdminService extends CompositeService implements @Override public RefreshAdminAclsResponse refreshAdminAcls( - RefreshAdminAclsRequest request) throws YarnException { - UserGroupInformation user = checkAcls("refreshAdminAcls"); + RefreshAdminAclsRequest request) throws YarnException, IOException { + String argName = "refreshAdminAcls"; + UserGroupInformation user = checkAcls(argName); - Configuration conf = new Configuration(); + if (!isRMActive()) { + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh user-groups."); + throwStandbyException(); + } + Configuration conf = + getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE); adminAcl = new AccessControlList(conf.get( YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); - RMAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls", + RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class); @@ -504,4 +513,9 @@ public class AdminService extends CompositeService implements throws YarnException, IOException { return this.configurationProvider.getConfiguration(confFileName); } + + @VisibleForTesting + public AccessControlList getAccessControlList() { + return this.adminAcl; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index d800f5067d9..4b7018528fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -50,6 +51,7 @@ public class TestRMAdminService { @Before public void setup() throws IOException { + Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); fs = FileSystem.get(configuration); workingPath = new Path(new File("target", this.getClass().getSimpleName() @@ -72,6 +74,7 @@ public class TestRMAdminService { fs.delete(workingPath, true); fs.delete(tmpDir, true); } + @Test public void testAdminRefreshQueuesWithLocalConfigurationProvider() throws IOException, YarnException { @@ -95,7 +98,6 @@ public class TestRMAdminService { @Test public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { - Configuration.addDefaultResource(YarnConfiguration.CS_CONFIGURATION_FILE); configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); rm = new MockRM(configuration); @@ -134,6 +136,58 @@ public class TestRMAdminService { Assert.assertTrue(maxAppsAfter != maxAppsBefore); } + @Test + public void testAdminAclsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void testAdminAclsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: yarn-site.xml")); + } + + String aclStringBefore = + rm.adminService.getAccessControlList().getAclString().trim(); + + YarnConfiguration yarnConf = new YarnConfiguration(); + yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda"); + String yarnConfFile = writeConfigurationXML(yarnConf, "yarn-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(yarnConfFile)); + rm.adminService.refreshAdminAcls(RefreshAdminAclsRequest.newInstance()); + + String aclStringAfter = + rm.adminService.getAccessControlList().getAclString().trim(); + + Assert.assertTrue(!aclStringAfter.equals(aclStringBefore)); + Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; From 24fa232707b56f6ccc88a460dc0d708d35a28ff4 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 22:10:56 +0000 Subject: [PATCH 08/42] YARN-1667. Modified RM HA handling of super users (with proxying ability) to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564100 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/security/authorize/ProxyUsers.java | 11 ++++ hadoop-yarn-project/CHANGES.txt | 33 ++++++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../server/resourcemanager/AdminService.java | 15 ++--- .../resourcemanager/TestRMAdminService.java | 61 +++++++++++++++++++ 5 files changed, 116 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java index 341285e1a75..52952588739 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java @@ -30,6 +30,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class ProxyUsers { @@ -177,4 +179,13 @@ public class ProxyUsers { (list.contains("*")); } + @VisibleForTesting + public static Map> getProxyGroups() { + return proxyGroups; + } + + @VisibleForTesting + public static Map> getProxyHosts() { + return proxyHosts; + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9a947f713cd..e4b8900843a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -114,6 +114,39 @@ Release 2.4.0 - UNRELEASED failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + YARN-1667. Modified RM HA handling of super users (with proxying ability) to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + + OPTIMIZATIONS + + BUG FIXES + + YARN-935. Correcting pom.xml to build applicationhistoryserver module + successfully. (Zhijie Shen via vinodkv) + + YARN-962. Fixed bug in application-history proto file and renamed it be just + a client proto file. (Zhijie Shen via vinodkv) + + YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl + package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv) + + YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv) + + YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar + Vavilapalli via mayank) + + YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to + YARN-888. (Vinod Kumar Vavilapalli via zjshen) + + YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli + via zjshen) + + YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli + via zjshen) + + YARN-1595. Made enabling history service configurable and fixed test failures on + branch YARN-321. (Vinod Kumar Vavilapalli via zjshen) OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index af385f81db2..919ed901439 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -43,6 +43,9 @@ public class YarnConfiguration extends Configuration { @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + @Private + public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; static { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 9a33b706f90..3bfd47d1373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -363,21 +363,22 @@ public class AdminService extends CompositeService implements @Override public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration"); + throws YarnException, IOException { + String argName = "refreshSuperUserGroupsConfiguration"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling super-user-groups on Standby RM if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh super-user-groups."); throwStandbyException(); } - ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration()); + Configuration conf = + getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", "AdminService"); + argName, "AdminService"); return recordFactory.newRecordInstance( RefreshSuperUserGroupsConfigurationResponse.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 4b7018528fe..797b4226842 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -29,10 +29,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -188,6 +190,65 @@ public class TestRMAdminService { Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); } + @Test + public void + testRefreshSuperUserGroupsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void + testRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: core-site.xml")); + } + + Configuration coreConf = new Configuration(false); + coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); + coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); + String coreConfFile = writeConfigurationXML(coreConf, + "core-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(coreConfFile)); + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + Assert.assertTrue(ProxyUsers.getProxyGroups() + .get("hadoop.proxyuser.test.groups").size() == 1); + Assert.assertTrue(ProxyUsers.getProxyGroups() + .get("hadoop.proxyuser.test.groups").contains("test_groups")); + + Assert.assertTrue(ProxyUsers.getProxyHosts() + .get("hadoop.proxyuser.test.hosts").size() == 1); + Assert.assertTrue(ProxyUsers.getProxyHosts() + .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null; From 60c0fe8f6d9fcd1bbfeef2f81afe490cbabad4a7 Mon Sep 17 00:00:00 2001 From: Owen O'Malley Date: Tue, 4 Feb 2014 00:11:54 +0000 Subject: [PATCH 09/42] HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry McCay via omalley) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564137 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/crypto/key/TestKeyShell.java | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d4db1854569..06d97b1428e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -113,6 +113,9 @@ Trunk (Unreleased) HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley) + HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry + McCay via omalley) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java index 2d2f7ee431b..a05e9bbc2b5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java @@ -41,7 +41,7 @@ public class TestKeyShell { @Test public void testKeySuccessfulKeyLifecycle() throws Exception { - outContent.flush(); + outContent.reset(); String[] args1 = {"create", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; int rc = 0; @@ -52,14 +52,14 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "created.")); - outContent.flush(); + outContent.reset(); String[] args2 = {"list", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args2); assertEquals(0, rc); assertTrue(outContent.toString().contains("key1")); - outContent.flush(); + outContent.reset(); String[] args3 = {"roll", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args3); @@ -67,7 +67,7 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "rolled.")); - outContent.flush(); + outContent.reset(); String[] args4 = {"delete", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args4); @@ -75,12 +75,12 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "deleted.")); - outContent.flush(); + outContent.reset(); String[] args5 = {"list", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args5); assertEquals(0, rc); - assertTrue(outContent.toString().contains("key1")); + assertFalse(outContent.toString(), outContent.toString().contains("key1")); } @Test @@ -165,7 +165,7 @@ public class TestKeyShell { assertTrue(outContent.toString().contains("key1 has been successfully " + "created.")); - outContent.flush(); + outContent.reset(); String[] args2 = {"delete", "key1", "--provider", "jceks://file" + tmpDir + "/keystore.jceks"}; rc = ks.run(args2); From 6439cd0f691069cefb6da4ba261ffe60cc13bbd0 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 4 Feb 2014 00:27:25 +0000 Subject: [PATCH 10/42] HDFS-5767. NFS implementation assumes userName userId mapping to be unique, which is not true sometimes. Contributed by Yongjun Zhang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564141 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 4 +- .../apache/hadoop/nfs/nfs3/IdUserGroup.java | 52 +++++++++++-------- .../hadoop/nfs/nfs3/TestIdUserGroup.java | 41 +++++++++------ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 10fcdb785e7..7143b514060 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -425,9 +425,9 @@ checkJavadocWarnings () { echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." - #There are 14 warnings that are caused by things that are caused by using sun internal APIs. + #There are 12 warnings that are caused by things that are caused by using sun internal APIs. #There are 2 warnings that are caused by the Apache DS Dn class used in MiniKdc. - OK_JAVADOC_WARNINGS=16; + OK_JAVADOC_WARNINGS=14; ### if current warnings greater than OK_JAVADOC_WARNINGS if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java index a1d48aadc85..bf2b542d85b 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java @@ -50,14 +50,6 @@ public class IdUserGroup { private BiMap gidNameMap = HashBiMap.create(); private long lastUpdateTime = 0; // Last time maps were updated - - static public class DuplicateNameOrIdException extends IOException { - private static final long serialVersionUID = 1L; - - public DuplicateNameOrIdException(String msg) { - super(msg); - } - } public IdUserGroup() throws IOException { updateMaps(); @@ -80,7 +72,8 @@ public class IdUserGroup { } } - private static final String DUPLICATE_NAME_ID_DEBUG_INFO = "NFS gateway can't start with duplicate name or id on the host system.\n" + private static final String DUPLICATE_NAME_ID_DEBUG_INFO = + "NFS gateway could have problem starting with duplicate name or id on the host system.\n" + "This is because HDFS (non-kerberos cluster) uses name as the only way to identify a user or group.\n" + "The host system with duplicated user/group name or id might work fine most of the time by itself.\n" + "However when NFS gateway talks to HDFS, HDFS accepts only user and group name.\n" @@ -88,6 +81,16 @@ public class IdUserGroup { + " and on Linux systms,\n" + " and on MacOS."; + private static void reportDuplicateEntry(final String header, + final Integer key, final String value, + final Integer ekey, final String evalue) { + LOG.warn("\n" + header + String.format( + "new entry (%d, %s), existing entry: (%d, %s).\n%s\n%s", + key, value, ekey, evalue, + "The new entry is to be ignored for the following reason.", + DUPLICATE_NAME_ID_DEBUG_INFO)); + } + /** * Get the whole list of users and groups and save them in the maps. * @throws IOException @@ -108,22 +111,27 @@ public class IdUserGroup { } LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]); // HDFS can't differentiate duplicate names with simple authentication - Integer key = Integer.valueOf(nameId[1]); - String value = nameId[0]; + final Integer key = Integer.valueOf(nameId[1]); + final String value = nameId[0]; if (map.containsKey(key)) { - LOG.error(String.format( - "Got duplicate id:(%d, %s), existing entry: (%d, %s).\n%s", key, - value, key, map.get(key), DUPLICATE_NAME_ID_DEBUG_INFO)); - throw new DuplicateNameOrIdException("Got duplicate id."); + final String prevValue = map.get(key); + if (value.equals(prevValue)) { + // silently ignore equivalent entries + continue; + } + reportDuplicateEntry( + "Got multiple names associated with the same id: ", + key, value, key, prevValue); + continue; } - if (map.containsValue(nameId[0])) { - LOG.error(String.format( - "Got duplicate name:(%d, %s), existing entry: (%d, %s) \n%s", - key, value, map.inverse().get(value), value, - DUPLICATE_NAME_ID_DEBUG_INFO)); - throw new DuplicateNameOrIdException("Got duplicate name"); + if (map.containsValue(value)) { + final Integer prevKey = map.inverse().get(value); + reportDuplicateEntry( + "Got multiple ids associated with the same name: ", + key, value, prevKey, value); + continue; } - map.put(Integer.valueOf(nameId[1]), nameId[0]); + map.put(key, value); } LOG.info("Updated " + mapName + " map size:" + map.size()); diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java index db2b27016e4..4331238b166 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java @@ -17,11 +17,10 @@ */ package org.apache.hadoop.nfs.nfs3; -import static org.junit.Assert.fail; - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; -import org.apache.hadoop.nfs.nfs3.IdUserGroup.DuplicateNameOrIdException; import org.junit.Test; import com.google.common.collect.BiMap; @@ -33,24 +32,36 @@ public class TestIdUserGroup { public void testDuplicates() throws IOException { String GET_ALL_USERS_CMD = "echo \"root:x:0:0:root:/root:/bin/bash\n" + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" - + "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\"" + + "hdfs:x:11502:10788:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "hdfs1:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "hdfs2:x:11502:10787:Grid Distributed File System:/home/hdfs:/bin/bash\n" + + "bin:x:2:2:bin:/bin:/bin/sh\n" + + "bin:x:1:1:bin:/bin:/sbin/nologin\n" + + "daemon:x:1:1:daemon:/usr/sbin:/bin/sh\n" + + "daemon:x:2:2:daemon:/sbin:/sbin/nologin\"" + " | cut -d: -f1,3"; String GET_ALL_GROUPS_CMD = "echo \"hdfs:*:11501:hrt_hdfs\n" - + "mapred:x:497\n" + "mapred2:x:497\"" + " | cut -d: -f1,3"; + + "mapred:x:497\n" + + "mapred2:x:497\n" + + "mapred:x:498\n" + + "mapred3:x:498\"" + + " | cut -d: -f1,3"; // Maps for id to name map BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - try { - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); - fail("didn't detect the duplicate name"); - } catch (DuplicateNameOrIdException e) { - } + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); + assertTrue(uMap.size() == 5); + assertEquals(uMap.get(0), "root"); + assertEquals(uMap.get(11501), "hdfs"); + assertEquals(uMap.get(11502), "hdfs2"); + assertEquals(uMap.get(2), "bin"); + assertEquals(uMap.get(1), "daemon"); - try { - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); - fail("didn't detect the duplicate id"); - } catch (DuplicateNameOrIdException e) { - } + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); + assertTrue(gMap.size() == 3); + assertEquals(gMap.get(11501), "hdfs"); + assertEquals(gMap.get(497), "mapred"); + assertEquals(gMap.get(498), "mapred3"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0b9caec75b6..f9abed76c5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -328,6 +328,9 @@ Release 2.4.0 - UNRELEASED the same node group when dfs.namenode.avoid.write.stale.datanode is true. (Buddy via junping_du) + HDFS-5767. NFS implementation assumes userName userId mapping to be unique, + which is not true sometimes (Yongjun Zhang via brandonli) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES From dd7fa037192d7a64bc29fc184c080efbbb14c9cb Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 04:05:58 +0000 Subject: [PATCH 11/42] Fix YARN's CHANGES.txt issue caused during the commit for YARN-1667. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564185 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e4b8900843a..7d7542dfbd5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -118,37 +118,6 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) - OPTIMIZATIONS - - BUG FIXES - - YARN-935. Correcting pom.xml to build applicationhistoryserver module - successfully. (Zhijie Shen via vinodkv) - - YARN-962. Fixed bug in application-history proto file and renamed it be just - a client proto file. (Zhijie Shen via vinodkv) - - YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl - package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv) - - YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv) - - YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar - Vavilapalli via mayank) - - YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to - YARN-888. (Vinod Kumar Vavilapalli via zjshen) - - YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli - via zjshen) - - YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli - via zjshen) - - YARN-1595. Made enabling history service configurable and fixed test failures on - branch YARN-321. (Vinod Kumar Vavilapalli via zjshen) - OPTIMIZATIONS - BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module From a8c780d378df86aafba09751c0c43dd4e0d54c0a Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 04:08:09 +0000 Subject: [PATCH 12/42] YARN-1673. Fix option parsing in YARN's application CLI after it is broken by YARN-967. Contributed by Mayank Bansal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564188 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/yarn/client/cli/ApplicationCLI.java | 2 +- .../org/apache/hadoop/yarn/client/cli/TestYarnCLI.java | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7d7542dfbd5..4110dfe7421 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -165,6 +165,9 @@ Release 2.4.0 - UNRELEASED YARN-1632. TestApplicationMasterServices should be under org.apache.hadoop.yarn.server.resourcemanager package (Chen He via jeagles) + YARN-1673. Fix option parsing in YARN's application CLI after it is broken + by YARN-967. (Mayank Bansal via vinodkv) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 9b465b78b28..d520866e51c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -197,7 +197,7 @@ public class ApplicationCLI extends YarnCLI { listApplications(appTypes, appStates); } } else if (cliParser.hasOption(KILL_CMD)) { - if (args.length != 2) { + if (args.length != 3) { printUsage(opts); return exitCode; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index dd6be0d8a57..48ac5484893 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -681,7 +681,7 @@ public class TestYarnCLI { sysOutStream.reset(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); result = - cli.run(new String[] { "-kill", applicationId.toString(), "args" }); + cli.run(new String[] {"application", "-kill", applicationId.toString(), "args" }); verify(spyCli).printUsage(any(Options.class)); Assert.assertEquals(createApplicationCLIHelpMessage(), sysOutStream.toString()); @@ -717,7 +717,7 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); - int result = cli.run(new String[] { "-kill", applicationId.toString() }); + int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); assertEquals(0, result); verify(client, times(0)).killApplication(any(ApplicationId.class)); verify(sysOut).println( @@ -730,7 +730,7 @@ public class TestYarnCLI { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); - result = cli.run(new String[] { "-kill", applicationId.toString() }); + result = cli.run(new String[] { "application","-kill", applicationId.toString() }); assertEquals(0, result); verify(client).killApplication(any(ApplicationId.class)); verify(sysOut).println("Killing application application_1234_0005"); @@ -740,7 +740,7 @@ public class TestYarnCLI { .getApplicationReport(applicationId); cli = createAndGetAppCLI(); try { - cli.run(new String[] { "-kill", applicationId.toString() }); + cli.run(new String[] { "application","-kill", applicationId.toString() }); Assert.fail(); } catch (Exception ex) { Assert.assertTrue(ex instanceof ApplicationNotFoundException); From 39ecc46ff6c4d11175f6995289e9697bc23436c5 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 04:21:40 +0000 Subject: [PATCH 13/42] YARN-1285. Changed the default value of yarn.acl.enable in yarn-default.xml to be consistent with what exists (false) in the code and documentation. Contributed by Kenji Kikushima. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564190 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++++ .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4110dfe7421..cf0a269047e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -118,6 +118,10 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + YARN-1285. Changed the default value of yarn.acl.enable in yarn-default.xml + to be consistent with what exists (false) in the code and documentation. + (Kenji Kikushima via vinodkv) + BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 78ceb64b3d3..a5906986665 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -130,7 +130,7 @@ Are acls enabled. yarn.acl.enable - true + false From 786d71bfb57dd6b0e6ca6583e9fe9a95130b38e3 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Tue, 4 Feb 2014 04:44:00 +0000 Subject: [PATCH 14/42] YARN-1684. Fixed history server heap size in yarn script. Contributed by Billie Rinaldi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564193 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cf0a269047e..79be4506d7f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -172,6 +172,9 @@ Release 2.4.0 - UNRELEASED YARN-1673. Fix option parsing in YARN's application CLI after it is broken by YARN-967. (Mayank Bansal via vinodkv) + YARN-1684. Fixed history server heap size in yarn script. (Billie Rinaldi + via zjshen) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index ac42a9a442a..7b805ffe86f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -204,7 +204,7 @@ elif [ "$COMMAND" = "historyserver" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS" - if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then + if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m" fi elif [ "$COMMAND" = "nodemanager" ] ; then diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 121f864f838..8fba1b5a03f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -207,7 +207,7 @@ goto :eof set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\ahs-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer set YARN_OPTS=%YARN_OPTS% %HADOOP_HISTORYSERVER_OPTS% - if defined YARN_RESOURCEMANAGER_HEAPSIZE ( + if defined YARN_HISTORYSERVER_HEAPSIZE ( set JAVA_HEAP_MAX=-Xmx%YARN_HISTORYSERVER_HEAPSIZE%m ) goto :eof From cd1e8d2e3470a5ce7aced0108f0587a655652008 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 4 Feb 2014 05:11:11 +0000 Subject: [PATCH 15/42] HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding error during testing. Contributed by Haohui Mai git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564198 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f9abed76c5f..bf8633747d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -331,6 +331,9 @@ Release 2.4.0 - UNRELEASED HDFS-5767. NFS implementation assumes userName userId mapping to be unique, which is not true sometimes (Yongjun Zhang via brandonli) + HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding + error during testing (Haohui Mai via brandonli) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java index 883fdeacf79..c4f30b3ebf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java @@ -52,6 +52,7 @@ public class TestHttpsFileSystem { conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); + conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); File base = new File(BASEDIR); FileUtil.fullyDelete(base); From b812af964d100c50d065cdd9007cef31ea2642a8 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Feb 2014 22:46:52 +0000 Subject: [PATCH 16/42] YARN-1669. Modified RM HA handling of protocol level service-ACLS to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564549 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/ipc/Server.java | 8 ++ .../ServiceAuthorizationManager.java | 29 +++-- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../server/resourcemanager/AdminService.java | 52 +++++--- .../ApplicationMasterService.java | 25 +++- .../resourcemanager/ClientRMService.java | 19 ++- .../ResourceTrackerService.java | 20 ++- .../scheduler/capacity/CapacityScheduler.java | 9 +- .../resourcemanager/TestRMAdminService.java | 119 +++++++++++++++++- 10 files changed, 254 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index c7cada825d9..9c67146265b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -450,6 +450,14 @@ public abstract class Server { serviceAuthorizationManager.refresh(conf, provider); } + /** + * Refresh the service authorization ACL for the service handled by this server + * using the specified Configuration. + */ + public void refreshServiceAclWithConfigration(Configuration conf, + PolicyProvider provider) { + serviceAuthorizationManager.refreshWithConfiguration(conf, provider); + } /** * Returns a handle to the serviceAuthorizationManager (required in tests) * @return instance of ServiceAuthorizationManager for this server diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index 8523f38dec7..cf032ba0980 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -33,6 +33,8 @@ import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import com.google.common.annotations.VisibleForTesting; + /** * An authorization manager which handles service-level authorization * for incoming service requests. @@ -120,19 +122,23 @@ public class ServiceAuthorizationManager { // Make a copy of the original config, and load the policy file Configuration policyConf = new Configuration(conf); policyConf.addResource(policyFile); - + refreshWithConfiguration(policyConf, provider); + } + + public synchronized void refreshWithConfiguration(Configuration conf, + PolicyProvider provider) { final Map, AccessControlList> newAcls = - new IdentityHashMap, AccessControlList>(); + new IdentityHashMap, AccessControlList>(); // Parse the config file Service[] services = provider.getServices(); if (services != null) { for (Service service : services) { - AccessControlList acl = - new AccessControlList( - policyConf.get(service.getServiceKey(), - AccessControlList.WILDCARD_ACL_VALUE) - ); + AccessControlList acl = + new AccessControlList( + conf.get(service.getServiceKey(), + AccessControlList.WILDCARD_ACL_VALUE) + ); newAcls.put(service.getProtocol(), acl); } } @@ -141,8 +147,13 @@ public class ServiceAuthorizationManager { protocolToAcl = newAcls; } - // Package-protected for use in tests. - Set> getProtocolsWithAcls() { + @VisibleForTesting + public Set> getProtocolsWithAcls() { return protocolToAcl.keySet(); } + + @VisibleForTesting + public AccessControlList getProtocolsAcls(Class className) { + return protocolToAcl.get(className); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 79be4506d7f..8e8095dfb32 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -122,6 +122,10 @@ Release 2.4.0 - UNRELEASED to be consistent with what exists (false) in the code and documentation. (Kenji Kikushima via vinodkv) + YARN-1669. Modified RM HA handling of protocol level service-ACLS to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 919ed901439..44f6e9b3724 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -40,6 +40,10 @@ public class YarnConfiguration extends Configuration { @Private public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; + @Private + public static final String HADOOP_POLICY_CONFIGURATION_FILE = + "hadoop-policy.xml"; + @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 3bfd47d1373..d9c239e220a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -45,6 +45,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.conf.ConfigurationProvider; @@ -432,9 +433,8 @@ public class AdminService extends CompositeService implements @Override public RefreshServiceAclsResponse refreshServiceAcls( - RefreshServiceAclsRequest request) throws YarnException { - Configuration conf = new Configuration(); - if (!conf.getBoolean( + RefreshServiceAclsRequest request) throws YarnException, IOException { + if (!getConfig().getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { throw RPCUtil.getRemoteException( @@ -442,27 +442,38 @@ public class AdminService extends CompositeService implements CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION + ") not enabled.")); } - - PolicyProvider policyProvider = new RMPolicyProvider(); - - refreshServiceAcls(conf, policyProvider); - if (isRMActive()) { - rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); - rmContext.getApplicationMasterService().refreshServiceAcls( - conf, policyProvider); - rmContext.getResourceTrackerService().refreshServiceAcls( - conf, policyProvider); - } else { - LOG.warn("ResourceManager is not active. Not refreshing ACLs for " + - "Clients, ApplicationMasters and NodeManagers"); + + String argName = "refreshServiceAcls"; + if (!isRMActive()) { + RMAuditLogger.logFailure(UserGroupInformation.getCurrentUser() + .getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not refresh Service ACLs."); + throwStandbyException(); } + + PolicyProvider policyProvider = new RMPolicyProvider(); + Configuration conf = + getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + + refreshServiceAcls(conf, policyProvider); + rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); + rmContext.getApplicationMasterService().refreshServiceAcls( + conf, policyProvider); + rmContext.getResourceTrackerService().refreshServiceAcls( + conf, policyProvider); return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); } - void refreshServiceAcls(Configuration configuration, + synchronized void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.configurationProvider instanceof LocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } @Override @@ -519,4 +530,9 @@ public class AdminService extends CompositeService implements public AccessControlList getAccessControlList() { return this.adminAcl; } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 57605c09362..2c4be13ee92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -39,6 +39,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -86,6 +87,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import com.google.common.annotations.VisibleForTesting; + @SuppressWarnings("unchecked") @Private public class ApplicationMasterService extends AbstractService implements @@ -102,6 +105,7 @@ public class ApplicationMasterService extends AbstractService implements private final AllocateResponse resync = recordFactory.newRecordInstance(AllocateResponse.class); private final RMContext rmContext; + private boolean useLocalConfigurationProvider; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { super(ApplicationMasterService.class.getName()); @@ -111,6 +115,15 @@ public class ApplicationMasterService extends AbstractService implements this.rmContext = rmContext; } + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); + super.serviceInit(conf); + } + @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); @@ -578,7 +591,12 @@ public class ApplicationMasterService extends AbstractService implements public void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } @Override @@ -604,4 +622,9 @@ public class ApplicationMasterService extends AbstractService implements this.response = response; } } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 8800f290cdc..086a6d823e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -43,6 +43,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -106,6 +107,7 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; @@ -133,6 +135,7 @@ public class ClientRMService extends AbstractService implements private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; + private boolean useLocalConfigurationProvider; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -150,6 +153,10 @@ public class ClientRMService extends AbstractService implements @Override protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -773,7 +780,12 @@ public class ClientRMService extends AbstractService implements void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } } private boolean isAllowedDelegationTokenOp() throws IOException { @@ -787,4 +799,9 @@ public class ClientRMService extends AbstractService implements return true; } } + + @VisibleForTesting + public Server getServer() { + return this.server; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index f80ce85d9d1..4f74179717f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -29,6 +29,7 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.VersionUtil; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -66,6 +67,8 @@ import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.hadoop.yarn.util.YarnVersionInfo; +import com.google.common.annotations.VisibleForTesting; + public class ResourceTrackerService extends AbstractService implements ResourceTracker { @@ -92,6 +95,7 @@ public class ResourceTrackerService extends AbstractService implements private int minAllocMb; private int minAllocVcores; + private boolean useLocalConfigurationProvider; static { resync.setNodeAction(NodeAction.RESYNC); @@ -141,6 +145,10 @@ public class ResourceTrackerService extends AbstractService implements YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); super.serviceInit(conf); } @@ -415,6 +423,16 @@ public class ResourceTrackerService extends AbstractService implements void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { - this.server.refreshServiceAcl(configuration, policyProvider); + if (this.useLocalConfigurationProvider) { + this.server.refreshServiceAcl(configuration, policyProvider); + } else { + this.server.refreshServiceAclWithConfigration(configuration, + policyProvider); + } + } + + @VisibleForTesting + public Server getServer() { + return this.server; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 1b47d69fa2c..b019a762515 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -262,10 +263,10 @@ public class CapacityScheduler extends AbstractYarnScheduler public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { if (!initialized) { - this.useLocalConfigurationProvider = conf.get( - YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals( - "org.apache.hadoop.yarn.LocalConfigurationProvider"); + this.useLocalConfigurationProvider = + (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass( + YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + LocalConfigurationProvider.class))); this.conf = new CapacitySchedulerConfiguration(conf, this.useLocalConfigurationProvider); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 797b4226842..5372c18832d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -24,16 +24,19 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -190,6 +193,120 @@ public class TestRMAdminService { Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); } + @Test + public void testServiceAclsRefreshWithLocalConfigurationProvider() { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + ResourceManager resourceManager = null; + + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + @SuppressWarnings("resource") + @Test + public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + ResourceManager resourceManager = null; + try { + resourceManager = new ResourceManager(); + resourceManager.init(configuration); + resourceManager.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + resourceManager.adminService + .refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: hadoop-policy.xml")); + } + + String aclsString = "alice,bob users,wheel"; + Configuration conf = new Configuration(); + conf.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + conf.set("security.applicationclient.protocol.acl", aclsString); + String hadoopConfFile = writeConfigurationXML(conf, "hadoop-policy.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(hadoopConfFile)); + + resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest + .newInstance()); + + // verify service Acls refresh for AdminService + ServiceAuthorizationManager adminServiceServiceManager = + resourceManager.adminService.getServer() + .getServiceAuthorizationManager(); + verifyServiceACLsRefresh(adminServiceServiceManager, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ClientRMService + ServiceAuthorizationManager clientRMServiceServiceManager = + resourceManager.getRMContext().getClientRMService().getServer() + .getServiceAuthorizationManager(); + verifyServiceACLsRefresh(clientRMServiceServiceManager, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ApplicationMasterService + ServiceAuthorizationManager appMasterService = + resourceManager.getRMContext().getApplicationMasterService() + .getServer().getServiceAuthorizationManager(); + verifyServiceACLsRefresh(appMasterService, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + + // verify service ACLs refresh for ResourceTrackerService + ServiceAuthorizationManager RTService = + resourceManager.getRMContext().getResourceTrackerService() + .getServer().getServiceAuthorizationManager(); + verifyServiceACLsRefresh(RTService, + org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, + aclsString); + } finally { + if (resourceManager != null) { + resourceManager.stop(); + } + } + } + + private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, + Class protocol, String aclString) { + for (Class protocolClass : manager.getProtocolsWithAcls()) { + AccessControlList accessList = + manager.getProtocolsAcls(protocolClass); + if (protocolClass == protocol) { + Assert.assertEquals(accessList.getAclString(), + aclString); + } else { + Assert.assertEquals(accessList.getAclString(), "*"); + } + } + } + @Test public void testRefreshSuperUserGroupsWithLocalConfigurationProvider() { From 96578f0e01ba751175d4bcbad48d6f679e662382 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 5 Feb 2014 00:32:02 +0000 Subject: [PATCH 17/42] YARN-1634. Added a testable in-memory implementation of ApplicationTimelineStore. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564583 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 5 + .../api/records/apptimeline/ATSEntity.java | 90 ++- .../apptimeline/EntityId.java | 100 ++++ .../MemoryApplicationTimelineStore.java | 288 ++++++++++ .../ApplicationTimelineStoreTestUtils.java | 532 ++++++++++++++++++ .../TestMemoryApplicationTimelineStore.java | 73 +++ 6 files changed, 1087 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8e8095dfb32..a9af3e0121a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -90,6 +90,9 @@ Release 2.4.0 - UNRELEASED implementing different storage impls for storing timeline information. (Billie Rinaldi via vinodkv) + YARN-1634. Added a testable in-memory implementation of + ApplicationTimelineStore. (Zhijie Shen via vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via @@ -126,6 +129,8 @@ Release 2.4.0 - UNRELEASED be available across RM failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + OPTIMIZATIONS + BUG FIXES YARN-935. Correcting pom.xml to build applicationhistoryserver module diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java index 6b3ea1013e8..709c79568e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java @@ -50,7 +50,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; @XmlAccessorType(XmlAccessType.NONE) @Public @Unstable -public class ATSEntity { +public class ATSEntity implements Comparable { private String entityType; private String entityId; @@ -310,4 +310,92 @@ public class ATSEntity { this.otherInfo = otherInfo; } + @Override + public int hashCode() { + // generated by eclipse + final int prime = 31; + int result = 1; + result = prime * result + ((entityId == null) ? 0 : entityId.hashCode()); + result = + prime * result + ((entityType == null) ? 0 : entityType.hashCode()); + result = prime * result + ((events == null) ? 0 : events.hashCode()); + result = prime * result + ((otherInfo == null) ? 0 : otherInfo.hashCode()); + result = + prime * result + + ((primaryFilters == null) ? 0 : primaryFilters.hashCode()); + result = + prime * result + + ((relatedEntities == null) ? 0 : relatedEntities.hashCode()); + result = prime * result + ((startTime == null) ? 0 : startTime.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + // generated by eclipse + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ATSEntity other = (ATSEntity) obj; + if (entityId == null) { + if (other.entityId != null) + return false; + } else if (!entityId.equals(other.entityId)) + return false; + if (entityType == null) { + if (other.entityType != null) + return false; + } else if (!entityType.equals(other.entityType)) + return false; + if (events == null) { + if (other.events != null) + return false; + } else if (!events.equals(other.events)) + return false; + if (otherInfo == null) { + if (other.otherInfo != null) + return false; + } else if (!otherInfo.equals(other.otherInfo)) + return false; + if (primaryFilters == null) { + if (other.primaryFilters != null) + return false; + } else if (!primaryFilters.equals(other.primaryFilters)) + return false; + if (relatedEntities == null) { + if (other.relatedEntities != null) + return false; + } else if (!relatedEntities.equals(other.relatedEntities)) + return false; + if (startTime == null) { + if (other.startTime != null) + return false; + } else if (!startTime.equals(other.startTime)) + return false; + return true; + } + + @Override + public int compareTo(ATSEntity other) { + int comparison = entityType.compareTo(other.entityType); + if (comparison == 0) { + long thisStartTime = + startTime == null ? Long.MIN_VALUE : startTime; + long otherStartTime = + other.startTime == null ? Long.MIN_VALUE : other.startTime; + if (thisStartTime > otherStartTime) { + return -1; + } else if (thisStartTime < otherStartTime) { + return 1; + } else { + return entityId.compareTo(other.entityId); + } + } else { + return comparison; + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java new file mode 100644 index 00000000000..26431f87569 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/EntityId.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The unique identifier for an entity + */ +@Private +@Unstable +public class EntityId implements Comparable { + + private String id; + private String type; + + public EntityId(String id, String type) { + this.id = id; + this.type = type; + } + + /** + * Get the entity Id. + * @return The entity Id. + */ + public String getId() { + return id; + } + + /** + * Get the entity type. + * @return The entity type. + */ + public String getType() { + return type; + } + + @Override + public int compareTo(EntityId other) { + int c = type.compareTo(other.type); + if (c != 0) return c; + return id.compareTo(other.id); + } + + @Override + public int hashCode() { + // generated by eclipse + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + // generated by eclipse + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + EntityId other = (EntityId) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + return true; + } + + @Override + public String toString() { + return "{ id: " + id + ", type: "+ type + " }"; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java new file mode 100644 index 00000000000..45f0a11d764 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/MemoryApplicationTimelineStore.java @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.SortedSet; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents.ATSEventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; + +/** + * In-memory implementation of {@link ApplicationTimelineStore}. This + * implementation is for test purpose only. If users improperly instantiate it, + * they may encounter reading and writing history data in different memory + * store. + * + */ +@Private +@Unstable +public class MemoryApplicationTimelineStore + extends AbstractService implements ApplicationTimelineStore { + + private Map entities = + new HashMap(); + + public MemoryApplicationTimelineStore() { + super(MemoryApplicationTimelineStore.class.getName()); + } + + @Override + public ATSEntities getEntities(String entityType, Long limit, + Long windowStart, Long windowEnd, NameValuePair primaryFilter, + Collection secondaryFilters, EnumSet fields) { + if (limit == null) { + limit = DEFAULT_LIMIT; + } + if (windowStart == null) { + windowStart = Long.MIN_VALUE; + } + if (windowEnd == null) { + windowEnd = Long.MAX_VALUE; + } + if (fields == null) { + fields = EnumSet.allOf(Field.class); + } + List entitiesSelected = new ArrayList(); + for (ATSEntity entity : new PriorityQueue(entities.values())) { + if (entitiesSelected.size() >= limit) { + break; + } + if (!entity.getEntityType().equals(entityType)) { + continue; + } + if (entity.getStartTime() <= windowStart) { + continue; + } + if (entity.getStartTime() > windowEnd) { + continue; + } + if (primaryFilter != null && + !matchFilter(entity.getPrimaryFilters(), primaryFilter)) { + continue; + } + if (secondaryFilters != null) { // OR logic + boolean flag = false; + for (NameValuePair secondaryFilter : secondaryFilters) { + if (secondaryFilter != null && + matchFilter(entity.getOtherInfo(), secondaryFilter)) { + flag = true; + break; + } + } + if (!flag) { + continue; + } + } + entitiesSelected.add(entity); + } + List entitiesToReturn = new ArrayList(); + for (ATSEntity entitySelected : entitiesSelected) { + entitiesToReturn.add(maskFields(entitySelected, fields)); + } + Collections.sort(entitiesToReturn); + ATSEntities entitiesWrapper = new ATSEntities(); + entitiesWrapper.setEntities(entitiesToReturn); + return entitiesWrapper; + } + + @Override + public ATSEntity getEntity(String entityId, String entityType, + EnumSet fieldsToRetrieve) { + if (fieldsToRetrieve == null) { + fieldsToRetrieve = EnumSet.allOf(Field.class); + } + ATSEntity entity = entities.get(new EntityId(entityId, entityType)); + if (entity == null) { + return null; + } else { + return maskFields(entity, fieldsToRetrieve); + } + } + + @Override + public ATSEvents getEntityTimelines(String entityType, + SortedSet entityIds, Long limit, Long windowStart, + Long windowEnd, + Set eventTypes) { + ATSEvents allEvents = new ATSEvents(); + if (entityIds == null) { + return allEvents; + } + if (limit == null) { + limit = DEFAULT_LIMIT; + } + if (windowStart == null) { + windowStart = Long.MIN_VALUE; + } + if (windowEnd == null) { + windowEnd = Long.MAX_VALUE; + } + for (String entityId : entityIds) { + EntityId entityID = new EntityId(entityId, entityType); + ATSEntity entity = entities.get(entityID); + if (entity == null) { + continue; + } + ATSEventsOfOneEntity events = new ATSEventsOfOneEntity(); + events.setEntityId(entityId); + events.setEntityType(entityType); + for (ATSEvent event : entity.getEvents()) { + if (events.getEvents().size() >= limit) { + break; + } + if (event.getTimestamp() <= windowStart) { + continue; + } + if (event.getTimestamp() > windowEnd) { + continue; + } + if (eventTypes != null && !eventTypes.contains(event.getEventType())) { + continue; + } + events.addEvent(event); + } + allEvents.addEvent(events); + } + return allEvents; + } + + @Override + public ATSPutErrors put(ATSEntities data) { + ATSPutErrors errors = new ATSPutErrors(); + for (ATSEntity entity : data.getEntities()) { + EntityId entityId = + new EntityId(entity.getEntityId(), entity.getEntityType()); + // store entity info in memory + ATSEntity existingEntity = entities.get(entityId); + if (existingEntity == null) { + existingEntity = new ATSEntity(); + existingEntity.setEntityId(entity.getEntityId()); + existingEntity.setEntityType(entity.getEntityType()); + existingEntity.setStartTime(entity.getStartTime()); + entities.put(entityId, existingEntity); + } + if (entity.getEvents() != null) { + if (existingEntity.getEvents() == null) { + existingEntity.setEvents(entity.getEvents()); + } else { + existingEntity.addEvents(entity.getEvents()); + } + Collections.sort(existingEntity.getEvents()); + } + // check startTime + if (existingEntity.getStartTime() == null) { + if (existingEntity.getEvents() == null + || existingEntity.getEvents().isEmpty()) { + ATSPutError error = new ATSPutError(); + error.setEntityId(entityId.getId()); + error.setEntityType(entityId.getType()); + error.setErrorCode(1); + errors.addError(error); + entities.remove(entityId); + continue; + } else { + existingEntity.setStartTime(entity.getEvents().get(0).getTimestamp()); + } + } + if (entity.getPrimaryFilters() != null) { + if (existingEntity.getPrimaryFilters() == null) { + existingEntity.setPrimaryFilters(entity.getPrimaryFilters()); + } else { + existingEntity.addPrimaryFilters(entity.getPrimaryFilters()); + } + } + if (entity.getOtherInfo() != null) { + if (existingEntity.getOtherInfo() == null) { + existingEntity.setOtherInfo(entity.getOtherInfo()); + } else { + existingEntity.addOtherInfo(entity.getOtherInfo()); + } + } + // relate it to other entities + if (entity.getRelatedEntities() == null) { + continue; + } + for (Map.Entry> partRelatedEntities : entity + .getRelatedEntities().entrySet()) { + if (partRelatedEntities == null) { + continue; + } + for (String idStr : partRelatedEntities.getValue()) { + EntityId relatedEntityId = + new EntityId(idStr, partRelatedEntities.getKey()); + ATSEntity relatedEntity = entities.get(relatedEntityId); + if (relatedEntity != null) { + relatedEntity.addRelatedEntity( + existingEntity.getEntityType(), existingEntity.getEntityId()); + } + } + } + } + return errors; + } + + private static ATSEntity maskFields( + ATSEntity entity, EnumSet fields) { + // Conceal the fields that are not going to be exposed + ATSEntity entityToReturn = new ATSEntity(); + entityToReturn.setEntityId(entity.getEntityId()); + entityToReturn.setEntityType(entity.getEntityType()); + entityToReturn.setStartTime(entity.getStartTime()); + entityToReturn.setEvents(fields.contains(Field.EVENTS) ? + entity.getEvents() : fields.contains(Field.LAST_EVENT_ONLY) ? + Arrays.asList(entity.getEvents().get(0)) : null); + entityToReturn.setRelatedEntities(fields.contains(Field.RELATED_ENTITIES) ? + entity.getRelatedEntities() : null); + entityToReturn.setPrimaryFilters(fields.contains(Field.PRIMARY_FILTERS) ? + entity.getPrimaryFilters() : null); + entityToReturn.setOtherInfo(fields.contains(Field.OTHER_INFO) ? + entity.getOtherInfo() : null); + return entityToReturn; + } + + private static boolean matchFilter(Map tags, + NameValuePair filter) { + Object value = tags.get(filter.getName()); + if (value == null) { // doesn't have the filter + return false; + } else if (!value.equals(filter.getValue())) { // doesn't match the filter + return false; + } + return true; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java new file mode 100644 index 00000000000..5825af192b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/ApplicationTimelineStoreTestUtils.java @@ -0,0 +1,532 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents.ATSEventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineReader.Field; + +public class ApplicationTimelineStoreTestUtils { + + private static final Map EMPTY_MAP = Collections.emptyMap(); + private static final Map> EMPTY_REL_ENTITIES = + new HashMap>(); + + protected ApplicationTimelineStore store; + private String entity1; + private String entityType1; + private String entity1b; + private String entity2; + private String entityType2; + private Map primaryFilters; + private Map secondaryFilters; + private Map allFilters; + private Map otherInfo; + private Map> relEntityMap; + private NameValuePair userFilter; + private Collection goodTestingFilters; + private Collection badTestingFilters; + private ATSEvent ev1; + private ATSEvent ev2; + private ATSEvent ev3; + private ATSEvent ev4; + private Map eventInfo; + private List events1; + private List events2; + + /** + * Load test data into the given store + */ + protected void loadTestData() { + ATSEntities atsEntities = new ATSEntities(); + Map primaryFilters = new HashMap(); + primaryFilters.put("user", "username"); + primaryFilters.put("appname", 12345l); + Map secondaryFilters = new HashMap(); + secondaryFilters.put("startTime", 123456l); + secondaryFilters.put("status", "RUNNING"); + Map otherInfo1 = new HashMap(); + otherInfo1.put("info1", "val1"); + otherInfo1.putAll(secondaryFilters); + + String entity1 = "id_1"; + String entityType1 = "type_1"; + String entity1b = "id_2"; + String entity2 = "id_2"; + String entityType2 = "type_2"; + + Map> relatedEntities = + new HashMap>(); + relatedEntities.put(entityType2, Collections.singletonList(entity2)); + + ATSEvent ev3 = createEvent(789l, "launch_event", null); + ATSEvent ev4 = createEvent(-123l, "init_event", null); + List events = new ArrayList(); + events.add(ev3); + events.add(ev4); + atsEntities.setEntities(Collections.singletonList(createEntity(entity2, + entityType2, null, events, null, null, null))); + ATSPutErrors response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + + ATSEvent ev1 = createEvent(123l, "start_event", null); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1, + entityType1, 123l, Collections.singletonList(ev1), + relatedEntities, primaryFilters, otherInfo1))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1b, + entityType1, null, Collections.singletonList(ev1), relatedEntities, + primaryFilters, otherInfo1))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + + Map eventInfo = new HashMap(); + eventInfo.put("event info 1", "val1"); + ATSEvent ev2 = createEvent(456l, "end_event", eventInfo); + Map otherInfo2 = new HashMap(); + otherInfo2.put("info2", "val2"); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1, + entityType1, null, Collections.singletonList(ev2), null, + primaryFilters, otherInfo2))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + atsEntities.setEntities(Collections.singletonList(createEntity(entity1b, + entityType1, 123l, Collections.singletonList(ev2), null, + primaryFilters, otherInfo2))); + response = store.put(atsEntities); + assertEquals(0, response.getErrors().size()); + + atsEntities.setEntities(Collections.singletonList(createEntity( + "badentityid", "badentity", null, null, null, null, otherInfo1))); + response = store.put(atsEntities); + assertEquals(1, response.getErrors().size()); + ATSPutError error = response.getErrors().get(0); + assertEquals("badentityid", error.getEntityId()); + assertEquals("badentity", error.getEntityType()); + assertEquals((Integer) 1, error.getErrorCode()); + } + + /** + * Load veification data + */ + protected void loadVerificationData() throws Exception { + userFilter = new NameValuePair("user", + "username"); + goodTestingFilters = new ArrayList(); + goodTestingFilters.add(new NameValuePair("appname", 12345l)); + goodTestingFilters.add(new NameValuePair("status", "RUNNING")); + badTestingFilters = new ArrayList(); + badTestingFilters.add(new NameValuePair("appname", 12345l)); + badTestingFilters.add(new NameValuePair("status", "FINISHED")); + + primaryFilters = new HashMap(); + primaryFilters.put("user", "username"); + primaryFilters.put("appname", 12345l); + secondaryFilters = new HashMap(); + secondaryFilters.put("startTime", 123456l); + secondaryFilters.put("status", "RUNNING"); + allFilters = new HashMap(); + allFilters.putAll(secondaryFilters); + allFilters.putAll(primaryFilters); + otherInfo = new HashMap(); + otherInfo.put("info1", "val1"); + otherInfo.put("info2", "val2"); + otherInfo.putAll(secondaryFilters); + + entity1 = "id_1"; + entityType1 = "type_1"; + entity1b = "id_2"; + entity2 = "id_2"; + entityType2 = "type_2"; + + ev1 = createEvent(123l, "start_event", null); + + eventInfo = new HashMap(); + eventInfo.put("event info 1", "val1"); + ev2 = createEvent(456l, "end_event", eventInfo); + events1 = new ArrayList(); + events1.add(ev2); + events1.add(ev1); + + relEntityMap = + new HashMap>(); + List ids = new ArrayList(); + ids.add(entity1); + ids.add(entity1b); + relEntityMap.put(entityType1, ids); + + ev3 = createEvent(789l, "launch_event", null); + ev4 = createEvent(-123l, "init_event", null); + events2 = new ArrayList(); + events2.add(ev3); + events2.add(ev4); + } + + public void testGetSingleEntity() { + // test getting entity info + verifyEntityInfo(null, null, null, null, null, null, + store.getEntity("id_1", "type_2", EnumSet.allOf(Field.class))); + + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, store.getEntity(entity1, entityType1, + EnumSet.allOf(Field.class))); + + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, store.getEntity(entity1b, entityType1, + EnumSet.allOf(Field.class))); + + verifyEntityInfo(entity2, entityType2, events2, relEntityMap, EMPTY_MAP, + EMPTY_MAP, store.getEntity(entity2, entityType2, + EnumSet.allOf(Field.class))); + + // test getting single fields + verifyEntityInfo(entity1, entityType1, events1, null, null, null, + store.getEntity(entity1, entityType1, EnumSet.of(Field.EVENTS))); + + verifyEntityInfo(entity1, entityType1, Collections.singletonList(ev2), + null, null, null, store.getEntity(entity1, entityType1, + EnumSet.of(Field.LAST_EVENT_ONLY))); + + verifyEntityInfo(entity1, entityType1, null, null, primaryFilters, null, + store.getEntity(entity1, entityType1, + EnumSet.of(Field.PRIMARY_FILTERS))); + + verifyEntityInfo(entity1, entityType1, null, null, null, otherInfo, + store.getEntity(entity1, entityType1, EnumSet.of(Field.OTHER_INFO))); + + verifyEntityInfo(entity2, entityType2, null, relEntityMap, null, null, + store.getEntity(entity2, entityType2, + EnumSet.of(Field.RELATED_ENTITIES))); + } + + public void testGetEntities() { + // test getting entities + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_0", null, null, null, null, null, + null).getEntities().size()); + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_3", null, null, null, null, null, + null).getEntities().size()); + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_0", null, null, null, userFilter, + null, null).getEntities().size()); + assertEquals("nonzero entities size for nonexistent type", 0, + store.getEntities("type_3", null, null, null, userFilter, + null, null).getEntities().size()); + + List entities = + store.getEntities("type_1", null, null, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_2", null, null, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity2, entityType2, events2, relEntityMap, EMPTY_MAP, + EMPTY_MAP, entities.get(0)); + + entities = store.getEntities("type_1", 1l, null, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", 1l, 0l, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", null, 234l, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, 123l, null, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, 234l, 345l, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, null, 345l, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_1", null, null, 123l, null, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + } + + public void testGetEntitiesWithPrimaryFilters() { + // test using primary filter + assertEquals("nonzero entities size for primary filter", 0, + store.getEntities("type_1", null, null, null, + new NameValuePair("none", "none"), null, + EnumSet.allOf(Field.class)).getEntities().size()); + assertEquals("nonzero entities size for primary filter", 0, + store.getEntities("type_2", null, null, null, + new NameValuePair("none", "none"), null, + EnumSet.allOf(Field.class)).getEntities().size()); + assertEquals("nonzero entities size for primary filter", 0, + store.getEntities("type_3", null, null, null, + new NameValuePair("none", "none"), null, + EnumSet.allOf(Field.class)).getEntities().size()); + + List entities = store.getEntities("type_1", null, null, null, + userFilter, null, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_2", null, null, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", 1l, null, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", 1l, 0l, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(1, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + + entities = store.getEntities("type_1", null, 234l, null, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, 234l, 345l, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, null, 345l, userFilter, null, + EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + } + + public void testGetEntitiesWithSecondaryFilters() { + // test using secondary filter + List entities = store.getEntities("type_1", null, null, null, + null, goodTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_1", null, null, null, userFilter, + goodTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(2, entities.size()); + verifyEntityInfo(entity1, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(0)); + verifyEntityInfo(entity1b, entityType1, events1, EMPTY_REL_ENTITIES, + primaryFilters, otherInfo, entities.get(1)); + + entities = store.getEntities("type_1", null, null, null, null, + badTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + + entities = store.getEntities("type_1", null, null, null, userFilter, + badTestingFilters, EnumSet.allOf(Field.class)).getEntities(); + assertEquals(0, entities.size()); + } + + public void testGetEvents() { + // test getting entity timelines + SortedSet sortedSet = new TreeSet(); + sortedSet.add(entity1); + List timelines = + store.getEntityTimelines(entityType1, sortedSet, null, null, + null, null).getAllEvents(); + assertEquals(1, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2, ev1); + + sortedSet.add(entity1b); + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2, ev1); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2, ev1); + + timelines = store.getEntityTimelines(entityType1, sortedSet, 1l, + null, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + 345l, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + 123l, null, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, 345l, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev1); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev1); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, 123l, null).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev1); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev1); + + timelines = store.getEntityTimelines(entityType1, sortedSet, null, + null, null, Collections.singleton("end_event")).getAllEvents(); + assertEquals(2, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity1, entityType1, ev2); + verifyEntityTimeline(timelines.get(1), entity1b, entityType1, ev2); + + sortedSet.add(entity2); + timelines = store.getEntityTimelines(entityType2, sortedSet, null, + null, null, null).getAllEvents(); + assertEquals(1, timelines.size()); + verifyEntityTimeline(timelines.get(0), entity2, entityType2, ev3, ev4); + } + + /** + * Verify a single entity + */ + private static void verifyEntityInfo(String entity, String entityType, + List events, Map> relatedEntities, + Map primaryFilters, Map otherInfo, + ATSEntity retrievedEntityInfo) { + if (entity == null) { + assertNull(retrievedEntityInfo); + return; + } + assertEquals(entity, retrievedEntityInfo.getEntityId()); + assertEquals(entityType, retrievedEntityInfo.getEntityType()); + if (events == null) + assertNull(retrievedEntityInfo.getEvents()); + else + assertEquals(events, retrievedEntityInfo.getEvents()); + if (relatedEntities == null) + assertNull(retrievedEntityInfo.getRelatedEntities()); + else + assertEquals(relatedEntities, retrievedEntityInfo.getRelatedEntities()); + if (primaryFilters == null) + assertNull(retrievedEntityInfo.getPrimaryFilters()); + else + assertTrue(primaryFilters.equals( + retrievedEntityInfo.getPrimaryFilters())); + if (otherInfo == null) + assertNull(retrievedEntityInfo.getOtherInfo()); + else + assertTrue(otherInfo.equals(retrievedEntityInfo.getOtherInfo())); + } + + /** + * Verify timeline events + */ + private static void verifyEntityTimeline( + ATSEventsOfOneEntity retrievedEvents, String entity, String entityType, + ATSEvent... actualEvents) { + assertEquals(entity, retrievedEvents.getEntityId()); + assertEquals(entityType, retrievedEvents.getEntityType()); + assertEquals(actualEvents.length, retrievedEvents.getEvents().size()); + for (int i = 0; i < actualEvents.length; i++) { + assertEquals(actualEvents[i], retrievedEvents.getEvents().get(i)); + } + } + + /** + * Create a test entity + */ + private static ATSEntity createEntity(String entity, String entityType, + Long startTime, List events, + Map> relatedEntities, + Map primaryFilters, Map otherInfo) { + ATSEntity atsEntity = new ATSEntity(); + atsEntity.setEntityId(entity); + atsEntity.setEntityType(entityType); + atsEntity.setStartTime(startTime); + atsEntity.setEvents(events); + if (relatedEntities != null) + for (Entry> e : relatedEntities.entrySet()) + for (String v : e.getValue()) + atsEntity.addRelatedEntity(e.getKey(), v); + else + atsEntity.setRelatedEntities(null); + atsEntity.setPrimaryFilters(primaryFilters); + atsEntity.setOtherInfo(otherInfo); + return atsEntity; + } + + /** + * Create a test event + */ + private static ATSEvent createEvent(long timestamp, String type, Map info) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(timestamp); + event.setEventType(type); + event.setEventInfo(info); + return event; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java new file mode 100644 index 00000000000..aa88b74a901 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/TestMemoryApplicationTimelineStore.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + + +public class TestMemoryApplicationTimelineStore + extends ApplicationTimelineStoreTestUtils { + + @Before + public void setup() throws Exception { + store = new MemoryApplicationTimelineStore(); + store.init(new YarnConfiguration()); + store.start(); + loadTestData(); + loadVerificationData(); + } + + @After + public void tearDown() throws Exception { + store.stop(); + } + + public ApplicationTimelineStore getApplicationTimelineStore() { + return store; + } + + @Test + public void testGetSingleEntity() { + super.testGetSingleEntity(); + } + + @Test + public void testGetEntities() { + super.testGetEntities(); + } + + @Test + public void testGetEntitiesWithPrimaryFilters() { + super.testGetEntitiesWithPrimaryFilters(); + } + + @Test + public void testGetEntitiesWithSecondaryFilters() { + super.testGetEntitiesWithSecondaryFilters(); + } + + @Test + public void testGetEvents() { + super.testGetEvents(); + } + +} From 0aa09f6d5a97f523e9ee6f30bb44f206433ead0a Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 5 Feb 2014 04:18:31 +0000 Subject: [PATCH 18/42] HDFS-5399. Revisit SafeModeException and corresponding retry policies. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564629 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/io/retry/RetryPolicies.java | 20 ++++++++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSClient.java | 6 ++++++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../apache/hadoop/hdfs/NameNodeProxies.java | 16 ++++++++++----- .../hdfs/server/namenode/FSNamesystem.java | 15 +++++++++++++- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 5 ++++- .../server/namenode/ha/TestHASafeMode.java | 5 +++++ 8 files changed, 66 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index 3d5992716f4..14ded8ea24d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -151,6 +151,13 @@ public class RetryPolicies { delayMillis, maxDelayBase); } + public static final RetryPolicy failoverOnNetworkException( + RetryPolicy fallbackPolicy, int maxFailovers, int maxRetries, + long delayMillis, long maxDelayBase) { + return new FailoverOnNetworkExceptionRetry(fallbackPolicy, maxFailovers, + maxRetries, delayMillis, maxDelayBase); + } + static class TryOnceThenFail implements RetryPolicy { @Override public RetryAction shouldRetry(Exception e, int retries, int failovers, @@ -516,18 +523,25 @@ public class RetryPolicies { private RetryPolicy fallbackPolicy; private int maxFailovers; + private int maxRetries; private long delayMillis; private long maxDelayBase; public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers) { - this(fallbackPolicy, maxFailovers, 0, 0); + this(fallbackPolicy, maxFailovers, 0, 0, 0); } public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, int maxFailovers, long delayMillis, long maxDelayBase) { + this(fallbackPolicy, maxFailovers, 0, delayMillis, maxDelayBase); + } + + public FailoverOnNetworkExceptionRetry(RetryPolicy fallbackPolicy, + int maxFailovers, int maxRetries, long delayMillis, long maxDelayBase) { this.fallbackPolicy = fallbackPolicy; this.maxFailovers = maxFailovers; + this.maxRetries = maxRetries; this.delayMillis = delayMillis; this.maxDelayBase = maxDelayBase; } @@ -549,6 +563,10 @@ public class RetryPolicies { "failovers (" + failovers + ") exceeded maximum allowed (" + maxFailovers + ")"); } + if (retries - failovers > maxRetries) { + return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "retries (" + + retries + ") exceeded maximum allowed (" + maxRetries + ")"); + } if (e instanceof ConnectException || e instanceof NoRouteToHostException || diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bf8633747d1..c84166b863c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -863,6 +863,9 @@ Release 2.3.0 - UNRELEASED HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs on a secure cluster. (jing9) + HDFS-5399. Revisit SafeModeException and corresponding retry policies. + (Haohui Mai via todd) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index b522acb09a5..c1a1a73bf77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -36,6 +36,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY; @@ -258,6 +260,7 @@ public class DFSClient implements java.io.Closeable { public static class Conf { final int hdfsTimeout; // timeout value for a DFS operation. final int maxFailoverAttempts; + final int maxRetryAttempts; final int failoverSleepBaseMillis; final int failoverSleepMaxMillis; final int maxBlockAcquireFailures; @@ -303,6 +306,9 @@ public class DFSClient implements java.io.Closeable { maxFailoverAttempts = conf.getInt( DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + maxRetryAttempts = conf.getInt( + DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); failoverSleepBaseMillis = conf.getInt( DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 38c40c21f38..fe1d3d1570e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -82,6 +82,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_DEFAULT = 0; public static final String DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY = "dfs.client.failover.connection.retries.on.timeouts"; public static final int DFS_CLIENT_FAILOVER_CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0; + public static final String DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY = "dfs.client.retry.max.attempts"; + public static final int DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT = 10; public static final String DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY = "dfs.client.socketcache.expiryMsec"; public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 2 * 60 * 1000; @@ -574,6 +576,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT = "10000,6,60000,10"; //t1,n1,t2,n2,... public static final String DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY = "dfs.http.client.failover.max.attempts"; public static final int DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT = 15; + public static final String DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY = "dfs.http.client.retry.max.attempts"; + public static final int DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT = 10; public static final String DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY = "dfs.http.client.failover.sleep.base.millis"; public static final int DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT = 500; public static final String DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY = "dfs.http.client.failover.sleep.max.millis"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index bff0284c74f..1637bd03cd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -24,6 +24,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT; import java.io.IOException; import java.lang.reflect.Constructor; @@ -144,9 +146,10 @@ public class NameNodeProxies { .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface, nameNodeUri); Conf config = new Conf(conf); - T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies - .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, - config.maxFailoverAttempts, config.failoverSleepBaseMillis, + T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, + RetryPolicies.failoverOnNetworkException( + RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts, + config.maxRetryAttempts, config.failoverSleepBaseMillis, config.failoverSleepMaxMillis)); Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri); @@ -192,11 +195,14 @@ public class NameNodeProxies { int maxFailoverAttempts = config.getInt( DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + int maxRetryAttempts = config.getInt( + DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); InvocationHandler dummyHandler = new LossyRetryInvocationHandler( numResponseToDrop, failoverProxyProvider, RetryPolicies.failoverOnNetworkException( - RetryPolicies.TRY_ONCE_THEN_FAIL, - Math.max(numResponseToDrop + 1, maxFailoverAttempts), delay, + RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, + Math.max(numResponseToDrop + 1, maxRetryAttempts), delay, maxCap)); T proxy = (T) Proxy.newProxyInstance( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ef2c28622f9..4e209767dfc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1161,7 +1161,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if (isInSafeMode()) { SafeModeException se = new SafeModeException(errorMsg, safeMode); if (haEnabled && haContext != null - && haContext.getState().getServiceState() == HAServiceState.ACTIVE) { + && haContext.getState().getServiceState() == HAServiceState.ACTIVE + && shouldRetrySafeMode(this.safeMode)) { throw new RetriableException(se); } else { throw se; @@ -1169,6 +1170,18 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } + /** + * We already know that the safemode is on. We will throw a RetriableException + * if the safemode is not manual or caused by low resource. + */ + private boolean shouldRetrySafeMode(SafeModeInfo safeMode) { + if (safeMode == null) { + return false; + } else { + return !safeMode.isManual() && !safeMode.areResourcesLow(); + } + } + public static Collection getNamespaceDirs(Configuration conf) { return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 148dc72d76c..efaf9637d7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -188,6 +188,9 @@ public class WebHdfsFileSystem extends FileSystem int maxFailoverAttempts = conf.getInt( DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); + int maxRetryAttempts = conf.getInt( + DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY, + DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); int failoverSleepBaseMillis = conf.getInt( DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); @@ -197,7 +200,7 @@ public class WebHdfsFileSystem extends FileSystem this.retryPolicy = RetryPolicies .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, - maxFailoverAttempts, failoverSleepBaseMillis, + maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index 0c95764eba6..1c7b7f423cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.SafeModeInfo; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IOUtils; @@ -65,6 +66,7 @@ import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; import com.google.common.base.Supplier; import com.google.common.collect.Lists; @@ -124,6 +126,9 @@ public class TestHASafeMode { final Path test = new Path("/test"); // let nn0 enter safemode NameNodeAdapter.enterSafeMode(nn0, false); + SafeModeInfo safeMode = (SafeModeInfo) Whitebox.getInternalState( + nn0.getNamesystem(), "safeMode"); + Whitebox.setInternalState(safeMode, "extension", Integer.valueOf(30000)); LOG.info("enter safemode"); new Thread() { @Override From 747cce814cfc1fdb89569eeb85b4712f10beea0e Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 5 Feb 2014 04:20:48 +0000 Subject: [PATCH 19/42] Correct CHANGES.txt entry for HDFS-5399 (contributed by Jing, not Haohui) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564632 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c84166b863c..0b68cd35a0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -864,7 +864,7 @@ Release 2.3.0 - UNRELEASED on a secure cluster. (jing9) HDFS-5399. Revisit SafeModeException and corresponding retry policies. - (Haohui Mai via todd) + (Jing Zhao via todd) BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS From ebe0c17a95ae37d4768f2928ea193e89db34ead5 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 5 Feb 2014 04:55:00 +0000 Subject: [PATCH 20/42] YARN-1461. Added tags for YARN applications and changed RM to handle them. Contributed by Karthik Kambatla. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564633 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../ApplicationsRequestScope.java | 38 ++++++ .../GetApplicationsRequest.java | 117 +++++++++++++++++- .../yarn/api/records/ApplicationReport.java | 14 +++ .../records/ApplicationSubmissionContext.java | 26 +++- .../hadoop/yarn/conf/YarnConfiguration.java | 7 ++ .../src/main/proto/yarn_protos.proto | 2 + .../src/main/proto/yarn_service_protos.proto | 8 ++ .../impl/pb/GetApplicationsRequestPBImpl.java | 62 +++++++++- .../impl/pb/ApplicationReportPBImpl.java | 31 +++++ .../ApplicationSubmissionContextPBImpl.java | 60 ++++++++- .../yarn/api/records/impl/pb/ProtoUtils.java | 14 +++ .../yarn/server/utils/BuilderUtils.java | 4 +- .../resourcemanager/ClientRMService.java | 33 ++++- .../server/resourcemanager/RMAppManager.java | 3 +- .../server/resourcemanager/rmapp/RMApp.java | 9 +- .../resourcemanager/rmapp/RMAppImpl.java | 15 ++- .../resourcemanager/webapp/AppBlock.java | 1 + .../resourcemanager/webapp/RMWebServices.java | 13 +- .../resourcemanager/webapp/dao/AppInfo.java | 10 +- .../resourcemanager/TestClientRMService.java | 51 +++++++- .../applicationsmanager/MockAsm.java | 12 +- .../resourcemanager/rmapp/MockRMApp.java | 6 + .../rmapp/TestRMAppTransitions.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 6 +- .../webapp/TestRMWebServicesApps.java | 4 +- .../src/site/apt/ResourceManagerRest.apt.vm | 1 + 27 files changed, 528 insertions(+), 24 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a9af3e0121a..ee4ac79d57b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -93,6 +93,9 @@ Release 2.4.0 - UNRELEASED YARN-1634. Added a testable in-memory implementation of ApplicationTimelineStore. (Zhijie Shen via vinodkv) + YARN-1461. Added tags for YARN applications and changed RM to handle them. + (Karthik Kambatla via zjshen) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java new file mode 100644 index 00000000000..168223340c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ApplicationsRequestScope.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Enumeration that controls the scope of applications fetched + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public enum ApplicationsRequestScope { + /** All jobs */ + ALL, + + /** Jobs viewable by current user */ + VIEWABLE, + + /** Jobs owned by current user */ + OWN +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java index 95254b2cb13..4cc0b70e4af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.api.protocolrecords; import java.util.EnumSet; import java.util.Set; -import org.apache.commons.collections.buffer.UnboundedFifoBuffer; import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -49,6 +48,86 @@ public abstract class GetApplicationsRequest { return request; } + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + * + *

Setting any of the parameters to null, would just disable that + * filter

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @param users list of users to filter by + * @param queues list of scheduler queues to filter by + * @param applicationTypes types of applications + * @param applicationTags application tags to filter by + * @param applicationStates application states to filter by + * @param startRange range of application start times to filter by + * @param finishRange range of application finish times to filter by + * @param limit number of applications to limit to + * @return {@link GetApplicationsRequest} to be used with + * {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)} + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope, + Set users, + Set queues, + Set applicationTypes, + Set applicationTags, + EnumSet applicationStates, + LongRange startRange, + LongRange finishRange, + Long limit) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + if (scope != null) { + request.setScope(scope); + } + request.setUsers(users); + request.setQueues(queues); + request.setApplicationTypes(applicationTypes); + request.setApplicationTags(applicationTags); + request.setApplicationStates(applicationStates); + if (startRange != null) { + request.setStartRange( + startRange.getMinimumLong(), startRange.getMaximumLong()); + } + if (finishRange != null) { + request.setFinishRange( + finishRange.getMinimumLong(), finishRange.getMaximumLong()); + } + if (limit != null) { + request.setLimit(limit); + } + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setScope(scope); + return request; + } + /** *

* The request from clients to get a report of Applications matching the @@ -257,4 +336,40 @@ public abstract class GetApplicationsRequest { @Private @Unstable public abstract void setFinishRange(long begin, long end); + + /** + * Get the tags to filter applications on + * + * @return list of tags to filter on + */ + @Private + @Unstable + public abstract Set getApplicationTags(); + + /** + * Set the list of tags to filter applications on + * + * @param tags list of tags to filter on + */ + @Private + @Unstable + public abstract void setApplicationTags(Set tags); + + /** + * Get the {@link ApplicationsRequestScope} of applications to be filtered. + * + * @return {@link ApplicationsRequestScope} of applications to return. + */ + @Private + @Unstable + public abstract ApplicationsRequestScope getScope(); + + /** + * Set the {@link ApplicationsRequestScope} of applications to filter. + * + * @param scope scope to use for filtering applications + */ + @Private + @Unstable + public abstract void setScope(ApplicationsRequestScope scope); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 0854fdb89c9..412c22b13db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -25,6 +25,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationReport is a report of an application.

* @@ -321,6 +323,18 @@ public abstract class ApplicationReport { @Unstable public abstract void setApplicationType(String applicationType); + /** + * Get all tags corresponding to the application + * @return Application's tags + */ + @Public + @Stable + public abstract Set getApplicationTags(); + + @Private + @Unstable + public abstract void setApplicationTags(Set tags); + @Private @Stable public abstract void setAMRMToken(Token amRmToken); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index a320bb26972..529df113c27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -25,8 +25,11 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationSubmissionContext represents all of the * information needed by the ResourceManager to launch @@ -284,7 +287,6 @@ public abstract class ApplicationSubmissionContext { @Stable public abstract void setApplicationType(String applicationType); - /** * Get the flag which indicates whether to keep containers across application * attempts or not. @@ -314,4 +316,26 @@ public abstract class ApplicationSubmissionContext { @Stable public abstract void setKeepContainersAcrossApplicationAttempts( boolean keepContainers); + + /** + * Get tags for the application + * + * @return the application tags + */ + @Public + @Stable + public abstract Set getApplicationTags(); + + /** + * Set tags for the application. A maximum of + * {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed + * per application. Each tag can be at most + * {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH} + * characters, and can contain only ASCII characters. + * + * @param tags tags to set + */ + @Public + @Stable + public abstract void setApplicationTags(Set tags); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 44f6e9b3724..d0df7d07f34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -50,6 +51,12 @@ public class YarnConfiguration extends Configuration { @Private public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + @Evolving + public static final int APPLICATION_MAX_TAGS = 10; + + @Evolving + public static final int APPLICATION_MAX_TAG_LENGTH = 100; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; static { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c86d97149c7..8f6cf4c783b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -190,6 +190,7 @@ message ApplicationReportProto { optional float progress = 17; optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; + repeated string applicationTags = 20; } message ApplicationAttemptReportProto { @@ -287,6 +288,7 @@ message ApplicationSubmissionContextProto { optional ResourceProto resource = 9; optional string applicationType = 10 [default = "YARN"]; optional bool keep_containers_across_application_attempts = 11 [default = false]; + repeated string applicationTags = 12; } enum ApplicationAccessTypeProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 139b3bb1394..eff5cd7ae82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -136,6 +136,12 @@ message MoveApplicationAcrossQueuesRequestProto { message MoveApplicationAcrossQueuesResponseProto { } +enum ApplicationsRequestScopeProto { + ALL = 0; + VIEWABLE = 1; + OWN = 2; +} + message GetApplicationsRequestProto { repeated string application_types = 1; repeated YarnApplicationStateProto application_states = 2; @@ -146,6 +152,8 @@ message GetApplicationsRequestProto { optional int64 start_end = 7; optional int64 finish_begin = 8; optional int64 finish_end = 9; + repeated string applicationTags = 10; + optional ApplicationsRequestScopeProto scope = 11 [default = ALL]; } message GetApplicationsResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index 10513a3dc18..733f19f715d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; @@ -49,6 +50,8 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { Set queues = null; long limit = Long.MAX_VALUE; LongRange start = null, finish = null; + private Set applicationTags; + private ApplicationsRequestScope scope; public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); @@ -112,6 +115,12 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { }; builder.addAllApplicationStates(iterable); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } + if (this.scope != null) { + builder.setScope(ProtoUtils.convertToProtoFormat(scope)); + } } private void addLocalApplicationTypesToProto() { @@ -187,12 +196,64 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { this.applicationTypes = applicationTypes; } + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + this.applicationTags = null; + return; + } + // Convert applicationTags to lower case and add + this.applicationTags = new HashSet(); + for (String tag : tags) { + this.applicationTags.add(tag.toLowerCase()); + } + } + @Override public EnumSet getApplicationStates() { initApplicationStates(); return this.applicationStates; } + private void initScope() { + if (this.scope != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.scope = ProtoUtils.convertFromProtoFormat(p.getScope()); + } + + @Override + public ApplicationsRequestScope getScope() { + initScope(); + return this.scope; + } + + public void setScope(ApplicationsRequestScope scope) { + maybeInitBuilder(); + if (scope == null) { + builder.clearScope(); + } + this.scope = scope; + } + @Override public void setApplicationStates(EnumSet applicationStates) { maybeInitBuilder(); @@ -223,7 +284,6 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { return this.users; } - @Override public void setUsers(Set users) { maybeInitBuilder(); if (users == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 9716f74a681..7e19d8fa2f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -38,6 +38,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationReportPBImpl extends ApplicationReport { @@ -49,6 +52,7 @@ public class ApplicationReportPBImpl extends ApplicationReport { private ApplicationAttemptId currentApplicationAttemptId; private Token clientToAMToken = null; private Token amRmToken = null; + private Set applicationTags = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -245,6 +249,21 @@ public class ApplicationReportPBImpl extends ApplicationReport { return amRmToken; } + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); @@ -355,6 +374,15 @@ public class ApplicationReportPBImpl extends ApplicationReport { builder.setApplicationType((applicationType)); } + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + } + this.applicationTags = tags; + } + @Override public void setDiagnostics(String diagnostics) { maybeInitBuilder(); @@ -450,6 +478,9 @@ public class ApplicationReportPBImpl extends ApplicationReport { builder.getAmRmToken())) { builder.setAmRmToken(convertToProtoFormat(this.amRmToken)); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } } private void mergeLocalToProto() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index ffaaf3556b2..c4a3a721990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import com.google.common.base.CharMatcher; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -25,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; @@ -34,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationSubmissionContextPBImpl @@ -47,6 +52,7 @@ extends ApplicationSubmissionContext { private Priority priority = null; private ContainerLaunchContext amContainer = null; private Resource resource = null; + private Set applicationTags = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -100,6 +106,9 @@ extends ApplicationSubmissionContext { builder.getResource())) { builder.setResource(convertToProtoFormat(this.resource)); } + if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.addAllApplicationTags(this.applicationTags); + } } private void mergeLocalToProto() { @@ -196,7 +205,22 @@ extends ApplicationSubmissionContext { } return (p.getApplicationType()); } - + + private void initApplicationTags() { + if (this.applicationTags != null) { + return; + } + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + this.applicationTags = new HashSet(); + this.applicationTags.addAll(p.getApplicationTagsList()); + } + + @Override + public Set getApplicationTags() { + initApplicationTags(); + return this.applicationTags; + } + @Override public void setQueue(String queue) { maybeInitBuilder(); @@ -217,6 +241,40 @@ extends ApplicationSubmissionContext { builder.setApplicationType((applicationType)); } + private void checkTags(Set tags) { + if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) { + throw new IllegalArgumentException("Too many applicationTags, a maximum of only " + + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!"); + } + for (String tag : tags) { + if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) { + throw new IllegalArgumentException("Tag " + tag + " is too long, " + + "maximum allowed length of a tag is " + + YarnConfiguration.APPLICATION_MAX_TAG_LENGTH); + } + if (!CharMatcher.ASCII.matchesAllOf(tag)) { + throw new IllegalArgumentException("A tag can only have ASCII " + + "characters! Invalid tag - " + tag); + } + } + } + + @Override + public void setApplicationTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearApplicationTags(); + this.applicationTags = null; + return; + } + checkTags(tags); + // Convert applicationTags to lower case and add + this.applicationTags = new HashSet(); + for (String tag : tags) { + this.applicationTags.add(tag.toLowerCase()); + } + } + @Override public ContainerLaunchContext getAMContainerSpec() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 8d737448567..4a3c13792ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.ByteString; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; @Private @Unstable @@ -113,6 +115,18 @@ public class ProtoUtils { YARN_APPLICATION_ATTEMPT_STATE_PREFIX, "")); } + /* + * ApplicationsRequestScope + */ + public static YarnServiceProtos.ApplicationsRequestScopeProto + convertToProtoFormat(ApplicationsRequestScope e) { + return YarnServiceProtos.ApplicationsRequestScopeProto.valueOf(e.name()); + } + public static ApplicationsRequestScope convertFromProtoFormat + (YarnServiceProtos.ApplicationsRequestScopeProto e) { + return ApplicationsRequestScope.valueOf(e.name()); + } + /* * ApplicationResourceUsageReport */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 1b2a03e5511..ac25c001248 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; @@ -312,7 +313,7 @@ public class BuilderUtils { String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String appType, Token amRmToken) { + float progress, String appType, Token amRmToken, Set tags) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -334,6 +335,7 @@ public class BuilderUtils { report.setProgress(progress); report.setApplicationType(appType); report.setAMRMToken(amRmToken); + report.setApplicationTags(tags); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 086a6d823e3..2f8526a7c71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -45,6 +45,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -445,9 +446,11 @@ public class ClientRMService extends AbstractService implements request.getApplicationStates(); Set users = request.getUsers(); Set queues = request.getQueues(); + Set tags = request.getApplicationTags(); long limit = request.getLimit(); LongRange start = request.getStartRange(); LongRange finish = request.getFinishRange(); + ApplicationsRequestScope scope = request.getScope(); final Map apps = rmContext.getRMApps(); Iterator appsIter; @@ -494,6 +497,17 @@ public class ClientRMService extends AbstractService implements List reports = new ArrayList(); while (appsIter.hasNext() && reports.size() < limit) { RMApp application = appsIter.next(); + + // Check if current application falls under the specified scope + boolean allowAccess = checkAccess(callerUGI, application.getUser(), + ApplicationAccessType.VIEW_APP, application); + if (scope == ApplicationsRequestScope.OWN && + !callerUGI.getUserName().equals(application.getUser())) { + continue; + } else if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) { + continue; + } + if (applicationTypes != null && !applicationTypes.isEmpty()) { String appTypeToMatch = caseSensitive ? application.getApplicationType() @@ -523,8 +537,23 @@ public class ClientRMService extends AbstractService implements continue; } - boolean allowAccess = checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.VIEW_APP, application); + if (tags != null && !tags.isEmpty()) { + Set appTags = application.getApplicationTags(); + if (appTags == null || appTags.isEmpty()) { + continue; + } + boolean match = false; + for (String tag : tags) { + if (appTags.contains(tag)) { + match = true; + break; + } + } + if (!match) { + continue; + } + } + reports.add(application.createAndGetApplicationReport( callerUGI.getUserName(), allowAccess)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7dbc4cc2800..7855042db78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -320,7 +320,8 @@ public class RMAppManager implements EventHandler, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, - submitTime, submissionContext.getApplicationType()); + submitTime, submissionContext.getApplicationType(), + submissionContext.getApplicationTags()); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 1809a4bb470..1a7e74ef672 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -194,7 +195,13 @@ public interface RMApp extends EventHandler { * Returns the application type * @return the application type. */ - String getApplicationType(); + String getApplicationType(); + + /** + * Get tags for the application + * @return tags corresponding to the application + */ + Set getApplicationTags(); /** * Check whether this application is safe to terminate. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index d4cf416dd33..edbe676badd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -104,6 +104,7 @@ public class RMAppImpl implements RMApp, Recoverable { private final long submitTime; private final Set updatedNodes = new HashSet(); private final String applicationType; + private final Set applicationTags; // Mutable fields private long startTime; @@ -302,9 +303,9 @@ public class RMAppImpl implements RMApp, Recoverable { public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, - ApplicationSubmissionContext submissionContext, - YarnScheduler scheduler, - ApplicationMasterService masterService, long submitTime, String applicationType) { + ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, + ApplicationMasterService masterService, long submitTime, + String applicationType, Set applicationTags) { this.applicationId = applicationId; this.name = name; @@ -320,6 +321,7 @@ public class RMAppImpl implements RMApp, Recoverable { this.submitTime = submitTime; this.startTime = System.currentTimeMillis(); this.applicationType = applicationType; + this.applicationTags = applicationTags; int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -553,7 +555,7 @@ public class RMAppImpl implements RMApp, Recoverable { createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken); + amrmToken, applicationTags); } finally { this.readLock.unlock(); } @@ -1085,6 +1087,11 @@ public class RMAppImpl implements RMApp, Recoverable { return this.applicationType; } + @Override + public Set getApplicationTags() { + return this.applicationTags; + } + @Override public boolean isAppSafeToTerminate() { RMAppState state = getState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index ed147fbb733..445a5a2ca99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -110,6 +110,7 @@ public class AppBlock extends HtmlBlock { _("User:", app.getUser()). _("Name:", app.getName()). _("Application Type:", app.getApplicationType()). + _("Application Tags:", app.getApplicationTags()). _("State:", app.getState()). _("FinalStatus:", app.getFinalStatus()). _("Started:", Times.format(app.getStartTime())). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index e18c04740cb..e854e2ce525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -261,12 +261,14 @@ public class RMWebServices { @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, - @QueryParam("applicationTypes") Set applicationTypes) { + @QueryParam("applicationTypes") Set applicationTypes, + @QueryParam("applicationTags") Set applicationTags) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; boolean checkAppStates = false; + boolean checkAppTags = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -327,6 +329,11 @@ public class RMWebServices { checkAppTypes = true; } + Set appTags = parseQueries(applicationTags, false); + if (!appTags.isEmpty()) { + checkAppTags = true; + } + // stateQuery is deprecated. if (stateQuery != null && !stateQuery.isEmpty()) { statesQuery.add(stateQuery); @@ -354,6 +361,10 @@ public class RMWebServices { request.setApplicationTypes(appTypes); } + if (checkAppTags) { + request.setApplicationTags(appTags); + } + if (checkAppStates) { request.setApplicationStates(appStates); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 295e1f3a41a..a4f4873ed4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import com.google.common.base.Joiner; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; @@ -67,6 +67,7 @@ public class AppInfo { protected String diagnostics; protected long clusterId; protected String applicationType; + protected String applicationTags = ""; // these are only allowed if acls allow protected long startedTime; @@ -117,6 +118,9 @@ public class AppInfo { if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } + if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { + this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); + } this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { @@ -239,6 +243,10 @@ public class AppInfo { public String getApplicationType() { return this.applicationType; } + + public String getApplicationTags() { + return this.applicationTags; + } public int getRunningContainers() { return this.runningContainers; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1894a115ac6..7c496815ab2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -41,8 +41,10 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Sets; import junit.framework.Assert; +import org.apache.commons.lang.math.LongRange; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +53,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; @@ -72,6 +75,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; @@ -465,6 +469,7 @@ public class TestClientRMService { {MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName()}; ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102), getApplicationId(103)}; + List tags = Arrays.asList("Tag1", "Tag2", "Tag3"); // Submit applications for (int i = 0; i < appIds.length; i++) { @@ -472,7 +477,8 @@ public class TestClientRMService { when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true); SubmitApplicationRequest submitRequest = mockSubmitAppRequest( - appId, appNames[i], queues[i % queues.length]); + appId, appNames[i], queues[i % queues.length], + new HashSet(tags.subList(0, i + 1))); rmService.submitApplication(submitRequest); } @@ -513,6 +519,41 @@ public class TestClientRMService { userSet.add(UserGroupInformation.getCurrentUser().getShortUserName()); assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size()); + + // Check tags + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.ALL, null, null, null, null, null, null, + null, null); + Set tagSet = new HashSet(); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 6, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(0)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 3, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(1)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 2, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(2)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 1, + rmService.getApplications(request).getApplicationList().size()); + + // Check scope + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.VIEWABLE); + assertEquals("Incorrect number of applications for the scope", 6, + rmService.getApplications(request).getApplicationList().size()); + + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.OWN); + assertEquals("Incorrect number of applications for the scope", 3, + rmService.getApplications(request).getApplicationList().size()); } @Test(timeout=4000) @@ -583,6 +624,11 @@ public class TestClientRMService { private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue) { + return mockSubmitAppRequest(appId, name, queue, null); + } + + private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, + String name, String queue, Set tags) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( @@ -596,6 +642,7 @@ public class TestClientRMService { submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); + submissionContext.setApplicationTags(tags); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); @@ -664,7 +711,7 @@ public class TestClientRMService { when(asContext.getMaxAppAttempts()).thenReturn(1); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null , System - .currentTimeMillis(), "YARN")); + .currentTimeMillis(), "YARN", null)); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId3, 1); RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index aa116bf85b2..01e5eea96cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.MockApps; @@ -139,6 +140,11 @@ public abstract class MockAsm extends MockApps { throw new UnsupportedOperationException("Not supported yet."); } + @Override + public Set getApplicationTags() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public void setQueue(String name) { throw new UnsupportedOperationException("Not supported yet."); @@ -235,7 +241,11 @@ public abstract class MockAsm extends MockApps { public int getMaxAppAttempts() { return maxAppAttempts; } - + + @Override + public Set getApplicationTags() { + return null; + } }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index debcffe97dc..ac3751a191c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -217,6 +218,11 @@ public class MockRMApp implements RMApp { return YarnConfiguration.DEFAULT_APPLICATION_TYPE; } + @Override + public Set getApplicationTags() { + return null; + } + @Override public boolean isAppSafeToTerminate() { return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index a8c2ab65c96..5ac9353928e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -230,7 +230,7 @@ public class TestRMAppTransitions { RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b251ce7dd19..892438902de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -621,7 +621,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -647,7 +647,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -1765,7 +1765,7 @@ public class TestFairScheduler { RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 349bae4c384..18350fb30d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1317,8 +1317,8 @@ public class TestRMWebServicesApps extends JerseyTest { public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - // 15 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 19, info.length()); + // 20 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 20, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 73902912571..ac6b4466c27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1123,6 +1123,7 @@ ResourceManager REST API's. * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch * applicationTypes - applications matching the given application types, specified as a comma-separated list. + * applicationTags - applications matching any of the given application tags, specified as a comma-separated list. ------ ** Elements of the (Applications) object From cec369149b4205af0e20dfbe7f2f541f46688fc5 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 5 Feb 2014 05:42:02 +0000 Subject: [PATCH 21/42] HADOOP-10273. Fix 'mvn site'. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564638 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 06d97b1428e..e3a39daa581 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -320,6 +320,8 @@ Release 2.4.0 - UNRELEASED HADOOP-10085. CompositeService should allow adding services while being inited. (Steve Loughran via kasha) + HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/pom.xml b/pom.xml index 27db19ba353..a4f824102cc 100644 --- a/pom.xml +++ b/pom.xml @@ -152,7 +152,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.apache.maven.plugins maven-site-plugin - 3.2 + 3.3 org.apache.maven.wagon @@ -329,7 +329,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs maven-site-plugin - 3.0 + 3.3 attach-descriptor From d6bd920bba0d7cb77ca76c3a79d1ba1e039da9e5 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 5 Feb 2014 06:48:00 +0000 Subject: [PATCH 22/42] HDFS-5709. Improve NameNode upgrade with existing reserved paths and path components. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 61 +++++- .../hadoop/hdfs/protocol/HdfsConstants.java | 13 +- .../server/common/HdfsServerConstants.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 144 ++++++++------ .../hdfs/server/namenode/FSImageFormat.java | 179 +++++++++++++++++- .../hadoop/hdfs/server/namenode/NameNode.java | 39 +++- .../src/site/apt/HdfsUserGuide.apt.vm | 29 ++- .../src/site/xdoc/HdfsSnapshots.xml | 21 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 85 +++++++++ .../namenode/TestNameNodeOptionParsing.java | 88 +++++++++ .../src/test/resources/hadoop-2-reserved.tgz | Bin 0 -> 2838 bytes 12 files changed, 584 insertions(+), 81 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0b68cd35a0e..5b39abfeeaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding error during testing (Haohui Mai via brandonli) + HDFS-5709. Improve NameNode upgrade with existing reserved paths and path + components. (Andrew Wang via atm) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index b58f2732d0a..9274f505a4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -261,6 +261,47 @@ public class DFSUtil { return true; } + /** + * Checks if a string is a valid path component. For instance, components + * cannot contain a ":" or "/", and cannot be equal to a reserved component + * like ".snapshot". + *

+ * The primary use of this method is for validating paths when loading the + * FSImage. During normal NN operation, paths are sometimes allowed to + * contain reserved components. + * + * @return If component is valid + */ + public static boolean isValidNameForComponent(String component) { + if (component.equals(".") || + component.equals("..") || + component.indexOf(":") >= 0 || + component.indexOf("/") >= 0) { + return false; + } + return !isReservedPathComponent(component); + } + + + /** + * Returns if the component is reserved. + * + *

+ * Note that some components are only reserved under certain directories, e.g. + * "/.reserved" is reserved, while "/hadoop/.reserved" is not. + * + * @param component + * @return if the component is reserved + */ + public static boolean isReservedPathComponent(String component) { + for (String reserved : HdfsConstants.RESERVED_PATH_COMPONENTS) { + if (component.equals(reserved)) { + return true; + } + } + return false; + } + /** * Converts a byte array to a string using UTF8 encoding. */ @@ -312,7 +353,25 @@ public class DFSUtil { } return result.toString(); } - + + /** + * Converts a list of path components into a path using Path.SEPARATOR. + * + * @param components Path components + * @return Combined path as a UTF-8 string + */ + public static String strings2PathString(String[] components) { + if (components.length == 0) { + return ""; + } + if (components.length == 1) { + if (components[0] == null || components[0].isEmpty()) { + return Path.SEPARATOR; + } + } + return Joiner.on(Path.SEPARATOR).join(components); + } + /** * Given a list of path components returns a byte array */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index da934c36992..d1c7e143c90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; /************************************ * Some handy constants @@ -108,7 +109,17 @@ public class HdfsConstants { */ public static final int LAYOUT_VERSION = LayoutVersion .getCurrentLayoutVersion(); - + + /** + * Path components that are reserved in HDFS. + *

+ * .reserved is only reserved under root ("/"). + */ + public static final String[] RESERVED_PATH_COMPONENTS = new String[] { + HdfsConstants.DOT_SNAPSHOT_DIR, + FSDirectory.DOT_RESERVED_STRING + }; + /** * A special path component contained in the path for a snapshot file/dir */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 50f6e730408..77fe7c6dd06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -59,7 +59,8 @@ public final class HdfsServerConstants { INITIALIZESHAREDEDITS("-initializeSharedEdits"), RECOVER ("-recover"), FORCE("-force"), - NONINTERACTIVE("-nonInteractive"); + NONINTERACTIVE("-nonInteractive"), + RENAMERESERVED("-renameReserved"); private final String name; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index ce2cf2f7be7..7433b6b1094 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormat.renameReservedPathsOnUpgrade; import static org.apache.hadoop.util.Time.now; import java.io.FilterInputStream; @@ -292,8 +293,10 @@ public class FSEditLogLoader { switch (op.opCode) { case OP_ADD: { AddCloseOp addCloseOp = (AddCloseOp)op; + final String path = + renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + addCloseOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); @@ -304,9 +307,9 @@ public class FSEditLogLoader { // 3. OP_ADD to open file for append // See if the file already exists (persistBlocks call) - final INodesInPath iip = fsDir.getLastINodeInPath(addCloseOp.path); + final INodesInPath iip = fsDir.getLastINodeInPath(path); final INodeFile oldFile = INodeFile.valueOf( - iip.getINode(0), addCloseOp.path, true); + iip.getINode(0), path, true); INodeFile newFile = oldFile; if (oldFile == null) { // this is OP_ADD on a new file (case 1) // versions > 0 support per file replication @@ -319,10 +322,10 @@ public class FSEditLogLoader { inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId); newFile = fsDir.unprotectedAddFile(inodeId, - addCloseOp.path, addCloseOp.permissions, replication, + path, addCloseOp.permissions, replication, addCloseOp.mtime, addCloseOp.atime, addCloseOp.blockSize, true, addCloseOp.clientName, addCloseOp.clientMachine); - fsNamesys.leaseManager.addLease(addCloseOp.clientName, addCloseOp.path); + fsNamesys.leaseManager.addLease(addCloseOp.clientName, path); // add the op into retry cache if necessary if (toAddRetryCache) { @@ -338,11 +341,11 @@ public class FSEditLogLoader { FSNamesystem.LOG.debug("Reopening an already-closed file " + "for append"); } - LocatedBlock lb = fsNamesys.prepareFileForWrite(addCloseOp.path, + LocatedBlock lb = fsNamesys.prepareFileForWrite(path, oldFile, addCloseOp.clientName, addCloseOp.clientMachine, null, false, iip.getLatestSnapshotId(), false); - newFile = INodeFile.valueOf(fsDir.getINode(addCloseOp.path), - addCloseOp.path, true); + newFile = INodeFile.valueOf(fsDir.getINode(path), + path, true); // add the op into retry cache is necessary if (toAddRetryCache) { @@ -363,16 +366,17 @@ public class FSEditLogLoader { } case OP_CLOSE: { AddCloseOp addCloseOp = (AddCloseOp)op; - + final String path = + renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + addCloseOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); } - final INodesInPath iip = fsDir.getLastINodeInPath(addCloseOp.path); - final INodeFile file = INodeFile.valueOf(iip.getINode(0), addCloseOp.path); + final INodesInPath iip = fsDir.getLastINodeInPath(path); + final INodeFile file = INodeFile.valueOf(iip.getINode(0), path); // Update the salient file attributes. file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID); @@ -386,24 +390,26 @@ public class FSEditLogLoader { // could show up twice in a row. But after that version, this // should be fixed, so we should treat it as an error. throw new IOException( - "File is not under construction: " + addCloseOp.path); + "File is not under construction: " + path); } // One might expect that you could use removeLease(holder, path) here, // but OP_CLOSE doesn't serialize the holder. So, remove by path. if (file.isUnderConstruction()) { - fsNamesys.leaseManager.removeLeaseWithPrefixPath(addCloseOp.path); + fsNamesys.leaseManager.removeLeaseWithPrefixPath(path); file.toCompleteFile(file.getModificationTime()); } break; } case OP_UPDATE_BLOCKS: { UpdateBlocksOp updateOp = (UpdateBlocksOp)op; + final String path = + renameReservedPathsOnUpgrade(updateOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + updateOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + updateOp.blocks.length); } - INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(updateOp.path), - updateOp.path); + INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(path), + path); // Update in-memory data structures updateBlocks(fsDir, updateOp, oldFile); @@ -414,7 +420,7 @@ public class FSEditLogLoader { } case OP_ADD_BLOCK: { AddBlockOp addBlockOp = (AddBlockOp) op; - String path = addBlockOp.getPath(); + String path = renameReservedPathsOnUpgrade(addBlockOp.getPath(), logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug(op.opCode + ": " + path + " new block id : " + addBlockOp.getLastBlock().getBlockId()); @@ -428,14 +434,20 @@ public class FSEditLogLoader { SetReplicationOp setReplicationOp = (SetReplicationOp)op; short replication = fsNamesys.getBlockManager().adjustReplication( setReplicationOp.replication); - fsDir.unprotectedSetReplication(setReplicationOp.path, + fsDir.unprotectedSetReplication( + renameReservedPathsOnUpgrade(setReplicationOp.path, logVersion), replication, null); break; } case OP_CONCAT_DELETE: { ConcatDeleteOp concatDeleteOp = (ConcatDeleteOp)op; - fsDir.unprotectedConcat(concatDeleteOp.trg, concatDeleteOp.srcs, - concatDeleteOp.timestamp); + String trg = renameReservedPathsOnUpgrade(concatDeleteOp.trg, logVersion); + String[] srcs = new String[concatDeleteOp.srcs.length]; + for (int i=0; i removedINodes = new ChunkedArrayList(); + final String snapshotRoot = + renameReservedPathsOnUpgrade(deleteSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().deleteSnapshot( - deleteSnapshotOp.snapshotRoot, deleteSnapshotOp.snapshotName, + snapshotRoot, deleteSnapshotOp.snapshotName, collectedBlocks, removedINodes); fsNamesys.removeBlocksAndUpdateSafemodeTotal(collectedBlocks); collectedBlocks.clear(); @@ -617,8 +647,11 @@ public class FSEditLogLoader { } case OP_RENAME_SNAPSHOT: { RenameSnapshotOp renameSnapshotOp = (RenameSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(renameSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().renameSnapshot( - renameSnapshotOp.snapshotRoot, renameSnapshotOp.snapshotOldName, + snapshotRoot, renameSnapshotOp.snapshotOldName, renameSnapshotOp.snapshotNewName); if (toAddRetryCache) { @@ -629,14 +662,19 @@ public class FSEditLogLoader { } case OP_ALLOW_SNAPSHOT: { AllowSnapshotOp allowSnapshotOp = (AllowSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(allowSnapshotOp.snapshotRoot, logVersion); fsNamesys.getSnapshotManager().setSnapshottable( - allowSnapshotOp.snapshotRoot, false); + snapshotRoot, false); break; } case OP_DISALLOW_SNAPSHOT: { DisallowSnapshotOp disallowSnapshotOp = (DisallowSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(disallowSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().resetSnapshottable( - disallowSnapshotOp.snapshotRoot); + snapshotRoot); break; } case OP_SET_GENSTAMP_V2: { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 385917e8e0e..3ad258a4512 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -32,12 +32,13 @@ import java.security.DigestOutputStream; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; -import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -45,13 +46,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; -import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; @@ -67,6 +70,10 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * Contains inner classes for reading or writing the on-disk format for @@ -405,7 +412,8 @@ public class FSImageFormat { } /** - * load fsimage files assuming only local names are stored + * load fsimage files assuming only local names are stored. Used when + * snapshots are not supported by the layout version. * * @param numFiles number of files expected to be read * @param in image input stream @@ -521,6 +529,8 @@ public class FSImageFormat { */ private int loadDirectory(DataInput in, Counter counter) throws IOException { String parentPath = FSImageSerialization.readString(in); + // Rename .snapshot paths if we're doing an upgrade + parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion()); final INodeDirectory parent = INodeDirectory.valueOf( namesystem.dir.rootDir.getNode(parentPath, true), parentPath); return loadChildren(parent, in, counter); @@ -580,11 +590,9 @@ public class FSImageFormat { */ private void addToParent(INodeDirectory parent, INode child) { FSDirectory fsDir = namesystem.dir; - if (parent == fsDir.rootDir && FSDirectory.isReservedName(child)) { - throw new HadoopIllegalArgumentException("File name \"" - + child.getLocalName() + "\" is reserved. Please " - + " change the name of the existing file or directory to another " - + "name before upgrading to this release."); + if (parent == fsDir.rootDir) { + child.setLocalName(renameReservedRootComponentOnUpgrade( + child.getLocalNameBytes(), getLayoutVersion())); } // NOTE: This does not update space counts for parents if (!parent.addChild(child)) { @@ -621,7 +629,9 @@ public class FSImageFormat { public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in, boolean updateINodeMap, Counter counter) throws IOException { - final byte[] localName = FSImageSerialization.readLocalName(in); + byte[] localName = FSImageSerialization.readLocalName(in); + localName = + renameReservedComponentOnUpgrade(localName, getLayoutVersion()); INode inode = loadINode(localName, isSnapshotINode, in, counter); if (updateINodeMap && LayoutVersion.supports(Feature.ADD_INODE_ID, getLayoutVersion())) { @@ -926,7 +936,156 @@ public class FSImageFormat { return snapshotMap.get(in.readInt()); } } - + + @VisibleForTesting + public static TreeMap renameReservedMap = + new TreeMap(); + + /** + * Use the default key-value pairs that will be used to determine how to + * rename reserved paths on upgrade. + */ + @VisibleForTesting + public static void useDefaultRenameReservedPairs() { + renameReservedMap.clear(); + for (String key: HdfsConstants.RESERVED_PATH_COMPONENTS) { + renameReservedMap.put( + key, + key + "." + LayoutVersion.getCurrentLayoutVersion() + "." + + "UPGRADE_RENAMED"); + } + } + + /** + * Set the key-value pairs that will be used to determine how to rename + * reserved paths on upgrade. + */ + @VisibleForTesting + public static void setRenameReservedPairs(String renameReserved) { + // Clear and set the default values + useDefaultRenameReservedPairs(); + // Overwrite with provided values + setRenameReservedMapInternal(renameReserved); + } + + private static void setRenameReservedMapInternal(String renameReserved) { + Collection pairs = + StringUtils.getTrimmedStringCollection(renameReserved); + for (String p : pairs) { + String[] pair = StringUtils.split(p, '/', '='); + Preconditions.checkArgument(pair.length == 2, + "Could not parse key-value pair " + p); + String key = pair[0]; + String value = pair[1]; + Preconditions.checkArgument(DFSUtil.isReservedPathComponent(key), + "Unknown reserved path " + key); + Preconditions.checkArgument(DFSUtil.isValidNameForComponent(value), + "Invalid rename path for " + key + ": " + value); + LOG.info("Will rename reserved path " + key + " to " + value); + renameReservedMap.put(key, value); + } + } + + /** + * When upgrading from an old version, the filesystem could contain paths + * that are now reserved in the new version (e.g. .snapshot). This renames + * these new reserved paths to a user-specified value to avoid collisions + * with the reserved name. + * + * @param path Old path potentially containing a reserved path + * @return New path with reserved path components renamed to user value + */ + static String renameReservedPathsOnUpgrade(String path, + final int layoutVersion) { + final String oldPath = path; + // If any known LVs aren't supported, we're doing an upgrade + if (!LayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Only need to worry about the root directory + if (components.length > 1) { + components[1] = DFSUtil.bytes2String( + renameReservedRootComponentOnUpgrade( + DFSUtil.string2Bytes(components[1]), + layoutVersion)); + path = DFSUtil.strings2PathString(components); + } + } + if (!LayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Special case the root path + if (components.length == 0) { + return path; + } + for (int i=0; i] ] | [" + StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + StartupOption.IMPORT.getName() + "] | [" @@ -1056,7 +1058,8 @@ public class NameNode implements NameNodeStatusMXBean { out.println(USAGE + "\n"); } - private static StartupOption parseArguments(String args[]) { + @VisibleForTesting + static StartupOption parseArguments(String args[]) { int argsLen = (args == null) ? 0 : args.length; StartupOption startOpt = StartupOption.REGULAR; for(int i=0; i < argsLen; i++) { @@ -1103,11 +1106,33 @@ public class NameNode implements NameNodeStatusMXBean { startOpt = StartupOption.CHECKPOINT; } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.UPGRADE; - // might be followed by two args - if (i + 2 < argsLen - && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { - i += 2; - startOpt.setClusterId(args[i]); + /* Can be followed by CLUSTERID with a required parameter or + * RENAMERESERVED with an optional parameter + */ + while (i + 1 < argsLen) { + String flag = args[i + 1]; + if (flag.equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { + if (i + 2 < argsLen) { + i += 2; + startOpt.setClusterId(args[i]); + } else { + LOG.fatal("Must specify a valid cluster ID after the " + + StartupOption.CLUSTERID.getName() + " flag"); + return null; + } + } else if (flag.equalsIgnoreCase(StartupOption.RENAMERESERVED + .getName())) { + if (i + 2 < argsLen) { + FSImageFormat.setRenameReservedPairs(args[i + 2]); + i += 2; + } else { + FSImageFormat.useDefaultRenameReservedPairs(); + i += 1; + } + } else { + LOG.fatal("Unknown upgrade flag " + flag); + return null; + } } } else if (StartupOption.ROLLBACK.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.ROLLBACK; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm index b84da5991b3..9d6aeb97fb5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm @@ -435,7 +435,7 @@ HDFS Users Guide state it was in before the upgrade. HDFS upgrade is described in more detail in {{{http://wiki.apache.org/hadoop/Hadoop_Upgrade}Hadoop Upgrade}} Wiki page. HDFS can have one such backup at a time. Before upgrading, - administrators need to remove existing backupusing bin/hadoop dfsadmin + administrators need to remove existing backup using bin/hadoop dfsadmin <<<-finalizeUpgrade>>> command. The following briefly describes the typical upgrade procedure: @@ -459,6 +459,33 @@ HDFS Users Guide * start the cluster with rollback option. (<<>>). + When upgrading to a new version of HDFS, it is necessary to rename or + delete any paths that are reserved in the new version of HDFS. If the + NameNode encounters a reserved path during upgrade, it will print an + error like the following: + + <<< /.reserved is a reserved path and .snapshot is a + reserved path component in this version of HDFS. Please rollback and delete + or rename this path, or upgrade with the -renameReserved [key-value pairs] + option to automatically rename these paths during upgrade.>>> + + Specifying <<<-upgrade -renameReserved [optional key-value pairs]>>> causes + the NameNode to automatically rename any reserved paths found during + startup. For example, to rename all paths named <<<.snapshot>>> to + <<<.my-snapshot>>> and <<<.reserved>>> to <<<.my-reserved>>>, a user would + specify <<<-upgrade -renameReserved + .snapshot=.my-snapshot,.reserved=.my-reserved>>>. + + If no key-value pairs are specified with <<<-renameReserved>>>, the + NameNode will then suffix reserved paths with + <<<..UPGRADE_RENAMED>>>, e.g. + <<<.snapshot.-51.UPGRADE_RENAMED>>>. + + There are some caveats to this renaming process. It's recommended, + if possible, to first <<>> before upgrading. + This is because data inconsistency can result if an edit log operation + refers to the destination of an automatically renamed file. + * File Permissions and Security The file permissions are designed to be similar to file permissions on diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index 9aecf9cd55a..bd499c79c80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd"> - HFDS Snapshots + HDFS Snapshots @@ -99,15 +99,22 @@

  • Copying a file from snapshot s0: hdfs dfs -cp /foo/.snapshot/s0/bar /tmp
    • -

    - Note that the name ".snapshot" is now a reserved file name in HDFS - so that users cannot create a file/directory with ".snapshot" as the name. - If ".snapshot" is used in a previous version of HDFS, it must be renamed before upgrade; - otherwise, upgrade will fail. -

    +
    + +

    + The HDFS snapshot feature introduces a new reserved path name used to + interact with snapshots: .snapshot. When upgrading from an + older version of HDFS, existing paths named .snapshot need + to first be renamed or deleted to avoid conflicting with the reserved path. + See the upgrade section in + the HDFS user guide + for more information.

    + +
    +

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java index c686e4ebe7c..fda4e835304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.TreeMap; @@ -43,7 +44,9 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import org.junit.Test; @@ -67,6 +70,7 @@ public class TestDFSUpgradeFromImage { private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt"; private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz"; private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz"; + private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz"; private static class ReferenceFileInfo { String path; @@ -320,6 +324,87 @@ public class TestDFSUpgradeFromImage { assertEquals("Upgrade did not fail with bad MD5", 1, md5failures); } } + + /** + * Test upgrade from 2.0 image with a variety of .snapshot and .reserved + * paths to test renaming on upgrade + */ + @Test + public void testUpgradeFromRel2ReservedImage() throws IOException { + unpackStorage(HADOOP2_RESERVED_IMAGE); + MiniDFSCluster cluster = null; + // Try it once without setting the upgrade flag to ensure it fails + try { + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "reserved path component in this version", + e); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + // Try it again with a custom rename string + try { + FSImageFormat.setRenameReservedPairs( + ".snapshot=.user-snapshot," + + ".reserved=.my-reserved"); + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + // Make sure the paths were renamed as expected + DistributedFileSystem dfs = cluster.getFileSystem(); + ArrayList toList = new ArrayList(); + ArrayList found = new ArrayList(); + toList.add(new Path("/")); + while (!toList.isEmpty()) { + Path p = toList.remove(0); + FileStatus[] statuses = dfs.listStatus(p); + for (FileStatus status: statuses) { + final String path = status.getPath().toUri().getPath(); + System.out.println("Found path " + path); + found.add(path); + if (status.isDirectory()) { + toList.add(status.getPath()); + } + } + } + String[] expected = new String[] { + "/edits", + "/edits/.reserved", + "/edits/.user-snapshot", + "/edits/.user-snapshot/editsdir", + "/edits/.user-snapshot/editsdir/editscontents", + "/edits/.user-snapshot/editsdir/editsdir2", + "/image", + "/image/.reserved", + "/image/.user-snapshot", + "/image/.user-snapshot/imagedir", + "/image/.user-snapshot/imagedir/imagecontents", + "/image/.user-snapshot/imagedir/imagedir2", + "/.my-reserved", + "/.my-reserved/edits-touch", + "/.my-reserved/image-touch" + }; + + for (String s: expected) { + assertTrue("Did not find expected path " + s, found.contains(s)); + } + assertEquals("Found an unexpected path while listing filesystem", + found.size(), expected.length); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } static void recoverAllLeases(DFSClient dfs, Path path) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java new file mode 100644 index 00000000000..d43eda0cefd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java @@ -0,0 +1,88 @@ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.junit.Test; + +public class TestNameNodeOptionParsing { + + @Test(timeout = 10000) + public void testUpgrade() { + StartupOption opt = null; + // UPGRADE is set, but nothing else + opt = NameNode.parseArguments(new String[] {"-upgrade"}); + assertEquals(opt, StartupOption.UPGRADE); + assertNull(opt.getClusterId()); + assertTrue(FSImageFormat.renameReservedMap.isEmpty()); + // cluster ID is set + opt = NameNode.parseArguments(new String[] { "-upgrade", "-clusterid", + "mycid" }); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertTrue(FSImageFormat.renameReservedMap.isEmpty()); + // Everything is set + opt = NameNode.parseArguments(new String[] { "-upgrade", "-clusterid", + "mycid", "-renameReserved", + ".snapshot=.my-snapshot,.reserved=.my-reserved" }); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertEquals(".my-snapshot", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals(".my-reserved", + FSImageFormat.renameReservedMap.get(".reserved")); + // Reset the map + FSImageFormat.renameReservedMap.clear(); + // Everything is set, but in a different order + opt = NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.snapshot=.my-snapshot", "-clusterid", + "mycid"}); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertEquals(".my-snapshot", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals(".my-reserved", + FSImageFormat.renameReservedMap.get(".reserved")); + // Try the default renameReserved + opt = NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved"}); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals( + ".snapshot." + LayoutVersion.getCurrentLayoutVersion() + + ".UPGRADE_RENAMED", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals( + ".reserved." + LayoutVersion.getCurrentLayoutVersion() + + ".UPGRADE_RENAMED", + FSImageFormat.renameReservedMap.get(".reserved")); + + // Try some error conditions + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.not-reserved=.my-not-reserved" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Unknown reserved path", e); + } + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.snapshot=.snapshot" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Invalid rename path", e); + } + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".snapshot=.reserved" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Invalid rename path", e); + } + opt = NameNode.parseArguments(new String[] { "-upgrade", "-cid"}); + assertNull(opt); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3cb2ee63f0e2624968dcd97836a608b88e1a36c2 GIT binary patch literal 2838 zcmeH}`&-h79>+bM<<2pUo|(tI!Etlt(J8e|ElM`+pw5)E(o_`FO{XAeGR@FfD{RG- z@{)-;miHT8P*hYTC3R+sN`i_4MqU9|+|i2L1|o8sJzD6$S+xhuK;10RkLUaSD(1-s5?Gay#(gf;`He z#5OH<6JZN1y}Z7Tubim%Xx45EweO5B#fwOlgicHJKwXr4i&~aZst>oVBW>=jCOuxv z&3GJMO`=@@7)_UfbyQhkNqj_>BI>g(~37=EaW!aI(3KbS9 z;=b=_d#}s#{6Ds|<{V}8$Y#h-c?>N|^X-^)WMi&E6UkX6QInvCV(*q`4~;vlmU|QZ zHaI@N-F&ckF1{_DDS2*K)DV6)2Q% zUsw<$|JC}%iZJU=#iycMXGCWB1uY&P&7$QAj<;lM@2UlumO(^QQ?rM2n?7$PuRl%! zclzkctYmqq9$*{=blfhTHjB<(4esF*S0~ko*!iI4(-`Ehzt2R%R~y0C*T~9eR$^i{ z*%Ug9c|i|Lcf}CN*8tvIyU?E-S#GAoo`f88SJRDSJyJT06TPb)gCdwjfe7Xg4D#_r zg&zA7b_*7w`fVdxgZOR{O;L4Iuou-zcoO#AOM*^0#Pi+{>D*rSGSxxA^hdu;xYPY6 zjur+WdcRr%h0psC-u%HP5!&Z^8%&liK~VTeSfg&Zo%v6RjWV>1KI(z-xT+9`iZQkA zXMT7$j;+PN3D{yQ1QGby3z-|Q7Me1k8H=3$)a;X-n1YiwIqQMW^xLYMh%al95TY71 zT`sx$z*vID8xlzbUiP#*cid)4s^1I(da#e%4PoZ;G;O%{$ZZIzX$p6}mF#JWxbhf% zYKF>(YvNsJ#KISH!Pep$KV+DzVwD*he7Fw;J`ngnBd}Z%gvlAr4+tq6|DVsZg)~8x z-;vt9`Ky7au6E{vTygr>F20N6>iZxl7G@NGpg-9*AjLNqPl7s>$2%V~avK+AjhqOt z7#KT$o>t>hi>%jkSRP!@g1j)xrEdy6FZ;`*8HXlwo11HDTJsl;77`j<2%`Ts-09`R zi286x5sCHXTtY*A>LHWujy=N3K}!WZXMJsA3vx%(8nufjtC3G(U0sd38=V_zQq@~9 z$_xvMxV)p<9_07zjbbHr$&>S%2Hk}`=nI5h);*& ze+aPA6NPG{wV9XVv^d*_{kSl2X$qpM#U7(b-GKwcZmUC&>Pg-rM zWuup1%iZ>(xG>e^7#0-wr0mskO5%}zvb!UUbgUx=xEUj_suaf|!5&w4#yH*(+7z(5 z^Xzp}KJ;vIZe`3L-ySqR`~IoG0+dKVf@w4+nCd15Oo=$oU6v$cJ#N| zbZN|c6*Q1E+LegVmp7x52<3r^!B*4|$CA92E+-7J1ZBA+m(-u1YXf+R?SPIS`lP`m z)fu6*bDgc_7a(JK@YR9nE<}X7KBzF&Keyu%C1ov_*-c0xV1ge)8;b_I6#nvs=Iw@& ztc-}X-yy2g8+}NoEt=bL!=v~)4hqU(PT};Q8s~YUUYHyCA*{~jkH(hmK7V6zx+9Te zVaH`J%r@Yj+OWmju6ufZ4A$%2>L!bZgrF+N!g}fnI>4J{E&&-HzPoxnxmsom{%cDz R(m2VWuLLw?@YMl>KLII!>~8=7 literal 0 HcmV?d00001 From 484faadffec97f8e454cf18b821f6ef12a27616c Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 5 Feb 2014 17:02:26 +0000 Subject: [PATCH 23/42] YARN-1636. Augmented Application-history server's web-services to also expose new APIs for retrieving and storing timeline information. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564829 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 9 + .../webapp/YarnJacksonJaxbJsonProvider.java | 58 ++++ .../src/main/resources/yarn-default.xml | 8 + .../ApplicationHistoryServer.java | 18 +- .../webapp/AHSWebApp.java | 11 +- .../webapp/ATSWebServices.java | 297 ++++++++++++++++++ .../TestApplicationHistoryServer.java | 2 +- .../webapp/TestATSWebServices.java | 212 +++++++++++++ 9 files changed, 615 insertions(+), 4 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ee4ac79d57b..b2f70e41afd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,10 @@ Release 2.4.0 - UNRELEASED YARN-1461. Added tags for YARN applications and changed RM to handle them. (Karthik Kambatla via zjshen) + YARN-1636. Augmented Application-history server's web-services to also expose + new APIs for retrieving and storing timeline information. (Zhijie Shen via + vinodkv) + IMPROVEMENTS YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d0df7d07f34..9e3619e7f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1028,6 +1028,15 @@ public class YarnConfiguration extends Configuration { public static final String AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = AHS_PREFIX + "webapp.spnego-keytab-file"; + //////////////////////////////// + // ATS Configs + //////////////////////////////// + + public static final String ATS_PREFIX = YARN_PREFIX + "ats."; + + /** ATS store class */ + public static final String ATS_STORE = ATS_PREFIX + "store.class"; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java new file mode 100644 index 00000000000..100e91f7c8c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.webapp; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Provider; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider; +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; +import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; + +import com.google.inject.Singleton; + +/** + * YARN's implementation of JAX-RS abstractions based on + * {@link JacksonJaxbJsonProvider} needed for deserialize JSON content to or + * serialize it from POJO objects. + */ +@Singleton +@Provider +@Unstable +@Private +public class YarnJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider { + + public YarnJacksonJaxbJsonProvider() { + super(); + } + + @Override + public ObjectMapper locateMapper(Class type, MediaType mediaType) { + ObjectMapper mapper = super.locateMapper(type, mediaType); + AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); + mapper.setAnnotationIntrospector(introspector); + mapper.getSerializationConfig() + .setSerializationInclusion(Inclusion.NON_NULL); + return mapper; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a5906986665..6c8c1a77e14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1137,6 +1137,14 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore + + + + Store class name for application timeline store + yarn.ats.store.class + org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore + + The interval that the yarn client library uses to poll the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 3a864c80698..4ec986065b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -27,11 +27,14 @@ import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -51,6 +54,7 @@ public class ApplicationHistoryServer extends CompositeService { ApplicationHistoryClientService ahsClientService; ApplicationHistoryManager historyManager; + ApplicationTimelineStore timelineStore; private WebApp webApp; public ApplicationHistoryServer() { @@ -63,6 +67,8 @@ public class ApplicationHistoryServer extends CompositeService { ahsClientService = createApplicationHistoryClientService(historyManager); addService(ahsClientService); addService((Service) historyManager); + timelineStore = createApplicationTimelineStore(conf); + addIfService(timelineStore); super.serviceInit(conf); } @@ -135,6 +141,15 @@ public class ApplicationHistoryServer extends CompositeService { return new ApplicationHistoryManagerImpl(); } + protected ApplicationTimelineStore createApplicationTimelineStore( + Configuration conf) { + // TODO: need to replace the MemoryApplicationTimelineStore.class with the + // LevelDB implementation + return ReflectionUtils.newInstance(conf.getClass( + YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class, + ApplicationTimelineStore.class), conf); + } + protected void startWebApp() { String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); LOG.info("Instantiating AHSWebApp at " + bindAddress); @@ -148,7 +163,8 @@ public class ApplicationHistoryServer extends CompositeService { YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(bindAddress).start(new AHSWebApp(historyManager)); + .at(bindAddress) + .start(new AHSWebApp(historyManager, timelineStore)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 81f838396d7..d2cfc32dab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -21,24 +21,31 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; public class AHSWebApp extends WebApp implements YarnWebParams { private final ApplicationHistoryManager applicationHistoryManager; + private final ApplicationTimelineStore applicationTimelineStore; - public AHSWebApp(ApplicationHistoryManager applicationHistoryManager) { + public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, + ApplicationTimelineStore applicationTimelineStore) { this.applicationHistoryManager = applicationHistoryManager; + this.applicationTimelineStore = applicationTimelineStore; } @Override public void setup() { - bind(JAXBContextResolver.class); + bind(YarnJacksonJaxbJsonProvider.class); bind(AHSWebServices.class); + bind(ATSWebServices.class); bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); + bind(ApplicationTimelineStore.class).toInstance(applicationTimelineStore); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java new file mode 100644 index 00000000000..4ea501d89a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineReader.Field; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.NameValuePair; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +@Path("/ws/v1/apptimeline") +//TODO: support XML serialization/deserialization +public class ATSWebServices { + + private ApplicationTimelineStore store; + + @Inject + public ATSWebServices(ApplicationTimelineStore store) { + this.store = store; + } + + @XmlRootElement(name = "about") + @XmlAccessorType(XmlAccessType.NONE) + @Public + @Unstable + public static class AboutInfo { + + private String about; + + public AboutInfo() { + + } + + public AboutInfo(String about) { + this.about = about; + } + + @XmlElement(name = "About") + public String getAbout() { + return about; + } + + public void setAbout(String about) { + this.about = about; + } + + } + + /** + * Return the description of the application timeline web services. + */ + @GET + @Path("/") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public AboutInfo about( + @Context HttpServletRequest req, + @Context HttpServletResponse res) { + init(res); + return new AboutInfo("Application Timeline API"); + } + + /** + * Return a list of entities that match the given parameters. + */ + @GET + @Path("/{entityType}") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSEntities getEntities( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("entityType") String entityType, + @QueryParam("primaryFilter") String primaryFilter, + @QueryParam("secondaryFilter") String secondaryFilter, + @QueryParam("windowStart") String windowStart, + @QueryParam("windowEnd") String windowEnd, + @QueryParam("limit") String limit, + @QueryParam("fields") String fields) { + init(res); + ATSEntities entities = null; + try { + entities = store.getEntities( + parseStr(entityType), + parseLongStr(limit), + parseLongStr(windowStart), + parseLongStr(windowEnd), + parsePairStr(primaryFilter, ":"), + parsePairsStr(secondaryFilter, ",", ":"), + parseFieldsStr(fields, ",")); + } catch (NumberFormatException e) { + throw new BadRequestException( + "windowStart, windowEnd or limit is not a numeric value."); + } catch (IllegalArgumentException e) { + throw new BadRequestException("requested invalid field."); + } + if (entities == null) { + return new ATSEntities(); + } + return entities; + } + + /** + * Return a single entity of the given entity type and Id. + */ + @GET + @Path("/{entityType}/{entityId}") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSEntity getEntity( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("entityType") String entityType, + @PathParam("entityId") String entityId, + @QueryParam("fields") String fields) { + init(res); + ATSEntity entity = null; + try { + entity = + store.getEntity(parseStr(entityId), parseStr(entityType), + parseFieldsStr(fields, ",")); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + "requested invalid field."); + } + if (entity == null) { + throw new WebApplicationException(Response.Status.NOT_FOUND); + } + return entity; + } + + /** + * Return the events that match the given parameters. + */ + @GET + @Path("/{entityType}/events") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSEvents getEvents( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("entityType") String entityType, + @QueryParam("entityId") String entityId, + @QueryParam("eventType") String eventType, + @QueryParam("windowStart") String windowStart, + @QueryParam("windowEnd") String windowEnd, + @QueryParam("limit") String limit) { + init(res); + ATSEvents events = null; + try { + events = store.getEntityTimelines( + parseStr(entityType), + parseArrayStr(entityId, ","), + parseLongStr(limit), + parseLongStr(windowStart), + parseLongStr(windowEnd), + parseArrayStr(eventType, ",")); + } catch (NumberFormatException e) { + throw new BadRequestException( + "windowStart, windowEnd or limit is not a numeric value."); + } + if (events == null) { + return new ATSEvents(); + } + return events; + } + + /** + * Store the given entities into the timeline store, and return the errors + * that happen during storing. + */ + @POST + @Path("/") + @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public ATSPutErrors postEntities( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + ATSEntities entities) { + init(res); + if (entities == null) { + return new ATSPutErrors(); + } + return store.put(entities); + } + + private void init(HttpServletResponse response) { + response.setContentType(null); + } + + private static SortedSet parseArrayStr(String str, String delimiter) { + if (str == null) { + return null; + } + SortedSet strSet = new TreeSet(); + String[] strs = str.split(delimiter); + for (String aStr : strs) { + strSet.add(aStr.trim()); + } + return strSet; + } + + private static NameValuePair parsePairStr(String str, String delimiter) { + if (str == null) { + return null; + } + String[] strs = str.split(delimiter, 2); + return new NameValuePair(strs[0].trim(), strs[1].trim()); + } + + private static Collection parsePairsStr( + String str, String aDelimiter, String pDelimiter) { + if (str == null) { + return null; + } + String[] strs = str.split(aDelimiter); + Set pairs = new HashSet(); + for (String aStr : strs) { + pairs.add(parsePairStr(aStr, pDelimiter)); + } + return pairs; + } + + private static EnumSet parseFieldsStr(String str, String delimiter) { + if (str == null) { + return null; + } + String[] strs = str.split(delimiter); + List fieldList = new ArrayList(); + for (String s : strs) { + fieldList.add(Field.valueOf(s.toUpperCase())); + } + if (fieldList.size() == 0) + return null; + Field f1 = fieldList.remove(fieldList.size() - 1); + if (fieldList.size() == 0) + return EnumSet.of(f1); + else + return EnumSet.of(f1, fieldList.toArray(new Field[fieldList.size()])); + } + + private static Long parseLongStr(String str) { + return str == null ? null : Long.parseLong(str.trim()); + } + + private static String parseStr(String str) { + return str == null ? null : str.trim(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 8bd515b6abe..d6d20af189c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -40,7 +40,7 @@ public class TestApplicationHistoryServer { Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(2, historyServer.getServices().size()); + assertEquals(3, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java new file mode 100644 index 00000000000..1ff73ff35a2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; + +import static org.junit.Assert.assertEquals; + +import javax.ws.rs.core.MediaType; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvents; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.TestMemoryApplicationTimelineStore; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + + +public class TestATSWebServices extends JerseyTest { + + private static ApplicationTimelineStore store; + + private Injector injector = Guice.createInjector(new ServletModule() { + + @Override + protected void configureServlets() { + bind(YarnJacksonJaxbJsonProvider.class); + bind(ATSWebServices.class); + bind(GenericExceptionHandler.class); + try{ + store = mockApplicationTimelineStore(); + } catch (Exception e) { + Assert.fail(); + } + bind(ApplicationTimelineStore.class).toInstance(store); + serve("/*").with(GuiceContainer.class); + } + + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + private ApplicationTimelineStore mockApplicationTimelineStore() + throws Exception { + TestMemoryApplicationTimelineStore store = + new TestMemoryApplicationTimelineStore(); + store.setup(); + return store.getApplicationTimelineStore(); + } + + public TestATSWebServices() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter") + .servletPath("/") + .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class)) + .build()); + } + + @Test + public void testAbout() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSWebServices.AboutInfo about = + response.getEntity(ATSWebServices.AboutInfo.class); + Assert.assertNotNull(about); + Assert.assertEquals("Application Timeline API", about.getAbout()); + } + + @Test + public void testGetEntities() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEntities entities = response.getEntity(ATSEntities.class); + Assert.assertNotNull(entities); + Assert.assertEquals(2, entities.getEntities().size()); + ATSEntity entity1 = entities.getEntities().get(0); + Assert.assertNotNull(entity1); + Assert.assertEquals("id_1", entity1.getEntityId()); + Assert.assertEquals("type_1", entity1.getEntityType()); + Assert.assertEquals(123l, entity1.getStartTime().longValue()); + Assert.assertEquals(2, entity1.getEvents().size()); + Assert.assertEquals(2, entity1.getPrimaryFilters().size()); + Assert.assertEquals(4, entity1.getOtherInfo().size()); + ATSEntity entity2 = entities.getEntities().get(1); + Assert.assertNotNull(entity2); + Assert.assertEquals("id_2", entity2.getEntityId()); + Assert.assertEquals("type_1", entity2.getEntityType()); + Assert.assertEquals(123l, entity2.getStartTime().longValue()); + Assert.assertEquals(2, entity2.getEvents().size()); + Assert.assertEquals(2, entity2.getPrimaryFilters().size()); + Assert.assertEquals(4, entity2.getOtherInfo().size()); + } + + @Test + public void testGetEntity() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1").path("id_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEntity entity = response.getEntity(ATSEntity.class); + Assert.assertNotNull(entity); + Assert.assertEquals("id_1", entity.getEntityId()); + Assert.assertEquals("type_1", entity.getEntityType()); + Assert.assertEquals(123l, entity.getStartTime().longValue()); + Assert.assertEquals(2, entity.getEvents().size()); + Assert.assertEquals(2, entity.getPrimaryFilters().size()); + Assert.assertEquals(4, entity.getOtherInfo().size()); + } + + @Test + public void testGetEvents() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .path("type_1").path("events") + .queryParam("entityId", "id_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSEvents events = response.getEntity(ATSEvents.class); + Assert.assertNotNull(events); + Assert.assertEquals(1, events.getAllEvents().size()); + ATSEvents.ATSEventsOfOneEntity partEvents = events.getAllEvents().get(0); + Assert.assertEquals(2, partEvents.getEvents().size()); + ATSEvent event1 = partEvents.getEvents().get(0); + Assert.assertEquals(456l, event1.getTimestamp()); + Assert.assertEquals("end_event", event1.getEventType()); + Assert.assertEquals(1, event1.getEventInfo().size()); + ATSEvent event2 = partEvents.getEvents().get(1); + Assert.assertEquals(123l, event2.getTimestamp()); + Assert.assertEquals("start_event", event2.getEventType()); + Assert.assertEquals(0, event2.getEventInfo().size()); + } + + @Test + public void testPostEntities() throws Exception { + ATSEntities entities = new ATSEntities(); + ATSEntity entity = new ATSEntity(); + entity.setEntityId("test id"); + entity.setEntityType("test type"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("apptimeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + ATSPutErrors errors = response.getEntity(ATSPutErrors.class); + Assert.assertNotNull(errors); + Assert.assertEquals(0, errors.getErrors().size()); + // verify the entity exists in the store + response = r.path("ws").path("v1").path("apptimeline") + .path("test type").path("test id") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(ATSEntity.class); + Assert.assertNotNull(entity); + Assert.assertEquals("test id", entity.getEntityId()); + Assert.assertEquals("test type", entity.getEntityType()); + } + +} From e6eccf20defcaf84a9566482371c7be196779c0d Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Wed, 5 Feb 2014 18:09:07 +0000 Subject: [PATCH 24/42] YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564856 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/SchedulerApplication.java | 6 +- .../fair/AllocationConfiguration.java | 3 +- .../scheduler/fair/FairScheduler.java | 119 ++++++++++++++- .../fair/MaxRunningAppsEnforcer.java | 51 ++++--- .../scheduler/fair/TestFairScheduler.java | 137 ++++++++++++++++++ .../fair/TestMaxRunningAppsEnforcer.java | 3 +- .../src/site/apt/FairScheduler.apt.vm | 34 ++++- 8 files changed, 326 insertions(+), 30 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b2f70e41afd..0cdfc6fa567 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -14,6 +14,9 @@ Trunk - Unreleased YARN-1504. RM changes for moving apps between queues (Sandy Ryza) + YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy + Ryza) + IMPROVEMENTS OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 1c4a5a638c5..4d6ca0eedbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @Unstable public class SchedulerApplication { - private final Queue queue; + private Queue queue; private final String user; private SchedulerApplicationAttempt currentAttempt; @@ -37,6 +37,10 @@ public class SchedulerApplication { public Queue getQueue() { return queue; } + + public void setQueue(Queue queue) { + this.queue = queue; + } public String getUser() { return user; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 5a33dcb1829..6fc90f472c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -39,7 +39,8 @@ public class AllocationConfiguration { // Minimum resource allocation for each queue private final Map minQueueResources; // Maximum amount of resources per queue - private final Map maxQueueResources; + @VisibleForTesting + final Map maxQueueResources; // Sharing weights for each queue private final Map queueWeights; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b88ad503dd1..e057e740fb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -766,7 +766,9 @@ public class FairScheduler extends AbstractYarnScheduler { boolean wasRunnable = queue.removeApp(attempt); if (wasRunnable) { - maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt); + maxRunningEnforcer.untrackRunnableApp(attempt); + maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, + attempt.getQueue()); } else { maxRunningEnforcer.untrackNonRunnableApp(attempt); } @@ -1355,4 +1357,119 @@ public class FairScheduler extends AbstractYarnScheduler { queue.collectSchedulerApplications(apps); return apps; } + + @Override + public synchronized String moveApplication(ApplicationId appId, + String queueName) throws YarnException { + SchedulerApplication app = applications.get(appId); + if (app == null) { + throw new YarnException("App to be moved " + appId + " not found."); + } + FSSchedulerApp attempt = (FSSchedulerApp) app.getCurrentAppAttempt(); + + FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); + FSLeafQueue targetQueue = queueMgr.getLeafQueue(queueName, false); + if (targetQueue == null) { + throw new YarnException("Target queue " + queueName + + " not found or is not a leaf queue."); + } + if (targetQueue == oldQueue) { + return oldQueue.getQueueName(); + } + + if (oldQueue.getRunnableAppSchedulables().contains( + attempt.getAppSchedulable())) { + verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); + } + + executeMove(app, attempt, oldQueue, targetQueue); + return targetQueue.getQueueName(); + } + + private void verifyMoveDoesNotViolateConstraints(FSSchedulerApp app, + FSLeafQueue oldQueue, FSLeafQueue targetQueue) throws YarnException { + String queueName = targetQueue.getQueueName(); + ApplicationAttemptId appAttId = app.getApplicationAttemptId(); + // When checking maxResources and maxRunningApps, only need to consider + // queues before the lowest common ancestor of the two queues because the + // total running apps in queues above will not be changed. + FSQueue lowestCommonAncestor = findLowestCommonAncestorQueue(oldQueue, + targetQueue); + Resource consumption = app.getCurrentConsumption(); + + // Check whether the move would go over maxRunningApps or maxShare + FSQueue cur = targetQueue; + while (cur != lowestCommonAncestor) { + // maxRunningApps + if (cur.getNumRunnableApps() == allocConf.getQueueMaxApps(cur.getQueueName())) { + throw new YarnException("Moving app attempt " + appAttId + " to queue " + + queueName + " would violate queue maxRunningApps constraints on" + + " queue " + cur.getQueueName()); + } + + // maxShare + if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption), + cur.getMaxShare())) { + throw new YarnException("Moving app attempt " + appAttId + " to queue " + + queueName + " would violate queue maxShare constraints on" + + " queue " + cur.getQueueName()); + } + + cur = cur.getParent(); + } + } + + /** + * Helper for moveApplication, which is synchronized, so all operations will + * be atomic. + */ + private void executeMove(SchedulerApplication app, FSSchedulerApp attempt, + FSLeafQueue oldQueue, FSLeafQueue newQueue) { + boolean wasRunnable = oldQueue.removeApp(attempt); + // if app was not runnable before, it may be runnable now + boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue, + attempt.getUser()); + if (wasRunnable && !nowRunnable) { + throw new IllegalStateException("Should have already verified that app " + + attempt.getApplicationId() + " would be runnable in new queue"); + } + + if (wasRunnable) { + maxRunningEnforcer.untrackRunnableApp(attempt); + } else if (nowRunnable) { + // App has changed from non-runnable to runnable + maxRunningEnforcer.untrackNonRunnableApp(attempt); + } + + attempt.move(newQueue); // This updates all the metrics + app.setQueue(newQueue); + newQueue.addApp(attempt, nowRunnable); + + if (nowRunnable) { + maxRunningEnforcer.trackRunnableApp(attempt); + } + if (wasRunnable) { + maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, oldQueue); + } + } + + private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { + // Because queue names include ancestors, separated by periods, we can find + // the lowest common ancestors by going from the start of the names until + // there's a character that doesn't match. + String name1 = queue1.getName(); + String name2 = queue2.getName(); + // We keep track of the last period we encounter to avoid returning root.apple + // when the queues are root.applepie and root.appletart + int lastPeriodIndex = -1; + for (int i = 0; i < Math.max(name1.length(), name2.length()); i++) { + if (name1.length() <= i || name2.length() <= i || + name1.charAt(i) != name2.charAt(i)) { + return queueMgr.getQueue(name1.substring(lastPeriodIndex)); + } else if (name1.charAt(i) == '.') { + lastPeriodIndex = i; + } + } + return queue1; // names are identical + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java index 862e44d0b98..359519a2f29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java @@ -105,26 +105,15 @@ public class MaxRunningAppsEnforcer { } /** - * Updates the relevant tracking variables after a runnable app with the given - * queue and user has been removed. Checks to see whether any other applications - * are now runnable and makes them so. + * Checks to see whether any other applications runnable now that the given + * application has been removed from the given queue. And makes them so. * * Runs in O(n log(n)) where n is the number of queues that are under the * highest queue that went from having no slack to having slack. */ - public void updateRunnabilityOnAppRemoval(FSSchedulerApp app) { + public void updateRunnabilityOnAppRemoval(FSSchedulerApp app, FSLeafQueue queue) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); - // Update usersRunnableApps - String user = app.getUser(); - int newUserNumRunning = usersNumRunnableApps.get(user) - 1; - if (newUserNumRunning == 0) { - usersNumRunnableApps.remove(user); - } else { - usersNumRunnableApps.put(user, newUserNumRunning); - } - - // Update runnable app bookkeeping for queues: // childqueueX might have no pending apps itself, but if a queue higher up // in the hierarchy parentqueueY has a maxRunningApps set, an app completion // in childqueueX could allow an app in some other distant child of @@ -133,16 +122,14 @@ public class MaxRunningAppsEnforcer { // the queue was already at its max before the removal. // Thus we find the ancestor queue highest in the tree for which the app // that was at its maxRunningApps before the removal. - FSLeafQueue queue = app.getQueue(); FSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() == allocConf.getQueueMaxApps(queue.getName()) - 1) ? queue : null; FSParentQueue parent = queue.getParent(); while (parent != null) { if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent - .getName())) { + .getName()) - 1) { highestQueueWithAppsNowRunnable = parent; } - parent.decrementRunnableApps(); parent = parent.getParent(); } @@ -157,7 +144,12 @@ public class MaxRunningAppsEnforcer { gatherPossiblyRunnableAppLists(highestQueueWithAppsNowRunnable, appsNowMaybeRunnable); } - if (newUserNumRunning == allocConf.getUserMaxApps(user) - 1) { + String user = app.getUser(); + Integer userNumRunning = usersNumRunnableApps.get(user); + if (userNumRunning == null) { + userNumRunning = 0; + } + if (userNumRunning == allocConf.getUserMaxApps(user) - 1) { List userWaitingApps = usersNonRunnableApps.get(user); if (userWaitingApps != null) { appsNowMaybeRunnable.add(userWaitingApps); @@ -208,6 +200,29 @@ public class MaxRunningAppsEnforcer { } } + /** + * Updates the relevant tracking variables after a runnable app with the given + * queue and user has been removed. + */ + public void untrackRunnableApp(FSSchedulerApp app) { + // Update usersRunnableApps + String user = app.getUser(); + int newUserNumRunning = usersNumRunnableApps.get(user) - 1; + if (newUserNumRunning == 0) { + usersNumRunnableApps.remove(user); + } else { + usersNumRunnableApps.put(user, newUserNumRunning); + } + + // Update runnable app bookkeeping for queues + FSLeafQueue queue = app.getQueue(); + FSParentQueue parent = queue.getParent(); + while (parent != null) { + parent.decrementRunnableApps(); + parent = parent.getParent(); + } + } + /** * Stops tracking the given non-runnable app */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 892438902de..a84db75c2d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -56,10 +57,12 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -2547,4 +2550,138 @@ public class TestFairScheduler { TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( scheduler.getSchedulerApplications(), scheduler, "default"); } + + @Test + public void testMoveRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + ApplicationId appId = appAttId.getApplicationId(); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage()); + scheduler.update(); + assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand()); + + scheduler.moveApplication(appId, "queue2"); + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertSame(targetQueue, app.getQueue()); + assertFalse(oldQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage()); + assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage()); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + + scheduler.update(); + assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand()); + assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand()); + } + + @Test + public void testMoveNonRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + assertEquals(0, oldQueue.getNumRunnableApps()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(0, targetQueue.getNumRunnableApps()); + assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test + public void testMoveMakesAppRunnable() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertTrue(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertFalse(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertFalse(targetQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxAppsConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2", + Resource.newInstance(1024, 1)); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveToNonexistentQueue() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + scheduler.getQueueManager().getLeafQueue("queue1", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index 51daeec42e3..c1866f01cd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -77,7 +77,8 @@ public class TestMaxRunningAppsEnforcer { private void removeApp(FSSchedulerApp app) { app.getQueue().removeApp(app); - maxAppsEnforcer.updateRunnabilityOnAppRemoval(app); + maxAppsEnforcer.untrackRunnableApp(app); + maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 655de70c1d2..97436fb8e54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -349,16 +349,20 @@ Queue Access Control Lists (ACLs) * {Administration} - The fair scheduler provides support for administration at runtime through two mechanisms: + The fair scheduler provides support for administration at runtime through a few mechanisms: - * It is possible to modify minimum shares, limits, weights, preemption timeouts - and queue scheduling policies at runtime by editing the allocation file. The - scheduler will reload this file 10-15 seconds after it sees that it was - modified. +Modifying configuration at runtime - * Current applications, queues, and fair shares can be examined through the - ResourceManager's web interface, at - http:///cluster/scheduler. + It is possible to modify minimum shares, limits, weights, preemption timeouts + and queue scheduling policies at runtime by editing the allocation file. The + scheduler will reload this file 10-15 seconds after it sees that it was + modified. + +Monitoring through web UI + + Current applications, queues, and fair shares can be examined through the + ResourceManager's web interface, at + http:///cluster/scheduler. The following fields can be seen for each queue on the web interface: @@ -382,3 +386,17 @@ Queue Access Control Lists (ACLs) In addition to the information that the ResourceManager normally displays about each application, the web interface includes the application's fair share. +Moving applications between queues + + The Fair Scheduler supports moving a running application to a different queue. + This can be useful for moving an important application to a higher priority + queue, or for moving an unimportant application to a lower priority queue. + Apps can be moved by running "yarn application -movetoqueue appID -queue + targetQueueName". + + When an application is moved to a queue, its existing allocations become + counted with the new queue's allocations instead of the old for purposes + of determining fairness. An attempt to move an application to a queue will + fail if the addition of the app's resources to that queue would violate the + its maxRunningApps or maxResources constraints. + From dcaaeefeaf8e15998a7471fd463f3783f9cf9e5b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 5 Feb 2014 19:17:20 +0000 Subject: [PATCH 25/42] Addendum patch for HDFS-5709, add missing license header. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564885 13f79535-47bb-0310-9956-ffa450edef68 --- .../namenode/TestNameNodeOptionParsing.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java index d43eda0cefd..5f47cdabc30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; From 69dbf0b22587e4f8fe2651ebd53f873920f29c91 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 5 Feb 2014 19:43:56 +0000 Subject: [PATCH 26/42] HDFS-5876. SecureDataNodeStarter does not pick up configuration in hdfs-site.xml. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564897 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/datanode/DataNode.java | 11 ++++++----- .../hdfs/server/datanode/SecureDataNodeStarter.java | 5 ++++- .../hdfs/server/namenode/TestNameNodeHttpServer.java | 1 + 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5b39abfeeaf..d42457d6ea6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -869,6 +869,9 @@ Release 2.3.0 - UNRELEASED HDFS-5399. Revisit SafeModeException and corresponding retry policies. (Jing Zhao via todd) + HDFS-5876. SecureDataNodeStarter does not pick up configuration in + hdfs-site.xml. (Haohui Mai via jing9) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 1dabd4a94c6..ad580a53d1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -362,13 +362,13 @@ public class DataNode extends Configured .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))); HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); - InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); - String infoHost = infoSocAddr.getHostName(); if (policy.isHttpEnabled()) { if (secureResources == null) { + InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf); int port = infoSocAddr.getPort(); - builder.addEndpoint(URI.create("http://" + infoHost + ":" + port)); + builder.addEndpoint(URI.create("http://" + + NetUtils.getHostPortString(infoSocAddr))); if (port == 0) { builder.setFindPort(true); } @@ -381,7 +381,7 @@ public class DataNode extends Configured if (policy.isHttpsEnabled()) { InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get( - DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 0)); + DFS_DATANODE_HTTPS_ADDRESS_KEY, DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)); Configuration sslConf = DFSUtil.loadSslConfiguration(conf); DFSUtil.loadSslConfToHttpServerBuilder(builder, sslConf); @@ -390,7 +390,8 @@ public class DataNode extends Configured if (port == 0) { builder.setFindPort(true); } - builder.addEndpoint(URI.create("https://" + infoHost + ":" + port)); + builder.addEndpoint(URI.create("https://" + + NetUtils.getHostPortString(secInfoSocAddr))); } this.infoServer = builder.build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java index 000f7aedd2d..477b7f66558 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java @@ -25,6 +25,7 @@ import org.apache.commons.daemon.DaemonContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer2; @@ -62,7 +63,9 @@ public class SecureDataNodeStarter implements Daemon { @Override public void init(DaemonContext context) throws Exception { System.err.println("Initializing secure datanode resources"); - Configuration conf = new Configuration(); + // Create a new HdfsConfiguration object to ensure that the configuration in + // hdfs-site.xml is picked up. + Configuration conf = new HdfsConfiguration(); // Stash command-line arguments for regular datanode args = context.getArguments(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java index be32b5b2728..39983b10ed2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -85,6 +85,7 @@ public class TestNameNodeHttpServer { @Test public void testHttpPolicy() throws Exception { conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); + conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); InetSocketAddress addr = InetSocketAddress.createUnresolved("localhost", 0); NameNodeHttpServer server = null; From d598b6ef9f10ae011fecbe198360cde63a4e4d50 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 5 Feb 2014 22:48:06 +0000 Subject: [PATCH 27/42] HDFS-5873. dfs.http.policy should have higher precedence over dfs.https.enable. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564973 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/http/HttpConfig.java | 11 ++-- .../src/site/apt/SecureMode.apt.vm | 3 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 39 +++++++------- .../apache/hadoop/hdfs/TestHttpPolicy.java | 54 +++++++++++++++++++ .../namenode/TestNameNodeHttpServer.java | 4 +- 6 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index c533fedf65e..d323f764359 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -34,13 +34,14 @@ public class HttpConfig { HTTPS_ONLY, HTTP_AND_HTTPS; + private static final Policy[] VALUES = values(); public static Policy fromString(String value) { - if (HTTPS_ONLY.name().equalsIgnoreCase(value)) { - return HTTPS_ONLY; - } else if (HTTP_AND_HTTPS.name().equalsIgnoreCase(value)) { - return HTTP_AND_HTTPS; + for (Policy p : VALUES) { + if (p.name().equalsIgnoreCase(value)) { + return p; + } } - return HTTP_ONLY; + return null; } public boolean isHttpEnabled() { diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm index 9bd55a67fff..68ca4b0da68 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm @@ -352,7 +352,8 @@ Configuration for <<>> | | | This value is deprecated. Use dfs.http.policy | *-------------------------+-------------------------+------------------------+ | <<>> | or or | | -| | | HTTPS_ONLY turns off http access | +| | | HTTPS_ONLY turns off http access. This option takes precedence over | +| | | the deprecated configuration dfs.https.enable and hadoop.ssl.enabled. | *-------------------------+-------------------------+------------------------+ | <<>> | | | *-------------------------+-------------------------+------------------------+ diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d42457d6ea6..0ad51b75692 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -872,6 +872,9 @@ Release 2.3.0 - UNRELEASED HDFS-5876. SecureDataNodeStarter does not pick up configuration in hdfs-site.xml. (Haohui Mai via jing9) + HDFS-5873. dfs.http.policy should have higher precedence over dfs.https.enable. + (Haohui Mai via jing9) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS HDFS-4985. Add storage type to the protocol and expose it in block report diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 9274f505a4a..8dccff0d32d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -1567,31 +1567,34 @@ public class DFSUtil { * configuration settings. */ public static HttpConfig.Policy getHttpPolicy(Configuration conf) { - String httpPolicy = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY, - DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); - - HttpConfig.Policy policy = HttpConfig.Policy.fromString(httpPolicy); - - if (policy == HttpConfig.Policy.HTTP_ONLY) { - boolean httpsEnabled = conf.getBoolean( - DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, + String policyStr = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY); + if (policyStr == null) { + boolean https = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, DFSConfigKeys.DFS_HTTPS_ENABLE_DEFAULT); - boolean hadoopSslEnabled = conf.getBoolean( + boolean hadoopSsl = conf.getBoolean( CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY, CommonConfigurationKeys.HADOOP_SSL_ENABLED_DEFAULT); - if (hadoopSslEnabled) { + if (hadoopSsl) { LOG.warn(CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY - + " is deprecated. Please use " - + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + "."); - policy = HttpConfig.Policy.HTTPS_ONLY; - } else if (httpsEnabled) { - LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY - + " is deprecated. Please use " - + DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + "."); - policy = HttpConfig.Policy.HTTP_AND_HTTPS; + + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY + + "."); } + if (https) { + LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY + + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY + + "."); + } + + return (hadoopSsl || https) ? HttpConfig.Policy.HTTP_AND_HTTPS + : HttpConfig.Policy.HTTP_ONLY; + } + + HttpConfig.Policy policy = HttpConfig.Policy.fromString(policyStr); + if (policy == null) { + throw new HadoopIllegalArgumentException("Unregonized value '" + + policyStr + "' for " + DFSConfigKeys.DFS_HTTP_POLICY_KEY); } conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java new file mode 100644 index 00000000000..e448e151cb6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHttpPolicy.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import static org.apache.hadoop.http.HttpConfig.Policy.HTTP_AND_HTTPS; +import static org.apache.hadoop.http.HttpConfig.Policy.HTTP_ONLY; + +import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +public final class TestHttpPolicy { + + @Test(expected = HadoopIllegalArgumentException.class) + public void testInvalidPolicyValue() { + Configuration conf = new Configuration(); + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, "invalid"); + DFSUtil.getHttpPolicy(conf); + } + + @Test + public void testDeprecatedConfiguration() { + Configuration conf = new Configuration(false); + Assert.assertSame(HTTP_ONLY, DFSUtil.getHttpPolicy(conf)); + + conf.setBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, true); + Assert.assertSame(HTTP_AND_HTTPS, DFSUtil.getHttpPolicy(conf)); + + conf = new Configuration(false); + conf.setBoolean(DFSConfigKeys.HADOOP_SSL_ENABLED_KEY, true); + Assert.assertSame(HTTP_AND_HTTPS, DFSUtil.getHttpPolicy(conf)); + + conf = new Configuration(false); + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HTTP_ONLY.name()); + conf.setBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, true); + Assert.assertSame(HTTP_ONLY, DFSUtil.getHttpPolicy(conf)); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java index 39983b10ed2..975a6649e60 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -104,7 +104,9 @@ public class TestNameNodeHttpServer { server.getHttpsAddress() == null)); } finally { - server.stop(); + if (server != null) { + server.stop(); + } } } From 996e25c3d11228b1c30a811a5df2b3024fa4640b Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 6 Feb 2014 00:13:34 +0000 Subject: [PATCH 28/42] HADOOP-10325. Improve Jenkins Javadoc warnings from test-patch.sh (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565010 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 49 +++++++++++++------ .../hadoop-common/CHANGES.txt | 2 + 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 7143b514060..ed671a64ebb 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -300,6 +300,17 @@ prebuildWithoutPatch () { {color:red}-1 patch{color}. Trunk compilation may be broken." return 1 fi + + echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1" + $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "Trunk javadoc compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. Trunk compilation may be broken." + return 1 + fi + return 0 } @@ -401,6 +412,11 @@ applyPatch () { } ############################################################################### +calculateJavadocWarnings() { + WARNING_FILE="$1" + RET=$(egrep "^[0-9]+ warnings$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') +} + ### Check there are no javadoc warnings checkJavadocWarnings () { echo "" @@ -420,24 +436,29 @@ checkJavadocWarnings () { (cd hadoop-common-project/hadoop-annotations; $MVN install > /dev/null 2>&1) fi $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - echo "" - echo "" - echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." + calculateJavadocWarnings "$PATCH_DIR/trunkJavadocWarnings.txt" + numTrunkJavadocWarnings=$RET + calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" + numPatchJavadocWarnings=$RET + grep -i warning "$PATCH_DIR/trunkJavadocWarnings.txt" > "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" + grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + diff -u "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" \ + "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ + "$PATCH_DIR/diffJavadocWarnings.txt" + rm -f "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + echo "There appear to be $numTrunkJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." + if [[ $numTrunkJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then + if [[ $numPatchJavadocWarnings -gt $numTrunkJavadocWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT - #There are 12 warnings that are caused by things that are caused by using sun internal APIs. - #There are 2 warnings that are caused by the Apache DS Dn class used in MiniKdc. - OK_JAVADOC_WARNINGS=14; - ### if current warnings greater than OK_JAVADOC_WARNINGS - if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages." - return 1 + {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$numTrunkJavadocWarnings))` warning messages. + See $BUILD_URL/artifact/trunk/patchprocess/diffJavadocWarnings.txt for details." + return 1 + fi fi JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 javadoc{color}. The javadoc tool did not generate any warning messages." + {color:green}+1 javadoc{color}. There were no new javadoc warning messages." return 0 } diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e3a39daa581..31e673157eb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -116,6 +116,8 @@ Trunk (Unreleased) HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry McCay via omalley) + HADOOP-10325. Improve jenkins javadoc warnings from test-patch.sh (cmccabe) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. From b36cc29073a22ec3bb14f1f7b34dc5518fd8c6a2 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 6 Feb 2014 06:20:41 +0000 Subject: [PATCH 29/42] MAPREDUCE-5743. Fixed the test failure in TestRMContainerAllocator. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565087 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/v2/app/TestRMContainerAllocator.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fae27e4929a..23171ccc1bd 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -347,6 +347,9 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5723. MR AM container log can be truncated or empty. (Mohammad Kamrul Islam via kasha) + MAPREDUCE-5743. Fixed the test failure in TestRMContainerAllocator. + (Vinod Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 3eb5222865c..c62fa39c2ea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1652,8 +1652,16 @@ public class TestRMContainerAllocator { RMApp app = rm.submitApp(1024); dispatcher.await(); + // Make a node to register so as to launch the AM. + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + amNodeManager.nodeHeartbeat(true); + dispatcher.await(); + ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); + rm.sendAMLaunched(appAttemptId); + dispatcher.await(); + JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job job = mock(Job.class); when(job.getReport()).thenReturn( From 24775c10efbff28a323ef9128120f0cbf06d17a6 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 6 Feb 2014 07:21:42 +0000 Subject: [PATCH 30/42] YARN-1628. Fixed the test failure in TestContainerManagerSecurity. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565094 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/test/resources/core-site.xml | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0cdfc6fa567..7f7b795c0eb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -627,6 +627,9 @@ Release 2.3.0 - UNRELEASED YARN-1629. IndexOutOfBoundsException in MaxRunningAppsEnforcer (Sandy Ryza) + YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml new file mode 100644 index 00000000000..f0d3085ef85 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + + + hadoop.security.token.service.use_ip + false + + + From ab96a0838dafbfea77382135914feadbfd03cf53 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 6 Feb 2014 15:45:47 +0000 Subject: [PATCH 31/42] HDFS-5881. Fix skip() of the short-circuit local reader(legacy). Contributed by Kihwal Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565310 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java | 6 +++--- .../main/java/org/apache/hadoop/hdfs/DFSInputStream.java | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0ad51b75692..2720c7ddd1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -337,6 +337,8 @@ Release 2.4.0 - UNRELEASED HDFS-5709. Improve NameNode upgrade with existing reserved paths and path components. (Andrew Wang via atm) + HDFS-5881. Fix skip() of the short-circuit local reader(legacy). (kihwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java index 2f661933619..ffc4eb9f8ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java @@ -629,7 +629,7 @@ class BlockReaderLocalLegacy implements BlockReader { skipBuf = new byte[bytesPerChecksum]; } int ret = read(skipBuf, 0, (int)(n - remaining)); - return ret; + return (remaining + ret); } // optimize for big gap: discard the current buffer, skip to @@ -660,9 +660,9 @@ class BlockReaderLocalLegacy implements BlockReader { int ret = read(skipBuf, 0, myOffsetFromChunkBoundary); if (ret == -1) { // EOS - return toskip; + return (toskip + remaining); } else { - return (toskip + ret); + return (toskip + remaining + ret); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 73861bc8ade..438030eaa97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -1345,6 +1345,14 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, pos += blockReader.skip(diff); if (pos == targetPos) { done = true; + } else { + // The range was already checked. If the block reader returns + // something unexpected instead of throwing an exception, it is + // most likely a bug. + String errMsg = "BlockReader failed to seek to " + + targetPos + ". Instead, it seeked to " + pos + "."; + DFSClient.LOG.warn(errMsg); + throw new IOException(errMsg); } } catch (IOException e) {//make following read to retry if(DFSClient.LOG.isDebugEnabled()) { From 4594b74b8536aea308854493a0e82c8b2919174b Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 6 Feb 2014 18:29:30 +0000 Subject: [PATCH 32/42] MAPREDUCE-5699. Allow setting tags on MR jobs (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565384 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 ++ .../src/main/resources/mapred-default.xml | 10 +++++++++- .../main/java/org/apache/hadoop/mapred/YARNRunner.java | 7 +++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 23171ccc1bd..d3eac8d94b6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -153,6 +153,8 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5732. Report proper queue when job has been automatically placed (Sandy Ryza) + MAPREDUCE-5699. Allow setting tags on MR jobs (kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index f1a3e453fb3..ffb95433513 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -60,6 +60,8 @@ public interface MRJobConfig { public static final String QUEUE_NAME = "mapreduce.job.queuename"; + public static final String JOB_TAGS = "mapreduce.job.tags"; + public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; public static final String SPLIT_FILE = "mapreduce.job.splitfile"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index dea2adf1350..cc663923eaf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1,4 +1,5 @@ + - @@ -727,6 +727,14 @@ + + mapreduce.job.tags + + Tags for the job that will be passed to YARN at submission + time. Queries to YARN for applications can filter on these tags. + + + mapreduce.cluster.local.dir ${hadoop.tmp.dir}/mapred/local diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index fc23c6541f9..ce475c197c1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -21,7 +21,9 @@ package org.apache.hadoop.mapred; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Vector; @@ -467,6 +469,8 @@ public class YARNRunner implements ClientProtocol { ContainerLaunchContext.newInstance(localResources, environment, vargsFinal, null, securityTokens, acls); + Collection tagsFromConf = + jobConf.getTrimmedStringCollection(MRJobConfig.JOB_TAGS); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = @@ -486,6 +490,9 @@ public class YARNRunner implements ClientProtocol { MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); appContext.setResource(capability); appContext.setApplicationType(MRJobConfig.MR_APPLICATION_TYPE); + if (tagsFromConf != null && !tagsFromConf.isEmpty()) { + appContext.setApplicationTags(new HashSet(tagsFromConf)); + } return appContext; } From c4b0ce0ace8381be299a7fa013e17bfe3294e0bf Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 6 Feb 2014 18:40:23 +0000 Subject: [PATCH 33/42] HADOOP-10327. Trunk windows build broken after HDFS-5746. Contributed by Vinay. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565389 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/io/nativeio/NativeIO.c | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 31e673157eb..2698cc2a787 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -324,6 +324,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) + HADOOP-10327. Trunk windows build broken after HDFS-5746. + (Vinay via cnauroth) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 2b5c0a4ab0a..79c9b9d4697 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -671,6 +671,7 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mmap( JNIEnv *env, jclass clazz, jobject jfd, jint jprot, jboolean jshared, jlong length) { +#ifdef UNIX void *addr = 0; int prot, flags, fd; @@ -684,18 +685,33 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mmap( throw_ioe(env, errno); } return (jlong)(intptr_t)addr; +#endif // UNIX + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.mmap() is not supported on Windows"); + return NULL; +#endif } JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_munmap( JNIEnv *env, jclass clazz, jlong jaddr, jlong length) { +#ifdef UNIX void *addr; addr = (void*)(intptr_t)jaddr; if (munmap(addr, length) < 0) { throw_ioe(env, errno); } +#endif // UNIX + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.munmap() is not supported on Windows"); + return NULL; +#endif } @@ -1050,4 +1066,3 @@ JNIEnv *env, jclass clazz) /** * vim: sw=2: ts=2: et: */ - From 6182e7592d7255792ae6a04a5551296d755f6a37 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 6 Feb 2014 18:56:46 +0000 Subject: [PATCH 34/42] YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565394 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../distributedshell/ApplicationMaster.java | 15 ++++++++------- .../ContainerLaunchFailAppMaster.java | 10 +++++----- .../distributedshell/TestDSFailedAppMaster.java | 13 +++++++------ 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7f7b795c0eb..31103413074 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -630,6 +630,9 @@ Release 2.3.0 - UNRELEASED YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod Kumar Vavilapalli via zjshen) + YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 9b6e788c3cb..8d869a225cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -232,7 +232,6 @@ public class ApplicationMaster { private static final String shellArgsPath = "shellArgs"; private volatile boolean done; - private volatile boolean success; private ByteBuffer allTokens; @@ -254,8 +253,8 @@ public class ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); - appMaster.finish(); + appMaster.run(); + result = appMaster.finish(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); @@ -480,7 +479,7 @@ public class ApplicationMaster { * @throws IOException */ @SuppressWarnings({ "unchecked" }) - public boolean run() throws YarnException, IOException { + public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); Credentials credentials = @@ -561,7 +560,6 @@ public class ApplicationMaster { amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainersToRequest); - return success; } @VisibleForTesting @@ -569,7 +567,8 @@ public class ApplicationMaster { return new NMCallbackHandler(this); } - protected void finish() { + @VisibleForTesting + protected boolean finish() { // wait for completion. while (!done && (numCompletedContainers.get() != numTotalContainers)) { @@ -600,7 +599,7 @@ public class ApplicationMaster { FinalApplicationStatus appStatus; String appMessage = null; - success = true; + boolean success = true; if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) { appStatus = FinalApplicationStatus.SUCCEEDED; @@ -621,6 +620,8 @@ public class ApplicationMaster { } amRMClient.stop(); + + return success; } private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java index e845490e35b..8e561c67362 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java @@ -18,13 +18,13 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.nio.ByteBuffer; +import java.util.Map; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ContainerId; -import java.nio.ByteBuffer; -import java.util.Map; - public class ContainerLaunchFailAppMaster extends ApplicationMaster { private static final Log LOG = @@ -66,8 +66,8 @@ public class ContainerLaunchFailAppMaster extends ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); - appMaster.finish(); + appMaster.run(); + result = appMaster.finish(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java index 644f66799b2..db7419bc8e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java @@ -29,8 +29,8 @@ public class TestDSFailedAppMaster extends ApplicationMaster { private static final Log LOG = LogFactory.getLog(TestDSFailedAppMaster.class); @Override - public boolean run() throws YarnException, IOException { - boolean res = super.run(); + public void run() throws YarnException, IOException { + super.run(); // for the 2nd attempt. if (appAttemptID.getAttemptId() == 2) { @@ -39,11 +39,12 @@ public class TestDSFailedAppMaster extends ApplicationMaster { // numRequestedContainers should be set to 0. if (numAllocatedContainers.get() != 1 || numRequestedContainers.get() != 0) { - LOG.info("Application Master failed. exiting"); + LOG.info("NumAllocatedContainers is " + numAllocatedContainers.get() + + " and NumRequestedContainers is " + numAllocatedContainers.get() + + ".Application Master failed. exiting"); System.exit(200); } } - return res; } public static void main(String[] args) { @@ -54,7 +55,7 @@ public class TestDSFailedAppMaster extends ApplicationMaster { if (!doRun) { System.exit(0); } - result = appMaster.run(); + appMaster.run(); if (appMaster.appAttemptID.getAttemptId() == 1) { try { // sleep some time, wait for the AM to launch a container. @@ -63,7 +64,7 @@ public class TestDSFailedAppMaster extends ApplicationMaster { // fail the first am. System.exit(100); } - appMaster.finish(); + result = appMaster.finish(); } catch (Throwable t) { System.exit(1); } From b527a975a4899e2b04c94eba46879625de1111e6 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 6 Feb 2014 19:07:18 +0000 Subject: [PATCH 35/42] MAPREDUCE-5743. Fixed CHANGES.txt to give credit to Ted too. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565401 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d3eac8d94b6..33db5bf461f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -350,7 +350,7 @@ Release 2.3.0 - UNRELEASED (Mohammad Kamrul Islam via kasha) MAPREDUCE-5743. Fixed the test failure in TestRMContainerAllocator. - (Vinod Kumar Vavilapalli via zjshen) + (Ted Yu and Vinod Kumar Vavilapalli via zjshen) Release 2.2.0 - 2013-10-13 From e0cda4895948c500a7bbc0a1a553d3698be3e317 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 6 Feb 2014 21:08:09 +0000 Subject: [PATCH 36/42] HDFS-4911. Reduce PeerCache timeout to be commensurate with dfs.datanode.socket.reuse.keepalive (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565435 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +- .../org/apache/hadoop/hdfs/PeerCache.java | 20 ++- .../hdfs/TestDataTransferKeepalive.java | 126 ++++++++++++------ 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2720c7ddd1f..b0cc1fdebae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -308,6 +308,9 @@ Release 2.4.0 - UNRELEASED HDFS-5746. Add ShortCircuitSharedMemorySegment (cmccabe) + HDFS-4911. Reduce PeerCache timeout to be commensurate with + dfs.datanode.socket.reuse.keepalive (cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index fe1d3d1570e..02297a64fd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -86,7 +86,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT = 10; public static final String DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY = "dfs.client.socketcache.expiryMsec"; - public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 2 * 60 * 1000; + public static final long DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT = 3000; public static final String DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL = "dfs.client.write.exclude.nodes.cache.expiry.interval.millis"; public static final long DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT = 10 * 60 * 1000; // 10 minutes, in ms public static final String DFS_NAMENODE_BACKUP_ADDRESS_KEY = "dfs.namenode.backup.address"; @@ -215,7 +215,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose"; public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false; public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive"; - public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 1000; + public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 4000; public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check"; public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java index 424b641c8c3..ba6736a9db4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/PeerCache.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; @@ -25,6 +26,7 @@ import java.util.Map.Entry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.LinkedListMultimap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -118,6 +120,11 @@ class PeerCache { return instance; } + @VisibleForTesting + public static synchronized void setInstance(int c, long e) { + instance = new PeerCache(c, e); + } + private boolean isDaemonStarted() { return (daemon == null)? false: true; } @@ -171,8 +178,17 @@ class PeerCache { while (iter.hasNext()) { Value candidate = iter.next(); iter.remove(); - if (!candidate.getPeer().isClosed()) { - return candidate.getPeer(); + long ageMs = Time.monotonicNow() - candidate.getTime(); + Peer peer = candidate.getPeer(); + if (ageMs >= expiryPeriod) { + try { + peer.close(); + } catch (IOException e) { + LOG.warn("got IOException closing stale peer " + peer + + ", which is " + ageMs + " ms old"); + } + } else if (!peer.isClosed()) { + return peer; } } return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java index bf4e13bd027..bdfc62d5fde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java @@ -19,16 +19,19 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.common.base.Supplier; + import java.io.InputStream; import java.io.PrintWriter; -import java.net.InetSocketAddress; -import java.net.Socket; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -37,10 +40,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ReflectionUtils; import org.junit.After; import org.junit.Before; @@ -51,10 +52,7 @@ import com.google.common.io.NullOutputStream; public class TestDataTransferKeepalive { Configuration conf = new HdfsConfiguration(); private MiniDFSCluster cluster; - private FileSystem fs; - private InetSocketAddress dnAddr; private DataNode dn; - private DFSClient dfsClient; private static Path TEST_FILE = new Path("/test"); private static final int KEEPALIVE_TIMEOUT = 1000; @@ -69,15 +67,7 @@ public class TestDataTransferKeepalive { cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1).build(); - fs = cluster.getFileSystem(); - dfsClient = ((DistributedFileSystem)fs).dfs; - dfsClient.peerCache.clear(); - - String poolId = cluster.getNamesystem().getBlockPoolId(); dn = cluster.getDataNodes().get(0); - DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP( - dn, poolId); - dnAddr = NetUtils.createSocketAddr(dnReg.getXferAddr()); } @After @@ -90,34 +80,86 @@ public class TestDataTransferKeepalive { * its configured keepalive timeout. */ @Test(timeout=30000) - public void testKeepaliveTimeouts() throws Exception { + public void testDatanodeRespectsKeepAliveTimeout() throws Exception { + Configuration clientConf = new Configuration(conf); + // Set a client socket cache expiry time much longer than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 60000L; + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Clients that write aren't currently re-used. - assertEquals(0, dfsClient.peerCache.size()); + assertEquals(0, fs.dfs.peerCache.size()); assertXceiverCount(0); // Reads the file, so we should get a // cached socket, and should have an xceiver on the other side. DFSTestUtil.readFile(fs, TEST_FILE); - assertEquals(1, dfsClient.peerCache.size()); + assertEquals(1, fs.dfs.peerCache.size()); assertXceiverCount(1); // Sleep for a bit longer than the keepalive timeout // and make sure the xceiver died. - Thread.sleep(KEEPALIVE_TIMEOUT * 2); + Thread.sleep(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT + 1); assertXceiverCount(0); // The socket is still in the cache, because we don't // notice that it's closed until we try to read // from it again. - assertEquals(1, dfsClient.peerCache.size()); + assertEquals(1, fs.dfs.peerCache.size()); // Take it out of the cache - reading should // give an EOF. - Peer peer = dfsClient.peerCache.get(dn.getDatanodeId(), false); + Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); assertNotNull(peer); assertEquals(-1, peer.getInputStream().read()); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, + DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); + } + + /** + * Test that the client respects its keepalive timeout. + */ + @Test(timeout=30000) + public void testClientResponsesKeepAliveTimeout() throws Exception { + Configuration clientConf = new Configuration(conf); + // Set a client socket cache expiry time much shorter than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 10L; + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); + + DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); + + // Clients that write aren't currently re-used. + assertEquals(0, fs.dfs.peerCache.size()); + assertXceiverCount(0); + + // Reads the file, so we should get a + // cached socket, and should have an xceiver on the other side. + DFSTestUtil.readFile(fs, TEST_FILE); + assertEquals(1, fs.dfs.peerCache.size()); + assertXceiverCount(1); + + // Sleep for a bit longer than the client keepalive timeout. + Thread.sleep(CLIENT_EXPIRY_MS + 1); + + // Taking out a peer which is expired should give a null. + Peer peer = fs.dfs.peerCache.get(dn.getDatanodeId(), false); + assertTrue(peer == null); + + // The socket cache is now empty. + assertEquals(0, fs.dfs.peerCache.size()); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, + DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); } /** @@ -125,8 +167,17 @@ public class TestDataTransferKeepalive { * read bytes off the stream quickly. The datanode should time out sending the * chunks and the transceiver should die, even if it has a long keepalive. */ - @Test(timeout=30000) + @Test(timeout=300000) public void testSlowReader() throws Exception { + // Set a client socket cache expiry time much longer than + // the datanode-side expiration time. + final long CLIENT_EXPIRY_MS = 600000L; + Configuration clientConf = new Configuration(conf); + clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, CLIENT_EXPIRY_MS); + PeerCache.setInstance(DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT, CLIENT_EXPIRY_MS); + DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(), + clientConf); // Restart the DN with a shorter write timeout. DataNodeProperties props = cluster.stopDataNode(0); props.conf.setInt(DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY, @@ -134,38 +185,31 @@ public class TestDataTransferKeepalive { props.conf.setInt(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY, 120000); assertTrue(cluster.restartDataNode(props, true)); + dn = cluster.getDataNodes().get(0); // Wait for heartbeats to avoid a startup race where we // try to write the block while the DN is still starting. cluster.triggerHeartbeats(); - dn = cluster.getDataNodes().get(0); - DFSTestUtil.createFile(fs, TEST_FILE, 1024*1024*8L, (short)1, 0L); FSDataInputStream stm = fs.open(TEST_FILE); - try { - stm.read(); - assertXceiverCount(1); + stm.read(); + assertXceiverCount(1); - // Poll for 0 running xceivers. Allow up to 5 seconds for some slack. - long totalSleepTime = 0; - long sleepTime = WRITE_TIMEOUT + 100; - while (getXceiverCountWithoutServer() > 0 && totalSleepTime < 5000) { - Thread.sleep(sleepTime); - totalSleepTime += sleepTime; - sleepTime = 100; + GenericTestUtils.waitFor(new Supplier() { + public Boolean get() { + // DN should time out in sendChunks, and this should force + // the xceiver to exit. + return getXceiverCountWithoutServer() == 0; } + }, 500, 50000); - // DN should time out in sendChunks, and this should force - // the xceiver to exit. - assertXceiverCount(0); - } finally { - IOUtils.closeStream(stm); - } + IOUtils.closeStream(stm); } @Test(timeout=30000) public void testManyClosedSocketsInCache() throws Exception { // Make a small file + DistributedFileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L); // Insert a bunch of dead sockets in the cache, by opening From 12c2582c1f845b6ce344ec3181628d31739c8ec8 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 6 Feb 2014 21:16:01 +0000 Subject: [PATCH 37/42] HDFS-5895. HDFS cacheadmin -listPools has exit_code of 1 when the command returns 0 result. Contributed by Tassapol Athiapinya. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565440 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b0cc1fdebae..ce457bd51b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -342,6 +342,9 @@ Release 2.4.0 - UNRELEASED HDFS-5881. Fix skip() of the short-circuit local reader(legacy). (kihwal) + HDFS-5895. HDFS cacheadmin -listPools has exit_code of 1 when the command + returns 0 result. (Tassapol Athiapinya via cnauroth) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index b3538daf525..290e60087f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -962,9 +962,8 @@ public class CacheAdmin extends Configured implements Tool { if (numResults > 0) { System.out.print(listing); } - // If there are no results, we return 1 (failure exit code); - // otherwise we return 0 (success exit code). - return (numResults == 0) ? 1 : 0; + // If list pools succeed, we return 0 (success exit code) + return 0; } } From 0001e39cf0a913d2acb88ba177f72bf30d1ca655 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 6 Feb 2014 21:56:00 +0000 Subject: [PATCH 38/42] HADOOP-10273. Update CHANGES.txt to reflect new target version is 2.3 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565453 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2698cc2a787..b5b8e73445c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -322,8 +322,6 @@ Release 2.4.0 - UNRELEASED HADOOP-10085. CompositeService should allow adding services while being inited. (Steve Loughran via kasha) - HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) - HADOOP-10327. Trunk windows build broken after HDFS-5746. (Vinay via cnauroth) @@ -698,6 +696,8 @@ Release 2.3.0 - UNRELEASED HADOOP-10311. Cleanup vendor names from the code base. (tucu) + HADOOP-10273. Fix 'mvn site'. (Arpit Agarwal) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES From 914a9709b91dbbedf251e8c135019741e6ed01b2 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 6 Feb 2014 23:27:57 +0000 Subject: [PATCH 39/42] MAPREDUCE-5744. Job hangs because RMContainerAllocator.preemptReduce() violates the comparator contract (Gera Shegalov via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565478 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 33db5bf461f..3b72f402223 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -230,6 +230,10 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5725. Make explicit that TestNetworkedJob relies on the Capacity Scheduler (Sandy Ryza) + MAPREDUCE-5744. Job hangs because + RMContainerAllocator$AssignedRequests.preemptReduce() violates the + comparator contract (Gera Shegalov via kasha) + OPTIMIZATIONS MAPREDUCE-4680. Job history cleaner should only check timestamps of files in diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index a8ee06b3b48..a0c690c51ea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -1143,9 +1143,9 @@ public class RMContainerAllocator extends RMContainerRequestor new Comparator() { @Override public int compare(TaskAttemptId o1, TaskAttemptId o2) { - float p = getJob().getTask(o1.getTaskId()).getAttempt(o1).getProgress() - - getJob().getTask(o2.getTaskId()).getAttempt(o2).getProgress(); - return p >= 0 ? 1 : -1; + return Float.compare( + getJob().getTask(o1.getTaskId()).getAttempt(o1).getProgress(), + getJob().getTask(o2.getTaskId()).getAttempt(o2).getProgress()); } }); From 943b2190d72f930a76f15558fca0dbb128b2d592 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Fri, 7 Feb 2014 00:18:46 +0000 Subject: [PATCH 40/42] YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565497 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/rmapp/RMAppImpl.java | 6 +- .../yarn/server/resourcemanager/MockRM.java | 7 ++ .../yarn/server/resourcemanager/TestRM.java | 101 +++++++++++++++++- .../applicationsmanager/TestAMRestart.java | 2 +- .../rmapp/TestRMAppTransitions.java | 7 ++ .../webapp/TestRMWebServicesApps.java | 4 +- 7 files changed, 119 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 31103413074..6afd4bdbfed 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -633,6 +633,9 @@ Release 2.3.0 - UNRELEASED YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod Kumar Vavilapalli via zjshen) + YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod + Kumar Vavilapalli via zjshen) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index edbe676badd..196e89d32c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -196,10 +196,8 @@ public class RMAppImpl implements RMApp, Recoverable { // waiting for the previous AM to exit. RMAppEventType.ATTEMPT_FAILED, new AttemptFailedTransition(RMAppState.ACCEPTED)) - .addTransition(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING, - RMAppEventType.KILL, - new FinalSavingTransition( - new AppKilledTransition(), RMAppState.KILLED)) + .addTransition(RMAppState.ACCEPTED, RMAppState.KILLING, + RMAppEventType.KILL, new KillAttemptTransition()) // ACCECPTED state can once again receive APP_ACCEPTED event, because on // recovery the app returns ACCEPTED state and the app once again go // through the scheduler and triggers one more APP_ACCEPTED event at diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 935820e66b3..31035b420a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -482,6 +482,13 @@ public class MockRM extends ResourceManager { RMAppAttempt attempt = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); + rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); + return am; + } + + public static MockAM launchAndRegisterAM(RMApp app, MockRM rm, MockNM nm) + throws Exception { + MockAM am = launchAM(app, rm, nm); am.registerAppAttempt(); rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); return am; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index c7e6d7fb979..b899ea708bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.spy; + import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -33,6 +37,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; @@ -44,9 +49,17 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -54,7 +67,9 @@ import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Test; +import org.mockito.ArgumentMatcher; +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestRM { private static final Log LOG = LogFactory.getLog(TestRM.class); @@ -397,19 +412,19 @@ public class TestRM { MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - MockAM am1 = MockRM.launchAM(app1, rm1, nm1); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishApplicationMaster(app1, rm1, nm1, am1); // a failed app RMApp app2 = rm1.submitApp(200); - MockAM am2 = MockRM.launchAM(app2, rm1, nm1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED); // a killed app RMApp app3 = rm1.submitApp(200); - MockAM am3 = MockRM.launchAM(app3, rm1, nm1); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); rm1.killApp(app3.getApplicationId()); rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED); rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED); @@ -449,7 +464,7 @@ public class TestRM { // a failed app RMApp app2 = rm1.submitApp(200); - MockAM am2 = MockRM.launchAM(app2, rm1, nm1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1 .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); @@ -466,10 +481,88 @@ public class TestRM { Assert.assertEquals(-1, report1.getRpcPort()); } + /** + * Validate killing an application when it is at accepted state. + * @throws Exception exception + */ + @Test (timeout = 60000) + public void testApplicationKillAtAcceptedState() throws Exception { + + YarnConfiguration conf = new YarnConfiguration(); + final Dispatcher dispatcher = new AsyncDispatcher() { + @Override + public EventHandler getEventHandler() { + + class EventArgMatcher extends ArgumentMatcher { + @Override + public boolean matches(Object argument) { + if (argument instanceof RMAppAttemptEvent) { + if (((RMAppAttemptEvent) argument).getType().equals( + RMAppAttemptEventType.KILL)) { + return true; + } + } + return false; + } + } + + EventHandler handler = spy(super.getEventHandler()); + doNothing().when(handler).handle(argThat(new EventArgMatcher())); + return handler; + } + }; + + MockRM rm = new MockRM(conf) { + @Override + protected Dispatcher createDispatcher() { + return dispatcher; + } + }; + + rm.start(); + MockNM nm1 = + new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); + nm1.registerNode(); + + // a failed app + RMApp application = rm.submitApp(200); + MockAM am = MockRM.launchAM(application, rm, nm1); + am.waitForState(RMAppAttemptState.LAUNCHED); + nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); + rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED); + + // Now kill the application before new attempt is launched, the app report + // returns the invalid AM host and port. + KillApplicationRequest request = + KillApplicationRequest.newInstance(application.getApplicationId()); + rm.getClientRMService().forceKillApplication(request); + + // Specific test for YARN-1689 follows + // Now let's say a race causes AM to register now. This should not crash RM. + am.registerAppAttempt(false); + + // We explicitly intercepted the kill-event to RMAppAttempt, so app should + // still be in KILLING state. + rm.waitForState(application.getApplicationId(), RMAppState.KILLING); + // AM should now be in running + rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING); + + // Simulate that appAttempt is killed. + rm.getRMContext().getDispatcher().getEventHandler().handle( + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED)); + rm.waitForState(application.getApplicationId(), RMAppState.KILLED); + } + public static void main(String[] args) throws Exception { TestRM t = new TestRM(); t.testGetNewAppId(); t.testAppWithNoContainers(); t.testAppOnMultiNode(); + t.testNMToken(); + t.testActivatingApplicationAfterAddingNM(); + t.testInvalidateAMHostPortWhenAMFailedOrKilled(); + t.testInvalidatedAMHostPortOnAMRestart(); + t.testApplicationKillAtAcceptedState(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index f8329d68a75..ca9befd599c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -69,7 +69,7 @@ public class TestAMRestart { new MockNM("127.0.0.1:2351", 4089, rm1.getResourceTrackerService()); nm2.registerNode(); - MockAM am1 = MockRM.launchAM(app1, rm1, nm1); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); int NUM_CONTAINERS = 3; // allocate NUM_CONTAINERS containers am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 5ac9353928e..58482ee38ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -639,6 +639,13 @@ public class TestRMAppTransitions { RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + + assertAppState(RMAppState.KILLING, application); + RMAppEvent appAttemptKilled = + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED); + application.handle(appAttemptKilled); + assertAppState(RMAppState.FINAL_SAVING, application); sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 18350fb30d2..cfdf9283acf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1389,7 +1389,7 @@ public class TestRMWebServicesApps extends JerseyTest { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - MockAM am = MockRM.launchAM(app1, rm, amNodeManager); + MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -1405,7 +1405,7 @@ public class TestRMWebServicesApps extends JerseyTest { } // wait for app to start a new attempt. rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); - am = MockRM.launchAM(app1, rm, amNodeManager); + am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); numAttempt++; } assertEquals("incorrect number of attempts", maxAppAttempts, From fe2fb24a61a8dd0fc3bbee0b3434a503f4c55ad2 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Fri, 7 Feb 2014 00:43:45 +0000 Subject: [PATCH 41/42] HDFS-5807. TestBalancerWithNodeGroup.testBalancerWithNodeGroup fails intermittently. Contributed by Chen He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565505 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ce457bd51b5..09ec9867b13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -345,6 +345,9 @@ Release 2.4.0 - UNRELEASED HDFS-5895. HDFS cacheadmin -listPools has exit_code of 1 when the command returns 0 result. (Tassapol Athiapinya via cnauroth) + HDFS-5807. TestBalancerWithNodeGroup.testBalancerWithNodeGroup fails + intermittently. (Chen He via kihwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java index ff9ea0728c9..eefb6202540 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java @@ -65,7 +65,7 @@ public class TestBalancerWithNodeGroup { ClientProtocol client; - static final long TIMEOUT = 20000L; //msec + static final long TIMEOUT = 40000L; //msec static final double CAPACITY_ALLOWED_VARIANCE = 0.005; // 0.5% static final double BALANCE_ALLOWED_VARIANCE = 0.11; // 10%+delta static final int DEFAULT_BLOCK_SIZE = 10; From 0bf97bda18fc84f5687edc9744812f565d44116b Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 7 Feb 2014 01:12:52 +0000 Subject: [PATCH 42/42] HADOOP-10330. TestFrameDecoder fails if it cannot bind port 12345. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1565507 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../hadoop/oncrpc/TestFrameDecoder.java | 32 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b5b8e73445c..7a7106197d9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -325,6 +325,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10327. Trunk windows build broken after HDFS-5746. (Vinay via cnauroth) + HADOOP-10330. TestFrameDecoder fails if it cannot bind port 12345. + (Arpit Agarwal) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java index cdeaa3f2bed..9f951c3c1d9 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; +import java.util.Random; import org.apache.hadoop.oncrpc.RpcUtil.RpcFrameDecoder; import org.apache.hadoop.oncrpc.security.CredentialsNone; @@ -31,17 +32,17 @@ import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelException; import org.jboss.netty.channel.ChannelHandlerContext; import org.junit.Test; import org.mockito.Mockito; public class TestFrameDecoder { - private static int port = 12345; // some random server port private static int resultSize; - static void testRequest(XDR request) { - SimpleTcpClient tcpClient = new SimpleTcpClient("localhost", port, request, + static void testRequest(XDR request, int serverPort) { + SimpleTcpClient tcpClient = new SimpleTcpClient("localhost", serverPort, request, true); tcpClient.run(); } @@ -148,10 +149,25 @@ public class TestFrameDecoder { @Test public void testFrames() { - RpcProgram program = new TestFrameDecoder.TestRpcProgram("TestRpcProgram", - "localhost", port, 100000, 1, 2); - SimpleTcpServer tcpServer = new SimpleTcpServer(port, program, 1); - tcpServer.run(); + Random rand = new Random(); + int serverPort = 30000 + rand.nextInt(10000); + int retries = 10; // A few retries in case initial choice is in use. + + while (true) { + try { + RpcProgram program = new TestFrameDecoder.TestRpcProgram("TestRpcProgram", + "localhost", serverPort, 100000, 1, 2); + SimpleTcpServer tcpServer = new SimpleTcpServer(serverPort, program, 1); + tcpServer.run(); + break; // Successfully bound a port, break out. + } catch (ChannelException ce) { + if (retries-- > 0) { + serverPort += rand.nextInt(20); // Port in use? Try another. + } else { + throw ce; // Out of retries. + } + } + } XDR xdrOut = createGetportMount(); int headerSize = xdrOut.size(); @@ -161,7 +177,7 @@ public class TestFrameDecoder { int requestSize = xdrOut.size() - headerSize; // Send the request to the server - testRequest(xdrOut); + testRequest(xdrOut, serverPort); // Verify the server got the request with right size assertEquals(requestSize, resultSize);