From 5e64e62deeb467ac2969bc357a24810db855cffa Mon Sep 17 00:00:00 2001 From: Sunil G Date: Mon, 3 Sep 2018 12:11:19 +0530 Subject: [PATCH] YARN-8740. Clear node attribute path after each test run. Contributed by Bibin A Chundatt. --- .../nodelabels/NodeAttributesManagerImpl.java | 13 +++-- .../NodeAttributeTestUtils.java | 54 +++++++++++++++++++ .../resourcemanager/TestClientRMService.java | 9 ++-- .../resourcemanager/TestRMAdminService.java | 3 +- .../TestFileSystemNodeAttributeStore.java | 28 +++++++--- .../nodelabels/TestNodeAttributesManager.java | 9 +--- 6 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java index 81f42d92b52..55334365abf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -151,7 +152,8 @@ private NodeAttributeStore getAttributeStoreClass(Configuration conf) { } } - private void internalUpdateAttributesOnNodes( + @VisibleForTesting + protected void internalUpdateAttributesOnNodes( Map> nodeAttributeMapping, AttributeMappingOperationType op, Map newAttributesToBeAdded, @@ -195,7 +197,7 @@ private void internalUpdateAttributesOnNodes( logMsg.append(" NM = "); logMsg.append(entry.getKey()); logMsg.append(", attributes=[ "); - logMsg.append(StringUtils.join(entry.getValue().entrySet(), ",")); + logMsg.append(StringUtils.join(entry.getValue().keySet(), ",")); logMsg.append("] ,"); } @@ -700,9 +702,10 @@ private void processMapping( new HashMap<>(); Map> validMapping = validate(nodeAttributeMapping, newAttributesToBeAdded, false); - - internalUpdateAttributesOnNodes(validMapping, mappingType, - newAttributesToBeAdded, attributePrefix); + if (validMapping.size() > 0) { + internalUpdateAttributesOnNodes(validMapping, mappingType, + newAttributesToBeAdded, attributePrefix); + } } protected void stopDispatcher() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java new file mode 100644 index 00000000000..1d12cb7ec12 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.io.File; +import java.io.IOException; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * Test Utils for NodeAttribute. + */ +public final class NodeAttributeTestUtils { + + private NodeAttributeTestUtils() { + + } + + public static YarnConfiguration getRandomDirConf(Configuration conf) + throws IOException { + YarnConfiguration newConf; + if (conf == null) { + newConf = new YarnConfiguration(); + } else { + newConf = new YarnConfiguration(conf); + } + File tempDir = GenericTestUtils.getRandomizedTestDir(); + FileUtils.deleteDirectory(tempDir); + tempDir.mkdirs(); + tempDir.deleteOnExit(); + newConf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR, + tempDir.getAbsolutePath()); + return newConf; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 39892f22776..efabbe2c01e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -2012,7 +2012,8 @@ protected ClientRMService createClientRMService() { @Test(timeout = 120000) public void testGetClusterNodeAttributes() throws IOException, YarnException { - MockRM rm = new MockRM() { + Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null); + MockRM rm = new MockRM(newConf) { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, @@ -2061,7 +2062,8 @@ protected ClientRMService createClientRMService() { @Test(timeout = 120000) public void testGetAttributesToNodes() throws IOException, YarnException { - MockRM rm = new MockRM() { + Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null); + MockRM rm = new MockRM(newConf) { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, @@ -2152,7 +2154,8 @@ private boolean findHostnameAndValInMapping(String hostname, String attrVal, @Test(timeout = 120000) public void testGetNodesToAttributes() throws IOException, YarnException { - MockRM rm = new MockRM() { + Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null); + MockRM rm = new MockRM(newConf) { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index a1b95b87b48..475dd8cef89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -1557,7 +1557,8 @@ public void testMapAttributesToNodes() throws Exception, YarnException { // 1.5. Test with unknown node when failOnUnknownNodes is false // also test : 3. Ensure Appropriate manager Method call is done - rm = new MockRM(); + Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null); + rm = new MockRM(newConf); NodeAttributesManager spiedAttributesManager = Mockito.spy(rm.getRMContext().getNodeAttributesManager()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java index 07ea20ddda7..4a7a59fc38c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java @@ -17,6 +17,11 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeAttribute; @@ -25,12 +30,12 @@ import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.nodelabels.AttributeValue; import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore; +import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -68,12 +73,7 @@ public void before() throws IOException { conf = new Configuration(); conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS, FileSystemNodeAttributeStore.class, NodeAttributeStore.class); - File tempDir = File.createTempFile("nattr", ".tmp"); - tempDir.delete(); - tempDir.mkdirs(); - tempDir.deleteOnExit(); - conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR, - tempDir.getAbsolutePath()); + conf = NodeAttributeTestUtils.getRandomDirConf(conf); mgr.init(conf); mgr.start(); } @@ -86,6 +86,20 @@ public void after() throws IOException { mgr.stop(); } + @Test(timeout = 10000) + public void testEmptyRecoverSkipInternalUdpate() throws Exception { + // Stop manager + mgr.stop(); + + // Start new attribute manager with same path + mgr = spy(new MockNodeAttrbuteManager()); + mgr.init(conf); + mgr.start(); + + verify(mgr, times(0)) + .internalUpdateAttributesOnNodes(any(), any(), any(), any()); + } + @Test(timeout = 10000) public void testRecoverWithMirror() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java index 9bc9388fb96..b1816774241 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java @@ -28,12 +28,12 @@ import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore; import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil; +import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils; import org.junit.Test; import org.junit.Before; import org.junit.After; import org.junit.Assert; -import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; @@ -57,12 +57,7 @@ public void init() throws IOException { attributesManager = new NodeAttributesManagerImpl(); conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS, FileSystemNodeAttributeStore.class, NodeAttributeStore.class); - File tempDir = File.createTempFile("nattr", ".tmp"); - tempDir.delete(); - tempDir.mkdirs(); - tempDir.deleteOnExit(); - conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR, - tempDir.getAbsolutePath()); + conf = NodeAttributeTestUtils.getRandomDirConf(conf); attributesManager.init(conf); attributesManager.start(); }