From 5829029154b8e8e02bc6aeb45435046ca080bbe9 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 25 Oct 2013 18:43:06 +0000 Subject: [PATCH 1/4] HDFS-5257. addBlock() retry should return LocatedBlock with locations else client will get AIOBE. Contributed by Vinay. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1535811 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../apache/hadoop/hdfs/DFSOutputStream.java | 5 ++++ .../hdfs/server/namenode/FSNamesystem.java | 19 ++++++++---- .../server/namenode/TestAddBlockRetry.java | 30 +++++++++++++++++++ 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0e8d274f9e8..86dd218affb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -384,6 +384,9 @@ Release 2.3.0 - UNRELEASED HDFS-5400. DFS_CLIENT_MMAP_CACHE_THREAD_RUNS_PER_TIMEOUT constant is set to the wrong value. (Colin Patrick McCabe) + HDFS-5257. addBlock() retry should return LocatedBlock with locations else client + will get AIOBE. (Vinay via jing9) + Release 2.2.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index cac83d735cb..a1f3258d3a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -1097,6 +1097,11 @@ public class DFSOutputStream extends FSOutputSummer // private boolean createBlockOutputStream(DatanodeInfo[] nodes, long newGS, boolean recoveryFlag) { + if (nodes.length == 0) { + DFSClient.LOG.info("nodes are empty for write pipeline of block " + + block); + return false; + } Status pipelineStatus = SUCCESS; String firstBadLink = ""; if (DFSClient.LOG.isDebugEnabled()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8f1ef3c6f0a..908d02550b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2507,8 +2507,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, final INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) inodes[inodes.length - 1]; - if(onRetryBlock[0] != null) { - // This is a retry. Just return the last block. + if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) { + // This is a retry. Just return the last block if having locations. return onRetryBlock[0]; } if (pendingFile.getBlocks().length >= maxBlocksPerFile) { @@ -2545,9 +2545,18 @@ public class FSNamesystem implements Namesystem, FSClusterStats, final INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) inodes[inodes.length - 1]; - if(onRetryBlock[0] != null) { - // This is a retry. Just return the last block. - return onRetryBlock[0]; + if (onRetryBlock[0] != null) { + if (onRetryBlock[0].getLocations().length > 0) { + // This is a retry. Just return the last block if having locations. + return onRetryBlock[0]; + } else { + // add new chosen targets to already allocated block and return + BlockInfo lastBlockInFile = pendingFile.getLastBlock(); + ((BlockInfoUnderConstruction) lastBlockInFile) + .setExpectedLocations(targets); + offset = pendingFile.computeFileSize(); + return makeLocatedBlock(lastBlockInFile, targets, offset); + } } // commit the last block and complete it if it has minimum replicas diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java index 399dfc210d3..3114c824cea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; @@ -139,4 +140,33 @@ public class TestAddBlockRetry { assertEquals("Wrong replication", REPLICATION, lb1.getLocations().length); assertEquals("Blocks are not equal", lb1.getBlock(), lb2.getBlock()); } + + /* + * Since NameNode will not persist any locations of the block, addBlock() + * retry call after restart NN should re-select the locations and return to + * client. refer HDFS-5257 + */ + @Test + public void testAddBlockRetryShouldReturnBlockWithLocations() + throws Exception { + final String src = "/testAddBlockRetryShouldReturnBlockWithLocations"; + NamenodeProtocols nameNodeRpc = cluster.getNameNodeRpc(); + // create file + nameNodeRpc.create(src, FsPermission.getFileDefault(), "clientName", + new EnumSetWritable(EnumSet.of(CreateFlag.CREATE)), true, + (short) 3, 1024); + // start first addBlock() + LOG.info("Starting first addBlock for " + src); + LocatedBlock lb1 = nameNodeRpc.addBlock(src, "clientName", null, null, + INodeId.GRANDFATHER_INODE_ID, null); + assertTrue("Block locations should be present", + lb1.getLocations().length > 0); + + cluster.restartNameNode(); + nameNodeRpc = cluster.getNameNodeRpc(); + LocatedBlock lb2 = nameNodeRpc.addBlock(src, "clientName", null, null, + INodeId.GRANDFATHER_INODE_ID, null); + assertEquals("Blocks are not equal", lb1.getBlock(), lb2.getBlock()); + assertTrue("Wrong locations with retry", lb2.getLocations().length > 0); + } } From 025f1719472282a30aa26ae3e235e404f04ba932 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Sat, 26 Oct 2013 00:27:40 +0000 Subject: [PATCH 2/4] YARN-1333. Support blacklisting in the Fair Scheduler (Tsuyoshi Ozawa via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1535899 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/capacity/LeafQueue.java | 4 +- .../scheduler/fair/FSLeafQueue.java | 5 ++ .../scheduler/fair/FairScheduler.java | 2 + .../scheduler/fifo/FifoScheduler.java | 13 +--- .../scheduler/fair/TestFairScheduler.java | 65 +++++++++++++++---- 6 files changed, 66 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 79a0feb3fa3..5b7da5471f6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -111,6 +111,9 @@ Release 2.2.1 - UNRELEASED YARN-1335. Move duplicate code from FSSchedulerApp and FiCaSchedulerApp into SchedulerApplication (Sandy Ryza) + YARN-1333. Support blacklisting in the Fair Scheduler (Tsuyoshi Ozawa via + Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index d82e6737797..11e497373e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -57,9 +57,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.Lock; @@ -823,7 +823,7 @@ public class LeafQueue implements CSQueue { synchronized (application) { // Check if this resource is on the blacklist - if (FiCaSchedulerUtils.isBlacklisted(application, node, LOG)) { + if (SchedulerAppUtils.isBlacklisted(application, node, LOG)) { continue; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 2da754c03c6..3064903d426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -33,6 +33,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; import org.apache.hadoop.yarn.util.resource.Resources; @Private @@ -155,6 +156,10 @@ public class FSLeafQueue extends FSQueue { Collections.sort(appScheds, comparator); for (AppSchedulable sched : appScheds) { if (sched.getRunnable()) { + if (SchedulerAppUtils.isBlacklisted(sched.getApp(), node, LOG)) { + continue; + } + assigned = sched.assignContainer(node); if (!assigned.equals(Resources.none())) { break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index a3d57369dce..76e4b6a4267 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -882,6 +882,8 @@ public class FairScheduler implements ResourceScheduler { for (RMContainer container : application.getPreemptionContainers()) { preemptionContainerIds.add(container.getContainerId()); } + + application.updateBlacklist(blacklistAdditions, blacklistRemovals); return new Allocation(application.pullNewlyAllocatedContainers(), application.getHeadroom(), preemptionContainerIds); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 293811e63b0..a2171fb8b57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -65,18 +65,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; @@ -405,7 +396,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { application.showRequests(); synchronized (application) { // Check if this resource is on the blacklist - if (FiCaSchedulerUtils.isBlacklisted(application, node, LOG)) { + if (SchedulerAppUtils.isBlacklisted(application, node, LOG)) { continue; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 9562b6c793d..b2bc7069996 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -22,11 +22,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.File; import java.io.FileWriter; @@ -36,7 +33,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,8 +42,6 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -59,7 +53,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -80,7 +73,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; @@ -90,16 +82,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.xml.sax.SAXException; public class TestFairScheduler { @@ -2447,4 +2435,55 @@ public class TestFairScheduler { assertEquals(2, jerryQueue.getAppSchedulables().size()); assertEquals(2, defaultQueue.getAppSchedulables().size()); } + + @SuppressWarnings("resource") + @Test + public void testBlacklistNodes() throws Exception { + final int GB = 1024; + String host = "127.0.0.1"; + RMNode node = + MockNodes.newNodeInfo(1, Resources.createResource(16 * GB, 16), + 0, host); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + + ApplicationAttemptId appAttemptId = + createSchedulingRequest(GB, "root.default", "user", 1); + FSSchedulerApp app = scheduler.applications.get(appAttemptId); + + // Verify the blacklist can be updated independent of requesting containers + scheduler.allocate(appAttemptId, Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(host), null); + assertTrue(app.isBlacklisted(host)); + scheduler.allocate(appAttemptId, Collections.emptyList(), + Collections.emptyList(), null, + Collections.singletonList(host)); + assertFalse(scheduler.applications.get(appAttemptId).isBlacklisted(host)); + + List update = Arrays.asList( + createResourceRequest(GB, node.getHostName(), 1, 0, true)); + + // Verify a container does not actually get placed on the blacklisted host + scheduler.allocate(appAttemptId, update, + Collections.emptyList(), + Collections.singletonList(host), null); + assertTrue(app.isBlacklisted(host)); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Incorrect number of containers allocated", 0, app + .getLiveContainers().size()); + + // Verify a container gets placed on the empty blacklist + scheduler.allocate(appAttemptId, update, + Collections.emptyList(), null, + Collections.singletonList(host)); + assertFalse(app.isBlacklisted(host)); + createSchedulingRequest(GB, "root.default", "user", 1); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Incorrect number of containers allocated", 1, app + .getLiveContainers().size()); + } } From b4a5476d95271bf3df6eb19f5d6b4c65f816396d Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Sat, 26 Oct 2013 00:31:47 +0000 Subject: [PATCH 3/4] YARN-1333: Add missing file SchedulerAppUtils git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1535900 13f79535-47bb-0310-9956-ffa450edef68 --- .../scheduler/SchedulerAppUtils.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java new file mode 100644 index 00000000000..be68fe2e28f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.commons.logging.Log; + +public class SchedulerAppUtils { + + public static boolean isBlacklisted(SchedulerApplication application, + SchedulerNode node, Log LOG) { + if (application.isBlacklisted(node.getNodeName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping 'host' " + node.getNodeName() + + " for " + application.getApplicationId() + + " since it has been blacklisted"); + } + return true; + } + + if (application.isBlacklisted(node.getRackName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping 'rack' " + node.getRackName() + + " for " + application.getApplicationId() + + " since it has been blacklisted"); + } + return true; + } + + return false; + } + +} From 7e820b9965d2bf693a2f0c0c59a7cb23aa6169b3 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Sat, 26 Oct 2013 04:25:50 +0000 Subject: [PATCH 4/4] HADOOP-10072. TestNfsExports#testMultiMatchers fails due to non-deterministic timing around cache expiry check. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1535918 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/nfs/TestNfsExports.java | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1ebdfa426a9..636a10a3ad1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -449,6 +449,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10055. FileSystemShell.apt.vm doc has typo "numRepicas". (Akira Ajisaka via cnauroth) + HADOOP-10072. TestNfsExports#testMultiMatchers fails due to non-deterministic + timing around cache expiry check. (cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/TestNfsExports.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/TestNfsExports.java index 9acb29e589a..30b6ff932ae 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/TestNfsExports.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/TestNfsExports.java @@ -35,6 +35,7 @@ public class TestNfsExports { Nfs3Constant.EXPORTS_CACHE_EXPIRYTIME_MILLIS_DEFAULT * 1000 * 1000; private static final int CacheSize = Nfs3Constant.EXPORTS_CACHE_SIZE_DEFAULT; + private static final long NanosPerMillis = 1000000; @Test public void testWildcardRW() { @@ -185,7 +186,15 @@ public class TestNfsExports { Thread.sleep(1000); // no cache for address2 now - Assert.assertEquals(AccessPrivilege.NONE, - matcher.getAccessPrivilege(address2, address2)); + AccessPrivilege ap; + long startNanos = System.nanoTime(); + do { + ap = matcher.getAccessPrivilege(address2, address2); + if (ap == AccessPrivilege.NONE) { + break; + } + Thread.sleep(500); + } while ((System.nanoTime() - startNanos) / NanosPerMillis < 5000); + Assert.assertEquals(AccessPrivilege.NONE, ap); } }