From 2b97a4cce4dc1bd0528e960f364edfa132d6bfa3 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 9 Aug 2012 19:18:37 +0000 Subject: [PATCH] svn merge -c 1371390 FIXES: YARN-14. Symlinks to peer distributed cache files no longer work (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1371395 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 13 +++++ .../containermanager/container/Container.java | 3 +- .../container/ContainerImpl.java | 57 +++++++++++-------- .../launcher/ContainerLaunch.java | 11 ++-- .../container/TestContainer.java | 21 ++++--- .../launcher/TestContainerLaunch.java | 6 +- .../nodemanager/webapp/MockContainer.java | 6 +- 7 files changed, 76 insertions(+), 41 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1ae8a9dd1bf..9cfad504cc4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -36,3 +36,16 @@ Release 2.1.0-alpha - Unreleased YARN-12. Fix findbugs warnings in FairScheduler. (Junping Du via acmurthy) +Release 0.23.3 - Unreleased + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + BUG FIXES + + YARN-14. Symlinks to peer distributed cache files no longer work + (Jason Lowe via bobby) + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index e5ba3f29936..af0f92ee6fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.container; +import java.util.List; import java.util.Map; import org.apache.hadoop.fs.Path; @@ -38,7 +39,7 @@ public interface Container extends EventHandler { Credentials getCredentials(); - Map getLocalizedResources(); + Map> getLocalizedResources(); ContainerStatus cloneAndGetContainerStatus(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 1cbdbaa8146..c9802080854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -84,10 +84,10 @@ public class ContainerImpl implements Container { private static final Log LOG = LogFactory.getLog(Container.class); private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - private final Map pendingResources = - new HashMap(); - private final Map localizedResources = - new HashMap(); + private final Map> pendingResources = + new HashMap>(); + private final Map> localizedResources = + new HashMap>(); private final List publicRsrcs = new ArrayList(); private final List privateRsrcs = @@ -327,7 +327,7 @@ public String getUser() { } @Override - public Map getLocalizedResources() { + public Map> getLocalizedResources() { this.readLock.lock(); try { assert ContainerState.LOCALIZED == getContainerState(); // TODO: FIXME!! @@ -496,20 +496,25 @@ public ContainerState transition(ContainerImpl container, try { for (Map.Entry rsrc : cntrRsrc.entrySet()) { try { - LocalResourceRequest req = - new LocalResourceRequest(rsrc.getValue()); - container.pendingResources.put(req, rsrc.getKey()); - switch (rsrc.getValue().getVisibility()) { - case PUBLIC: - container.publicRsrcs.add(req); - break; - case PRIVATE: - container.privateRsrcs.add(req); - break; - case APPLICATION: - container.appRsrcs.add(req); - break; - } + LocalResourceRequest req = + new LocalResourceRequest(rsrc.getValue()); + List links = container.pendingResources.get(req); + if (links == null) { + links = new ArrayList(); + container.pendingResources.put(req, links); + } + links.add(rsrc.getKey()); + switch (rsrc.getValue().getVisibility()) { + case PUBLIC: + container.publicRsrcs.add(req); + break; + case PRIVATE: + container.privateRsrcs.add(req); + break; + case APPLICATION: + container.appRsrcs.add(req); + break; + } } catch (URISyntaxException e) { LOG.info("Got exception parsing " + rsrc.getKey() + " and value " + rsrc.getValue()); @@ -560,15 +565,16 @@ static class LocalizedTransition implements public ContainerState transition(ContainerImpl container, ContainerEvent event) { ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event; - String sym = container.pendingResources.remove(rsrcEvent.getResource()); - if (null == sym) { + List syms = + container.pendingResources.remove(rsrcEvent.getResource()); + if (null == syms) { LOG.warn("Localized unknown resource " + rsrcEvent.getResource() + " for container " + container.getContainerID()); assert false; // fail container? return ContainerState.LOCALIZING; } - container.localizedResources.put(rsrcEvent.getLocation(), sym); + container.localizedResources.put(rsrcEvent.getLocation(), syms); if (!container.pendingResources.isEmpty()) { return ContainerState.LOCALIZING; } @@ -728,15 +734,16 @@ static class LocalizedResourceDuringKillTransition implements @Override public void transition(ContainerImpl container, ContainerEvent event) { ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event; - String sym = container.pendingResources.remove(rsrcEvent.getResource()); - if (null == sym) { + List syms = + container.pendingResources.remove(rsrcEvent.getResource()); + if (null == syms) { LOG.warn("Localized unknown resource " + rsrcEvent.getResource() + " for container " + container.getContainerID()); assert false; // fail container? return; } - container.localizedResources.put(rsrcEvent.getLocation(), sym); + container.localizedResources.put(rsrcEvent.getLocation(), syms); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 821d4a042b3..b06788341fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -111,7 +111,8 @@ public ContainerLaunch(Configuration configuration, Dispatcher dispatcher, @SuppressWarnings("unchecked") // dispatcher not typed public Integer call() { final ContainerLaunchContext launchContext = container.getLaunchContext(); - final Map localResources = container.getLocalizedResources(); + final Map> localResources = + container.getLocalizedResources(); ContainerId containerID = container.getContainerID(); String containerIdStr = ConverterUtils.toString(containerID); final String user = launchContext.getUser(); @@ -533,7 +534,7 @@ public void sanitizeEnv(Map environment, } static void writeLaunchEnv(OutputStream out, - Map environment, Map resources, + Map environment, Map> resources, List command) throws IOException { ShellScriptBuilder sb = new ShellScriptBuilder(); @@ -543,8 +544,10 @@ static void writeLaunchEnv(OutputStream out, } } if (resources != null) { - for (Map.Entry link : resources.entrySet()) { - sb.symlink(link.getKey(), link.getValue()); + for (Map.Entry> entry : resources.entrySet()) { + for (String linkName : entry.getValue()) { + sb.symlink(entry.getKey(), linkName); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index bc6ec196e17..cb7c19dc2fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -29,12 +29,15 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Random; @@ -111,11 +114,12 @@ public void testLocalizationLaunch() throws Exception { wc = new WrappedContainer(8, 314159265358979L, 4344, "yak"); assertEquals(ContainerState.NEW, wc.c.getContainerState()); wc.initContainer(); - Map localPaths = wc.localizeResources(); + Map> localPaths = wc.localizeResources(); // all resources should be localized assertEquals(ContainerState.LOCALIZED, wc.c.getContainerState()); - for (Entry loc : wc.c.getLocalizedResources().entrySet()) { + for (Entry> loc : wc.c.getLocalizedResources() + .entrySet()) { assertEquals(localPaths.remove(loc.getKey()), loc.getValue()); } assertTrue(localPaths.isEmpty()); @@ -578,10 +582,12 @@ public void initContainer() { // Localize resources // Skip some resources so as to consider them failed - public Map doLocalizeResources(boolean checkLocalizingState, - int skipRsrcCount) throws URISyntaxException { + public Map> doLocalizeResources( + boolean checkLocalizingState, int skipRsrcCount) + throws URISyntaxException { Path cache = new Path("file:///cache"); - Map localPaths = new HashMap(); + Map> localPaths = + new HashMap>(); int counter = 0; for (Entry rsrc : localResources.entrySet()) { if (counter++ < skipRsrcCount) { @@ -592,7 +598,7 @@ public Map doLocalizeResources(boolean checkLocalizingState, } LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); Path p = new Path(cache, rsrc.getKey()); - localPaths.put(p, rsrc.getKey()); + localPaths.put(p, Arrays.asList(rsrc.getKey())); // rsrc copied to p c.handle(new ContainerResourceLocalizedEvent(c.getContainerID(), req, p)); @@ -602,7 +608,8 @@ public Map doLocalizeResources(boolean checkLocalizingState, } - public Map localizeResources() throws URISyntaxException { + public Map> localizeResources() + throws URISyntaxException { return doLocalizeResources(true, 0); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index bdd77f8a20b..822835dc3d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -28,6 +28,7 @@ import java.io.PrintWriter; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -95,9 +96,10 @@ public void testSpecialCharSymlinks() throws IOException { writer.println(timeoutCommand); writer.close(); - Map resources = new HashMap(); + Map> resources = + new HashMap>(); Path path = new Path(shellFile.getAbsolutePath()); - resources.put(path, badSymlink); + resources.put(path, Arrays.asList(badSymlink)); FileOutputStream fos = new FileOutputStream(tempFile); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 1b2e0653d06..519ff183484 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -43,7 +44,8 @@ public class MockContainer implements Container { private ContainerState state; private String user; private ContainerLaunchContext launchContext; - private final Map resource = new HashMap(); + private final Map> resource = + new HashMap>(); private RecordFactory recordFactory; public MockContainer(ApplicationAttemptId appAttemptId, @@ -92,7 +94,7 @@ public Credentials getCredentials() { } @Override - public Map getLocalizedResources() { + public Map> getLocalizedResources() { return resource; }