From 9f60339854ebf9d76c036c1838a2ea1c206f55ac Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 9 Oct 2012 02:27:18 +0000 Subject: [PATCH] merge YARN-33 from trunk. Change LocalDirsHandlerService to validate the configured local and log dirs. (Contributed by Mayank Bansal) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1395845 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../nodemanager/LocalDirsHandlerService.java | 43 ++++++++-- .../TestLocalDirsHandlerService.java | 79 +++++++++++++++++++ .../TestResourceLocalizationService.java | 8 +- 4 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 91ec0cc92f2..7adf2257db9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -35,6 +35,9 @@ Release 2.0.3-alpha - Unreleased YARN-40. Provided support for missing YARN commands (Devaraj K and Vinod Kumar Vavilapalli via vinodkv) + YARN-33. Change LocalDirsHandlerService to validate the configured local and + log dirs. (Mayank Bansal via sseth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index 0c7e01d17f9..96a58dda191 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -19,6 +19,9 @@ package org.apache.hadoop.yarn.server.nodemanager; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; import java.util.Timer; import java.util.TimerTask; @@ -77,6 +80,8 @@ public class LocalDirsHandlerService extends AbstractService { /** when disk health checking code was last run */ private long lastDisksCheckTime; + + private static String FILE_SCHEME = "file"; /** * Class which is used by the {@link Timer} class to periodically execute the @@ -84,13 +89,13 @@ public class LocalDirsHandlerService extends AbstractService { */ private final class MonitoringTimerTask extends TimerTask { - public MonitoringTimerTask(Configuration conf) { + public MonitoringTimerTask(Configuration conf) throws YarnException { localDirs = new DirectoryCollection( - conf.getTrimmedStrings(YarnConfiguration.NM_LOCAL_DIRS)); + validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOCAL_DIRS))); logDirs = new DirectoryCollection( - conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)); - localDirsAllocator = - new LocalDirAllocator(YarnConfiguration.NM_LOCAL_DIRS); + validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS))); + localDirsAllocator = new LocalDirAllocator( + YarnConfiguration.NM_LOCAL_DIRS); logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS); } @@ -106,6 +111,7 @@ public class LocalDirsHandlerService extends AbstractService { /** * Method which initializes the timertask and its interval time. + * */ @Override public void init(Configuration config) { @@ -294,4 +300,31 @@ public class LocalDirsHandlerService extends AbstractService { public Path getLogPathToRead(String pathStr) throws IOException { return logDirsAllocator.getLocalPathToRead(pathStr, getConfig()); } + + public static String[] validatePaths(String[] paths) { + ArrayList validPaths = new ArrayList(); + for (int i = 0; i < paths.length; ++i) { + try { + URI uriPath = new URI(paths[i]); + if (uriPath.getScheme() == null + || uriPath.getScheme().equals(FILE_SCHEME)) { + validPaths.add(uriPath.getPath()); + } else { + LOG.warn(paths[i] + " is not a valid path. Path should be with " + + FILE_SCHEME + " scheme or without scheme"); + throw new YarnException(paths[i] + + " is not a valid path. Path should be with " + FILE_SCHEME + + " scheme or without scheme"); + } + } catch (URISyntaxException e) { + LOG.warn(e.getMessage()); + throw new YarnException(paths[i] + + " is not a valid path. Path should be with " + FILE_SCHEME + + " scheme or without scheme"); + } + } + String[] arrValidPaths = new String[validPaths.size()]; + validPaths.toArray(arrValidPaths); + return arrValidPaths; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java new file mode 100644 index 00000000000..fc6fba04517 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.yarn.YarnException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.Service.STATE; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestLocalDirsHandlerService { + private static final File testDir = new File("target", + TestDirectoryCollection.class.getName()).getAbsoluteFile(); + private static final File testFile = new File(testDir, "testfile"); + + @BeforeClass + public static void setup() throws IOException { + testDir.mkdirs(); + testFile.createNewFile(); + } + + @AfterClass + public static void teardown() { + FileUtil.fullyDelete(testDir); + } + + @Test + public void testDirStructure() throws Exception { + Configuration conf = new YarnConfiguration(); + String localDir1 = new File("file:///" + testDir, "localDir1").getPath(); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1); + String logDir1 = new File("file:///" + testDir, "logDir1").getPath(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1); + LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); + dirSvc.init(conf); + Assert.assertEquals(1, dirSvc.getLocalDirs().size()); + } + + @Test + public void testValidPathsDirHandlerService() { + Configuration conf = new YarnConfiguration(); + String localDir1 = new File("file:///" + testDir, "localDir1").getPath(); + String localDir2 = new File("hdfs:///" + testDir, "localDir2").getPath(); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2); + String logDir1 = new File("file:///" + testDir, "logDir1").getPath(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1); + LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); + try { + dirSvc.init(conf); + Assert.fail("Service should have thrown an exception due to wrong URI"); + } catch (YarnException e) { + } + Assert.assertTrue("Service should not be inited", dirSvc.getServiceState() + .compareTo(STATE.NOTINITED) == 0); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index 1572f36f320..aa53a586d3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.net.InetSocketAddress; +import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -157,6 +158,7 @@ public class TestResourceLocalizationService { // verify directory creation for (Path p : localDirs) { + p = new Path((new URI(p.toString())).getPath()); Path usercache = new Path(p, ContainerLocalizer.USERCACHE); verify(spylfs) .mkdir(eq(usercache), @@ -192,7 +194,8 @@ public class TestResourceLocalizationService { sDirs[i] = localDirs.get(i).toString(); } conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); - + String logDir = lfs.makeQualified(new Path(basedir, "logdir " )).toString(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logDir); LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); @@ -379,7 +382,8 @@ public class TestResourceLocalizationService { sDirs[i] = localDirs.get(i).toString(); } conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); - + String logDir = lfs.makeQualified(new Path(basedir, "logdir " )).toString(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logDir); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start();