From a72cdcc47a1b3a4fcadbc9e12bd1bae2bbf04d83 Mon Sep 17 00:00:00 2001 From: Jian He Date: Tue, 23 Jan 2018 10:53:27 -0800 Subject: [PATCH] YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha --- .../yarn/service/client/ServiceClient.java | 35 ++++++++++++++----- .../yarn/service/conf/YarnServiceConf.java | 6 ++++ .../tarball/TarballProviderService.java | 4 +-- .../yarn/service/utils/CoreFileSystem.java | 32 +++++++---------- .../yarn/client/api/AppAdminClient.java | 12 ++++--- .../yarn/client/cli/ApplicationCLI.java | 13 ++++--- .../hadoop/yarn/client/cli/TestYarnCLI.java | 5 ++- 7 files changed, 68 insertions(+), 39 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index c224089119c..b8034b33314 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -718,14 +719,20 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, libPath, "lib", false); Path dependencyLibTarGzip = fs.getDependencyTarGzip(); if (fs.isFile(dependencyLibTarGzip)) { - LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/" - + dependencyLibTarGzip); + LOG.info("Loading lib tar from " + dependencyLibTarGzip); fs.submitTarGzipAndUpdate(localResources); } else { + if (dependencyLibTarGzip != null) { + LOG.warn("Property {} has a value {}, but is not a valid file", + YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip); + } String[] libs = ServiceUtils.getLibDirs(); - LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + - " apps, pre-upload dependency jars to HDFS " - + "using command: yarn app -enableFastLaunch"); + LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + + " apps, set config property {} to the dependency tarball location." + + " Dependency tarball can be uploaded to any HDFS path directly" + + " or by using command: yarn app -{} []", + YarnServiceConf.DEPENDENCY_TARBALL_PATH, + ApplicationCLI.ENABLE_FAST_LAUNCH); for (String libDirProp : libs) { ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib", libDirProp); @@ -988,16 +995,23 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, return this.yarnClient; } - public int enableFastLaunch() throws IOException, YarnException { - return actionDependency(true); + public int enableFastLaunch(String destinationFolder) + throws IOException, YarnException { + return actionDependency(destinationFolder, true); } - public int actionDependency(boolean overwrite) + public int actionDependency(String destinationFolder, boolean overwrite) throws IOException, YarnException { String currentUser = RegistryUtils.currentUser(); LOG.info("Running command as user {}", currentUser); - Path dependencyLibTarGzip = fs.getDependencyTarGzip(); + if (destinationFolder == null) { + destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR, + VersionInfo.getVersion()); + } + Path dependencyLibTarGzip = new Path(destinationFolder, + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); // Check if dependency has already been uploaded, in which case log // appropriately and exit success (unless overwrite has been requested) @@ -1019,6 +1033,9 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, LOG.info("Version Info: " + VersionInfo.getBuildVersion()); fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip, new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS)); + LOG.info("To let apps use this tarball, in yarn-site set config property " + + "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH, + dependencyLibTarGzip); return EXIT_SUCCESS; } else { return EXIT_FALSE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index 22926e23eb4..21470d41a68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java @@ -93,6 +93,12 @@ public class YarnServiceConf { public static final int DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS = 120000; + /** + * The dependency tarball file location. + */ + public static final String DEPENDENCY_TARBALL_PATH = YARN_SERVICE_PREFIX + + "framework.path"; + /** * Get long value for the property. First get from the userConf, if not * present, get from systemConf. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java index 9f29c8be31b..7f29d1f5c1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java @@ -38,9 +38,9 @@ public class TarballProviderService extends AbstractProviderService { Path artifact = new Path(instance.getCompSpec().getArtifact().getId()); if (!fileSystem.isFile(artifact)) { throw new IOException( - "Package doesn't exist as a resource: " + artifact.toString()); + "Package doesn't exist as a resource: " + artifact); } - log.info("Adding resource {}", artifact.toString()); + log.info("Adding resource {}", artifact); LocalResourceType type = LocalResourceType.ARCHIVE; LocalResource packageResource = fileSystem.createAmResource(artifact, type); launcher.addLocalResource(APP_LIB_DIR, packageResource); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index 6b9d4d51296..284825ee119 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -295,6 +295,9 @@ public class CoreFileSystem { * reasons including if file check throws IOException */ public boolean isFile(Path path) { + if (path == null) { + return false; + } boolean isFile = false; try { FileStatus status = fileSystem.getFileStatus(path); @@ -321,26 +324,18 @@ public class CoreFileSystem { } /** - * Get slider dependency parent dir in HDFS + * Get service dependency absolute filepath in HDFS used for application + * submission. * - * @return the parent dir path of slider.tar.gz in HDFS - */ - public Path getDependencyPath() { - String parentDir = YarnServiceConstants.DEPENDENCY_DIR; - return new Path(String.format(parentDir, VersionInfo.getVersion())); - } - - /** - * Get slider.tar.gz absolute filepath in HDFS - * - * @return the absolute path to slider.tar.gz in HDFS + * @return the absolute path to service dependency tarball in HDFS */ public Path getDependencyTarGzip() { - Path dependencyLibAmPath = getDependencyPath(); - Path dependencyLibTarGzip = new Path( - dependencyLibAmPath.toUri().toString(), - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME - + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); + Path dependencyLibTarGzip = null; + String configuredDependencyTarballPath = configuration + .get(YarnServiceConf.DEPENDENCY_TARBALL_PATH); + if (configuredDependencyTarballPath != null) { + dependencyLibTarGzip = new Path(configuredDependencyTarballPath); + } return dependencyLibTarGzip; } @@ -467,8 +462,7 @@ public class CoreFileSystem { fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); fileSystem.mkdirs(destPath.getParent(), fp); - log.info("Copying file {} to {}", localPath.toURI(), - fileSystem.getScheme() + ":/" + destPath.toUri()); + log.info("Copying file {} to {}", localPath.toURI(), destPath); fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), destPath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java index 6aba91a5c2d..55be13b10b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java @@ -196,14 +196,18 @@ public abstract class AppAdminClient extends CompositeService { * faster since the dependencies do not have to be uploaded on each launch. *

* + * @param destinationFolder + * an optional HDFS folder where dependency tarball will be uploaded * @return exit code - * @throws IOException IOException - * @throws YarnException exception in client or server + * @throws IOException + * IOException + * @throws YarnException + * exception in client or server */ @Public @Unstable - public abstract int enableFastLaunch() throws IOException, - YarnException; + public abstract int enableFastLaunch(String destinationFolder) + throws IOException, YarnException; /** *

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index c751f796c64..471b4d6d0ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -232,9 +232,10 @@ public class ApplicationCLI extends YarnCLI { "the number of components/containers running for an application / " + "long-running service. Supports absolute or relative changes, such " + "as +1, 2, or -3."); - opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " + - "to HDFS to make future launches faster. Supports -appTypes option" + - " to specify which client implementation to use."); + opts.addOption(ENABLE_FAST_LAUNCH, true, "Uploads AM dependencies " + + "to HDFS to make future launches faster. Supports -appTypes option " + + "to specify which client implementation to use. Optionally a " + + "destination folder for the tarball can be specified."); opts.getOption(LAUNCH_CMD).setArgName("Application Name> Uploads AM dependencies to HDFS"); pw.println(" to make future launches faster."); pw.println(" Supports -appTypes option to"); pw.println(" specify which client"); pw.println(" implementation to use."); + pw.println(" Optionally a destination folder"); + pw.println(" for the tarball can be"); + pw.println(" specified."); pw.println(" -flex Changes number of running"); pw.println(" containers for a component of an"); pw.println(" application / long-running");