YARN-7799. Improved YARN service jar file handling.

Contributed by Billie Rinaldi
(cherry picked from commit 24eeea8b18)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml

@@ -172,6 +172,11 @@
       <artifactId>hadoop-yarn-api</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java

@@ -28,7 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
@@ -37,8 +39,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
 import org.apache.hadoop.registry.client.binding.RegistryUtils;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
@@ -896,13 +898,13 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
 
   protected Path addJarResource(String serviceName,
       Map<String, LocalResource> localResources)
-      throws IOException, SliderException {
+      throws IOException, YarnException {
     Path libPath = fs.buildClusterDirPath(serviceName);
     ProviderUtils
         .addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
             libPath, "lib", false);
     Path dependencyLibTarGzip = fs.getDependencyTarGzip();
-    if (fs.isFile(dependencyLibTarGzip)) {
+    if (actionDependency(null, false) == EXIT_SUCCESS) {
       LOG.info("Loading lib tar from " + dependencyLibTarGzip);
       fs.submitTarGzipAndUpdate(localResources);
     } else {
@@ -1223,18 +1225,18 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
     return actionDependency(destinationFolder, true);
   }
 
-  public int actionDependency(String destinationFolder, boolean overwrite)
+  public int actionDependency(String destinationFolder, boolean overwrite) {
-      throws IOException, YarnException {
     String currentUser = RegistryUtils.currentUser();
     LOG.info("Running command as user {}", currentUser);
 
+    Path dependencyLibTarGzip;
     if (destinationFolder == null) {
-      destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR,
+      dependencyLibTarGzip = fs.getDependencyTarGzip();
-          VersionInfo.getVersion());
+    } else {
+      dependencyLibTarGzip = new Path(destinationFolder,
+          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+              + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
     }
-    Path dependencyLibTarGzip = new Path(destinationFolder,
-        YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
-            + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
 
     // Check if dependency has already been uploaded, in which case log
     // appropriately and exit success (unless overwrite has been requested)
@@ -1247,24 +1249,71 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
 
     String[] libDirs = ServiceUtils.getLibDirs();
     if (libDirs.length > 0) {
-      File tempLibTarGzipFile = File.createTempFile(
+      File tempLibTarGzipFile = null;
-          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
+      try {
-          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+        if (!checkPermissions(dependencyLibTarGzip)) {
-      // copy all jars
+          return EXIT_UNAUTHORIZED;
-      tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
+        }
 
-      LOG.info("Version Info: " + VersionInfo.getBuildVersion());
+        tempLibTarGzipFile = File.createTempFile(
-      fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
+            YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
-          new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+            YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
-      LOG.info("To let apps use this tarball, in yarn-site set config property "
+        // copy all jars
-          + "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+        tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
-          dependencyLibTarGzip);
+
-      return EXIT_SUCCESS;
+        fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
+            new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+        LOG.info("To let apps use this tarball, in yarn-site set config " +
+                "property {} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+            dependencyLibTarGzip);
+        return EXIT_SUCCESS;
+      } catch (IOException e) {
+        LOG.error("Got exception creating tarball and uploading to HDFS", e);
+        return EXIT_EXCEPTION_THROWN;
+      } finally {
+        if (tempLibTarGzipFile != null) {
+          if (!tempLibTarGzipFile.delete()) {
+            LOG.warn("Failed to delete tmp file {}", tempLibTarGzipFile);
+          }
+        }
+      }
     } else {
       return EXIT_FALSE;
     }
   }
 
+  private boolean checkPermissions(Path dependencyLibTarGzip) throws
+      IOException {
+    AccessControlList yarnAdminAcl = new AccessControlList(getConfig().get(
+        YarnConfiguration.YARN_ADMIN_ACL,
+        YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+    AccessControlList dfsAdminAcl = new AccessControlList(
+        getConfig().get(DFSConfigKeys.DFS_ADMIN, " "));
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    if (!yarnAdminAcl.isUserAllowed(ugi) && !dfsAdminAcl.isUserAllowed(ugi)) {
+      LOG.error("User must be on the {} or {} list to have permission to " +
+          "upload AM dependency tarball", YarnConfiguration.YARN_ADMIN_ACL,
+          DFSConfigKeys.DFS_ADMIN);
+      return false;
+    }
+
+    Path parent = dependencyLibTarGzip.getParent();
+    while (parent != null) {
+      if (fs.getFileSystem().exists(parent)) {
+        FsPermission perm = fs.getFileSystem().getFileStatus(parent)
+            .getPermission();
+        if (!perm.getOtherAction().implies(FsAction.READ_EXECUTE)) {
+          LOG.error("Parent directory {} of {} tarball location {} does not " +
+              "have world read/execute permission", parent, YarnServiceConf
+              .DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
+          return false;
+        }
+      }
+      parent = parent.getParent();
+    }
+    return true;
+  }
+
   protected ClientAMProtocol createAMProxy(String serviceName,
       ApplicationReport appReport) throws IOException, YarnException {
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -363,6 +364,12 @@ public class CoreFileSystem {
     if (configuredDependencyTarballPath != null) {
       dependencyLibTarGzip = new Path(configuredDependencyTarballPath);
     }
+    if (dependencyLibTarGzip == null) {
+      dependencyLibTarGzip = new Path(String.format(YarnServiceConstants
+          .DEPENDENCY_DIR, VersionInfo.getVersion()),
+          YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+              + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+    }
     return dependencyLibTarGzip;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java

@@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.service.client;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -27,12 +31,15 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,21 +47,33 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.Arrays;
 import java.util.List;
 
 import static org.apache.hadoop.yarn.client.api.AppAdminClient.YARN_APP_ADMIN_CLIENT_PREFIX;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEPENDENCY_TARBALL_PATH;
 import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_SUCCESS;
+import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_UNAUTHORIZED;
 import static org.mockito.Mockito.spy;
 
 public class TestServiceCLI {
   private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
       .class);
 
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
   private Configuration conf = new YarnConfiguration();
-  private File basedir;
   private SliderFileSystem fs;
-  private String basedirProp;
   private ApplicationCLI cli;
+  private File basedir;
+  private String basedirProp;
+  private File dependencyTarGzBaseDir;
+  private Path dependencyTarGz;
+  private String dependencyTarGzProp;
+  private String yarnAdminNoneAclProp;
+  private String dfsAdminAclProp;
 
   private void createCLI() {
     cli = new ApplicationCLI();
@@ -67,12 +86,17 @@ public class TestServiceCLI {
     cli.setConf(conf);
   }
 
+  private int runCLI(String[] args) throws Exception {
+    LOG.info("running CLI: yarn {}", Arrays.asList(args));
+    return ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+  }
+
   private void buildApp(String serviceName, String appDef) throws Throwable {
     String[] args = {"app",
         "-D", basedirProp, "-save", serviceName,
         ExampleAppJson.resourceName(appDef),
         "-appTypes", DUMMY_APP_TYPE};
-    ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
   }
 
   private void buildApp(String serviceName, String appDef,
@@ -83,7 +107,13 @@ public class TestServiceCLI {
         "-appTypes", DUMMY_APP_TYPE,
         "-updateLifetime", lifetime,
         "-changeQueue", queue};
-    ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
+  }
+
+  private static Path getDependencyTarGz(File dir) {
+    return new Path(new File(dir, YarnServiceConstants
+        .DEPENDENCY_TAR_GZ_FILE_NAME + YarnServiceConstants
+        .DEPENDENCY_TAR_GZ_FILE_EXT).getAbsolutePath());
   }
 
   @Before
@@ -91,12 +121,22 @@ public class TestServiceCLI {
     basedir = new File("target", "apps");
     basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
     conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
+    dependencyTarGzBaseDir = tmpFolder.getRoot();
+    dependencyTarGz = getDependencyTarGz(dependencyTarGzBaseDir);
+    dependencyTarGzProp = DEPENDENCY_TARBALL_PATH + "=" + dependencyTarGz
+        .toString();
+    conf.set(DEPENDENCY_TARBALL_PATH, dependencyTarGz.toString());
     fs = new SliderFileSystem(conf);
     if (basedir.exists()) {
       FileUtils.deleteDirectory(basedir);
     } else {
       basedir.mkdirs();
     }
+    yarnAdminNoneAclProp = YarnConfiguration.YARN_ADMIN_ACL + "=none";
+    dfsAdminAclProp = DFSConfigKeys.DFS_ADMIN + "=" +
+        UserGroupInformation.getCurrentUser();
+    System.setProperty(YarnServiceConstants.PROPERTY_LIB_DIR, basedir
+        .getAbsolutePath());
     createCLI();
   }
 
@@ -108,7 +148,7 @@ public class TestServiceCLI {
     cli.stop();
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testFlexComponents() throws Throwable {
     // currently can only test building apps, since that is the only
     // operation that doesn't require an RM
@@ -122,7 +162,7 @@ public class TestServiceCLI {
     checkApp(serviceName, "master", 1L, 1000L, "qname");
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testInitiateServiceUpgrade() throws Exception {
     String[] args = {"app", "-upgrade", "app-1",
         "-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON),
@@ -131,7 +171,7 @@ public class TestServiceCLI {
     Assert.assertEquals(result, 0);
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testInitiateAutoFinalizeServiceUpgrade() throws Exception {
     String[] args =  {"app", "-upgrade", "app-1",
         "-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON),
@@ -141,7 +181,7 @@ public class TestServiceCLI {
     Assert.assertEquals(result, 0);
   }
 
-  @Test
+  @Test (timeout = 180000)
   public void testUpgradeInstances() throws Exception {
     conf.set(YARN_APP_ADMIN_CLIENT_PREFIX + DUMMY_APP_TYPE,
         DummyServiceClient.class.getName());
@@ -153,6 +193,68 @@ public class TestServiceCLI {
     Assert.assertEquals(result, 0);
   }
 
+  @Test (timeout = 180000)
+  public void testEnableFastLaunch() throws Exception {
+    fs.getFileSystem().create(new Path(basedir.getAbsolutePath(), "test.jar"))
+        .close();
+
+    Path defaultPath = new Path(dependencyTarGz.toString());
+    Assert.assertFalse("Dependency tarball should not exist before the test",
+        fs.isFile(defaultPath));
+    String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
+    Assert.assertTrue("Dependency tarball did not exist after the test",
+        fs.isFile(defaultPath));
+
+    File secondBaseDir = new File(dependencyTarGzBaseDir, "2");
+    Path secondTarGz = getDependencyTarGz(secondBaseDir);
+    Assert.assertFalse("Dependency tarball should not exist before the test",
+        fs.isFile(secondTarGz));
+    String[] args2 = {"app", "-D", yarnAdminNoneAclProp, "-D",
+        dfsAdminAclProp, "-D", dependencyTarGzProp, "-enableFastLaunch",
+        secondBaseDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_SUCCESS, runCLI(args2));
+    Assert.assertTrue("Dependency tarball did not exist after the test",
+        fs.isFile(secondTarGz));
+  }
+
+  @Test (timeout = 180000)
+  public void testEnableFastLaunchUserPermissions() throws Exception {
+    String[] args = {"app", "-D", yarnAdminNoneAclProp, "-D",
+        dependencyTarGzProp, "-enableFastLaunch", "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args));
+  }
+
+  @Test (timeout = 180000)
+  public void testEnableFastLaunchFilePermissions() throws Exception {
+    File badDir = new File(dependencyTarGzBaseDir, "bad");
+    badDir.mkdir();
+    fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
+        new FsPermission("751"));
+
+    String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args));
+
+    badDir = new File(badDir, "child");
+    badDir.mkdir();
+    fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
+        new FsPermission("755"));
+
+    String[] args2 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args2));
+
+    badDir = new File(dependencyTarGzBaseDir, "badx");
+    badDir.mkdir();
+    fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
+        new FsPermission("754"));
+
+    String[] args3 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
+        badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
+    Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args3));
+  }
 
   private void checkApp(String serviceName, String compName, long count, Long
       lifetime, String queue) throws IOException {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md

@@ -100,7 +100,7 @@ System-wide service AM properties can only be configured in the cluster `yarn-si
 
 | System-Level Config Name | Description |
 | ------------ | ------------- |
-|yarn.service.framework.path | HDFS parent directory where the service AM dependency tarball can be found.|
+|yarn.service.framework.path | HDFS path of the service AM dependency tarball. When no file exists at this location, AM dependencies will be uploaded by the RM the first time a service is started or launched. If the RM user does not have permission to upload the file to this location or the location is not world readable, the AM dependency jars will be uploaded each time a service is started or launched. If unspecified, value will be assumed to be /yarn-services/${hadoop.version}/service-dep.tar.gz.|
 |yarn.service.base.path | HDFS parent directory where service artifacts will be stored (default ${user_home_dir}/.yarn/).
 |yarn.service.client-am.retry.max-wait-ms | Max retry time in milliseconds for the service client to talk to the service AM (default 900000, i.e. 15 minutes).|
 |yarn.service.client-am.retry-interval-ms | Retry interval in milliseconds for the service client to talk to the service AM (default 2000, i.e. 2 seconds).|