YARN-8734. Readiness check for remote service belonging to the same user. Contributed by Eric Yang

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml

@@ -254,6 +254,11 @@ definitions:
       docker_client_config:
         type: string
         description: URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json).
+      dependencies:
+        type: array
+        items:
+          type: string
+        description: An array of services which should be in STABLE state, before this service can be started.
   ResourceInformation:
     description:
       ResourceInformation determines unit/value of resource types in addition to memory and vcores. It will be part of Resource object.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java

@@ -170,7 +170,8 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
         } else {
           serviceManager.setServiceState(ServiceState.UPGRADING);
         }
-
+        ServiceApiUtil.checkServiceDependencySatisified(serviceManager
+            .getServiceSpec());
         return State.UPGRADING;
       } catch (Throwable e) {
         LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java

@@ -389,6 +389,8 @@ public class ServiceScheduler extends CompositeService {
     // Since AM has been started and registered, the service is in STARTED state
     app.setState(ServiceState.STARTED);
 
+    ServiceApiUtil.checkServiceDependencySatisified(context.service);
+
     // recover components based on containers sent from RM
     recoverComponents(response);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java

@@ -73,6 +73,7 @@ public class Service extends BaseResource {
   private String version = null;
   private String description = null;
   private String dockerClientConfig = null;
+  private List<String> dependencies = new ArrayList<String>();
 
   /**
    * A unique service name.
@@ -352,6 +353,18 @@ public class Service extends BaseResource {
     this.queue = queue;
   }
 
+  @ApiModelProperty(example = "null", value = "A list of dependent services.")
+  @XmlElement(name = "dependencies")
+  @JsonProperty("dependencies")
+  public List<String> getDependencies() {
+    return dependencies;
+  }
+
+  public void setDependencies(List<String>
+      dependencies) {
+    this.dependencies = dependencies;
+  }
+
   public Service kerberosPrincipal(KerberosPrincipal kerberosPrincipal) {
     this.kerberosPrincipal = kerberosPrincipal;
     return this;
@@ -437,6 +450,8 @@ public class Service extends BaseResource {
         .append(toIndentedString(kerberosPrincipal)).append("\n");
     sb.append("    dockerClientConfig: ")
         .append(toIndentedString(dockerClientConfig)).append("\n");
+    sb.append("    dependencies: ")
+        .append(toIndentedString(dependencies)).append("\n");
     sb.append("}");
     return sb.toString();
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java

@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.service.api.records.ComponentState;
 import org.apache.hadoop.yarn.service.api.records.Container;
 import org.apache.hadoop.yarn.service.api.records.ContainerState;
 import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
 import org.apache.hadoop.yarn.service.api.records.Artifact;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Configuration;
@@ -705,4 +707,45 @@ public class ServiceApiUtil {
     }
     return components;
   }
+
+  private static boolean serviceDependencySatisfied(Service service) {
+    boolean result = true;
+    try {
+      List<String> dependencies = service
+          .getDependencies();
+      org.apache.hadoop.conf.Configuration conf =
+          new org.apache.hadoop.conf.Configuration();
+      if (dependencies != null && dependencies.size() > 0) {
+        ServiceClient sc = new ServiceClient();
+        sc.init(conf);
+        sc.start();
+        for (String dependent : dependencies) {
+          Service dependentService = sc.getStatus(dependent);
+          if (dependentService.getState() == null ||
+              !dependentService.getState().equals(ServiceState.STABLE)) {
+            result = false;
+            LOG.info("Service dependency is not satisfied for " +
+                "service: {} state: {}", dependent,
+                dependentService.getState());
+          }
+        }
+        sc.close();
+      }
+    } catch (IOException | YarnException e) {
+      LOG.warn("Caught exception: ", e);
+      LOG.info("Service dependency is not satisified.");
+      result = false;
+    }
+    return result;
+  }
+
+  public static void checkServiceDependencySatisified(Service service) {
+    while (!serviceDependencySatisfied(service)) {
+      try {
+        LOG.info("Waiting for service dependencies.");
+        Thread.sleep(15000L);
+      } catch (InterruptedException e) {
+      }
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java

@@ -438,6 +438,8 @@ public class TestYarnNativeServices extends ServiceTestUtils {
     // wait for upgrade to complete
     waitForServiceToBeStable(client, service);
     Service active = client.getStatus(service.getName());
+    Assert.assertEquals("version mismatch", service.getVersion(),
+        active.getVersion());
     Assert.assertEquals("component not stable", ComponentState.STABLE,
         active.getComponent(component.getName()).getState());
     Assert.assertEquals("compa does not have new env", "val1",

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.PlacementScope;
 import org.apache.hadoop.yarn.service.api.records.PlacementType;
 import org.apache.hadoop.yarn.service.api.records.Resource;
 import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -733,6 +734,44 @@ public class TestServiceApiUtil extends ServiceTestUtils {
     }
   }
 
+  @Test(timeout = 1500)
+  public void testNoServiceDependencies() {
+    Service service = createExampleApplication();
+    Component compa = createComponent("compa");
+    Component compb = createComponent("compb");
+    service.addComponent(compa);
+    service.addComponent(compb);
+    List<String> dependencies = new ArrayList<String>();
+    service.setDependencies(dependencies);
+    ServiceApiUtil.checkServiceDependencySatisified(service);
+  }
+
+  @Test
+  public void testServiceDependencies() {
+    Thread thread = new Thread() {
+      @Override
+      public void run() {
+        Service service = createExampleApplication();
+        Component compa = createComponent("compa");
+        Component compb = createComponent("compb");
+        service.addComponent(compa);
+        service.addComponent(compb);
+        List<String> dependencies = new ArrayList<String>();
+        dependencies.add("abc");
+        service.setDependencies(dependencies);
+        Service dependent = createExampleApplication();
+        dependent.setState(ServiceState.STOPPED);
+        ServiceApiUtil.checkServiceDependencySatisified(service);
+      }
+    };
+    thread.start();
+    try {
+      Thread.sleep(1000);
+    } catch (InterruptedException e) {
+    }
+    Assert.assertTrue(thread.isAlive());
+  }
+
   public static Service createExampleApplication() {
 
     Service exampleApp = new Service();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md

@@ -406,7 +406,7 @@ a service resource has the following attributes.
 |queue|The YARN queue that this service should be submitted to.|false|string||
 |kerberos_principal|The principal info of the user who launches the service|false|KerberosPrincipal||
 |docker_client_config|URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json)|false|string||
-
+|dependencies|A list of service names that this service depends on.| false | string array ||
 
 ### ServiceState