From ce74e64363abc64561263be70a923ab3e67f043f Mon Sep 17 00:00:00 2001 From: Billie Rinaldi Date: Thu, 21 Sep 2017 10:18:42 -0700 Subject: [PATCH] YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He --- .../hadoop/yarn/service/ServiceScheduler.java | 6 +----- .../yarn/service/client/ServiceClient.java | 3 +++ .../AbstractClusterBuildingActionArgs.java | 2 +- .../yarn/service/client/params/Arguments.java | 1 + .../component/instance/ComponentInstance.java | 10 +++++----- .../provider/AbstractProviderService.java | 18 +++++++++++------- .../client/binding/RegistryTypeUtils.java | 3 ++- .../dns/ApplicationServiceRecordProcessor.java | 15 ++++++++++++++- .../server/dns/BaseServiceRecordProcessor.java | 8 ++++---- .../registry/server/dns/TestRegistryDNS.java | 12 ++++++++---- .../runtime/DockerLinuxContainerRuntime.java | 4 ++-- .../src/site/markdown/YarnCommands.md | 1 + 12 files changed, 53 insertions(+), 30 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 7b809b9bbe3..ec5f3ed1519 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService { attemptId.getApplicationId().toString()); serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.APPLICATION); - serviceRecord.description = "Yarn Service Master"; - - serviceRecord.addExternalEndpoint(RegistryTypeUtils - .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc", - context.clientAMService.getBindAddress())); + serviceRecord.description = "YarnServiceMaster"; // set any provided attributes setUserProvidedServiceRecordAttributes(service.getConfiguration(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 6890aefdda1..a3a9fd0ae5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService if (!StringUtils.isEmpty(args.getServiceName())) { service.setName(args.getServiceName()); } + if (!StringUtils.isEmpty(args.queue)) { + service.setQueue(args.queue); + } return service; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java index 434c6d5938e..4ecbe9c6423 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java @@ -37,7 +37,7 @@ public abstract class AbstractClusterBuildingActionArgs } @Parameter(names = { - ARG_QUEUE }, description = "Queue to submit the service") + ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service") public String queue; @Parameter(names = { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java index 9672621a110..67571e2b45a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java @@ -77,6 +77,7 @@ public interface Arguments { String ARG_PATH = "--path"; String ARG_PRINCIPAL = "--principal"; String ARG_QUEUE = "--queue"; + String ARG_SHORT_QUEUE = "-q"; String ARG_LIFETIME = "--lifetime"; String ARG_RESOURCE = "--resource"; String ARG_RESOURCE_MANAGER = "--rm"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java index 7d6525b567a..3c1e48ff1ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java @@ -59,6 +59,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import static org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes.*; import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER; import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE; import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*; @@ -356,12 +357,11 @@ public class ComponentInstance implements EventHandler, YarnRegistryViewForProviders yarnRegistry, ContainerStatus status) { ServiceRecord record = new ServiceRecord(); String containerId = status.getContainerId().toString(); - record.set(YarnRegistryAttributes.YARN_ID, containerId); + record.set(YARN_ID, containerId); record.description = getCompInstanceName(); - record.set(YarnRegistryAttributes.YARN_PERSISTENCE, - PersistencePolicies.CONTAINER); - record.set("yarn:ip", status.getIPs()); - record.set("yarn:hostname", status.getHost()); + record.set(YARN_PERSISTENCE, PersistencePolicies.CONTAINER); + record.set(YARN_IP, status.getIPs().get(0)); + record.set(YARN_HOSTNAME, status.getHost()); try { yarnRegistry .putComponent(RegistryPathUtils.encodeYarnID(containerId), record); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java index 8d607abf62d..6ffb84defb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.service.provider; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.service.api.records.Service; @@ -91,13 +92,16 @@ public abstract class AbstractProviderService implements ProviderService, component, tokensForSubstitution, instance, context); // substitute launch command - String launchCommand = ProviderUtils - .substituteStrWithTokens(component.getLaunchCommand(), - tokensForSubstitution); - CommandLineBuilder operation = new CommandLineBuilder(); - operation.add(launchCommand); - operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); - launcher.addCommand(operation.build()); + String launchCommand = component.getLaunchCommand(); + // docker container may have empty commands + if (!StringUtils.isEmpty(launchCommand)) { + launchCommand = ProviderUtils + .substituteStrWithTokens(launchCommand, tokensForSubstitution); + CommandLineBuilder operation = new CommandLineBuilder(); + operation.add(launchCommand); + operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); + launcher.addCommand(operation.build()); + } // By default retry forever every 30 seconds launcher.setRetryContext(YarnServiceConf diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java index ec59d5985a0..05df3255e3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; -import static org.apache.hadoop.registry.client.types.AddressTypes.*; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; import org.apache.hadoop.registry.client.types.ServiceRecord; @@ -36,6 +35,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.hadoop.registry.client.types.AddressTypes.*; + /** * Static methods to work with registry types —primarily endpoints and the * list representation of addresses. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java index e6a1b5b564e..0b5f724f869 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java @@ -18,6 +18,8 @@ package org.apache.hadoop.registry.server.dns; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xbill.DNS.Name; import org.xbill.DNS.Type; @@ -32,7 +34,8 @@ import java.util.List; */ public class ApplicationServiceRecordProcessor extends BaseServiceRecordProcessor { - + private static final Logger LOG = + LoggerFactory.getLogger(ApplicationServiceRecordProcessor.class); /** * Create an application service record processor. * @@ -57,6 +60,10 @@ public class ApplicationServiceRecordProcessor extends */ @Override public void initTypeToInfoMapping(ServiceRecord serviceRecord) throws Exception { + if (serviceRecord.external.isEmpty()) { + LOG.info(serviceRecord.description + ": No external endpoints defined."); + return; + } for (int type : getRecordTypes()) { switch (type) { case Type.A: @@ -309,6 +316,9 @@ public class ApplicationServiceRecordProcessor extends throws Exception { this.setNames(new Name[] {getServiceName()}); List endpoints = serviceRecord.external; + if (endpoints.isEmpty()) { + return; + } // TODO: do we need a "hostname" attribute for an application record or // can we rely on the first endpoint record. this.setTarget(InetAddress.getByName( @@ -342,6 +352,9 @@ public class ApplicationServiceRecordProcessor extends @Override protected void init(ServiceRecord serviceRecord) throws Exception { super.init(serviceRecord); + if (getTarget() == null) { + return; + } try { this.setTarget(getIpv6Address(getTarget())); } catch (UnknownHostException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java index 2fe3a6c9066..fd5c74f6490 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java @@ -52,8 +52,8 @@ public abstract class BaseServiceRecordProcessor private String domain; private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?"); - private static final String SLIDER_API_PREFIX = - "classpath:org.apache.slider."; + private static final String YARN_SERVICE_API_PREFIX = + "classpath:org.apache.hadoop.yarn.service."; private static final String HTTP_API_TYPE = "http://"; /** @@ -425,8 +425,8 @@ public abstract class BaseServiceRecordProcessor */ protected String getDNSApiFragment(String api) { String dnsApi = null; - if (api.startsWith(SLIDER_API_PREFIX)) { - dnsApi = api.substring(SLIDER_API_PREFIX.length()); + if (api.startsWith(YARN_SERVICE_API_PREFIX)) { + dnsApi = api.substring(YARN_SERVICE_API_PREFIX.length()); } else if (api.startsWith(HTTP_API_TYPE)) { dnsApi = "http"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java index cc839cc505b..ac8d9392d5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java @@ -69,7 +69,8 @@ public class TestRegistryDNS extends Assert { + " \"type\" : \"JSONServiceRecord\",\n" + " \"description\" : \"Slider Application Master\",\n" + " \"external\" : [ {\n" - + " \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.appmaster.ipc" + + "\",\n" + " \"addressType\" : \"host/port\",\n" + " \"protocolType\" : \"hadoop/IPC\",\n" + " \"addresses\" : [ {\n" @@ -84,7 +85,8 @@ public class TestRegistryDNS extends Assert { + " \"uri\" : \"http://192.168.1.5:1027\"\n" + " } ]\n" + " }, {\n" - + " \"api\" : \"classpath:org.apache.slider.management\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.management\"" + + ",\n" + " \"addressType\" : \"uri\",\n" + " \"protocolType\" : \"REST\",\n" + " \"addresses\" : [ {\n" @@ -92,14 +94,16 @@ public class TestRegistryDNS extends Assert { + " } ]\n" + " } ],\n" + " \"internal\" : [ {\n" - + " \"api\" : \"classpath:org.apache.slider.agents.secure\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.secure" + + "\",\n" + " \"addressType\" : \"uri\",\n" + " \"protocolType\" : \"REST\",\n" + " \"addresses\" : [ {\n" + " \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n" + " } ]\n" + " }, {\n" - + " \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.oneway" + + "\",\n" + " \"addressType\" : \"uri\",\n" + " \"protocolType\" : \"REST\",\n" + " \"addresses\" : [ {\n" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index f945a9b8a27..75a28e648b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -812,13 +812,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { .executePrivilegedOperation(null, privOp, null, null, true, false); LOG.info("Docker inspect output for " + containerId + ": " + output); + // strip off quotes if any + output = output.replaceAll("['\"]", ""); int index = output.lastIndexOf(','); if (index == -1) { LOG.error("Incorrect format for ip and host"); return null; } - // strip off quotes if any - output = output.replaceAll("['\"]", ""); String ips = output.substring(0, index).trim(); String host = output.substring(index+1).trim(); String[] ipAndHost = new String[2]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index caeaeb532e0..51e27cc1cf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -92,6 +92,7 @@ Usage `yarn service [sub-command] [service-name] [options]` Options: --file,-f The local path to the service definition file. + --queue,-q The queue to which the service is submitted. --example,-e The name of the example service such as: Sleeper A simple service that launches a few non-docker sleep containers on YARN. ```