YARN-6655. Validate yarn native services application submission side to ensure that the hostname should be less than 63 characters. Contributed by Billie Rinaldi

This commit is contained in:
Jian He 2017-06-29 13:11:06 -07:00
parent e86d828548
commit 59a9b4f438
10 changed files with 155 additions and 37 deletions

View File

@ -153,7 +153,7 @@ definitions:
properties:
name:
type: string
description: A unique application name.
description: A unique application name. If Registry DNS is enabled, the max length is 63 characters.
id:
type: string
description: A unique application id.
@ -255,7 +255,7 @@ definitions:
properties:
name:
type: string
description: Name of the application component (mandatory).
description: Name of the application component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
dependencies:
type: array
items:

View File

@ -637,7 +637,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionBuild(Application application) throws YarnException,
IOException {
Path appDir = checkAppNotExistOnHdfs(application);
ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
ServiceApiUtil.validateAndResolveApplication(application,
sliderFileSystem, getConfig());
persistApp(appDir, application);
deployedClusterName = application.getName();
return EXIT_SUCCESS;
@ -647,7 +648,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throws IOException, YarnException {
String appName = application.getName();
validateClusterName(appName);
ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
ServiceApiUtil.validateAndResolveApplication(application,
sliderFileSystem, getConfig());
verifyNoLiveApp(appName, "Create");
Path appDir = checkAppNotExistOnHdfs(application);
@ -1778,7 +1780,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path appDir = checkAppExistOnHdfs(appName);
Application application = ServiceApiUtil.loadApplication(sliderFileSystem,
appName);
ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
ServiceApiUtil.validateAndResolveApplication(application,
sliderFileSystem, getConfig());
// see if it is actually running and bail out;
verifyNoLiveApp(appName, "Thaw");
ApplicationId appId = submitApp(application);

View File

@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;

View File

@ -21,8 +21,13 @@ public interface RestApiErrorMessages {
String ERROR_APPLICATION_NAME_INVALID =
"Application name is either empty or not provided";
String ERROR_APPLICATION_NAME_INVALID_FORMAT =
"Application name %s is not valid - only lower case letters, digits,"
+ " underscore and hyphen are allowed";
"Application name %s is not valid - only lower case letters, digits, " +
"underscore and hyphen are allowed, and the name must be no more " +
"than 63 characters";
String ERROR_COMPONENT_NAME_INVALID =
"Component name must be no more than %s characters: %s";
String ERROR_USER_NAME_INVALID =
"User name must be no more than 63 characters";
String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";

View File

@ -22,6 +22,8 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Component;
@ -60,12 +62,22 @@ public class ServiceApiUtil {
@VisibleForTesting
public static void validateAndResolveApplication(Application application,
SliderFileSystem fs) throws IOException {
SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws
IOException {
boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
RegistryConstants.DEFAULT_DNS_ENABLED);
if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
.MAX_FQDN_LABEL_LENGTH) {
throw new IllegalArgumentException(RestApiErrorMessages
.ERROR_USER_NAME_INVALID);
}
if (StringUtils.isEmpty(application.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
}
if (!SliderUtils.isClusternameValid(application.getName())) {
if (!SliderUtils.isClusternameValid(application.getName()) || (dnsEnabled
&& application.getName().length() > RegistryConstants
.MAX_FQDN_LABEL_LENGTH)) {
throw new IllegalArgumentException(String.format(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT,
application.getName()));
@ -108,6 +120,14 @@ public class ServiceApiUtil {
List<Component> componentsToRemove = new ArrayList<>();
List<Component> componentsToAdd = new ArrayList<>();
for (Component comp : application.getComponents()) {
int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
if (comp.getUniqueComponentSupport()) {
maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
}
if (dnsEnabled && comp.getName().length() > maxCompLength) {
throw new IllegalArgumentException(String.format(RestApiErrorMessages
.ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
}
if (componentNames.contains(comp.getName())) {
throw new IllegalArgumentException("Component name collision: " +
comp.getName());

View File

@ -20,6 +20,7 @@ package org.apache.slider.core.conf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.resource.ConfigFile.TypeEnum;
@ -95,7 +96,8 @@ public class TestConfigurationResolve extends Assert {
expect(sfs.buildClusterDirPath(anyObject())).andReturn(
new Path("cluster_dir_path")).anyTimes();
replay(sfs, mockFs);
ServiceApiUtil.validateAndResolveApplication(orig, sfs);
ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
YarnConfiguration());
global = orig.getConfiguration();
LOG.info("global = {}", global);
@ -179,7 +181,8 @@ public class TestConfigurationResolve extends Assert {
new Path("cluster_dir_path")).anyTimes();
replay(sfs, mockFs);
Application ext = ExampleAppJson.loadResource(APP_JSON);
ServiceApiUtil.validateAndResolveApplication(ext, sfs);
ServiceApiUtil.validateAndResolveApplication(ext, sfs, new
YarnConfiguration());
reset(sfs, mockFs);
// perform the resolution on original application
@ -192,7 +195,8 @@ public class TestConfigurationResolve extends Assert {
.anyTimes();
replay(sfs, mockFs, jsonSerDeser);
ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
ServiceApiUtil.validateAndResolveApplication(orig, sfs);
ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
YarnConfiguration());
global = orig.getConfiguration();
assertEquals(0, global.getProperties().size());

View File

@ -20,6 +20,7 @@ package org.apache.slider.core.conf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.util.ServiceApiUtil;
@ -71,7 +72,8 @@ public class TestExampleAppJson extends Assert {
new Path("cluster_dir_path")).anyTimes();
replay(sfs, mockFs);
ServiceApiUtil.validateAndResolveApplication(application, sfs);
ServiceApiUtil.validateAndResolveApplication(application, sfs,
new YarnConfiguration());
} catch (Exception e) {
throw new Exception("exception loading " + resource + ":" + e.toString());
}

View File

@ -133,7 +133,7 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
AppStateBindingInfo binding = new AppStateBindingInfo();
binding.application = buildApplication();
ServiceApiUtil.validateAndResolveApplication(binding.application,
sliderFileSystem);
sliderFileSystem, SliderUtils.createConfiguration());
//binding.roles = new ArrayList<>(factory.ROLES);
binding.fs = fs;
binding.historyPath = historyPath;

View File

@ -19,6 +19,8 @@ package org.apache.slider.utils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Component;
@ -29,6 +31,7 @@ import org.apache.slider.util.RestApiConstants;
import org.apache.slider.util.RestApiErrorMessages;
import org.apache.slider.util.ServiceApiUtil;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,26 +64,42 @@ public class TestServiceApiUtil {
private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
"exception: ";
private static final String LEN_64_STR =
"abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
private static final YarnConfiguration CONF_DEFAULT_DNS = new
YarnConfiguration();
private static final YarnConfiguration CONF_DNS_ENABLED = new
YarnConfiguration();
@BeforeClass
public static void init() {
CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
}
@Test(timeout = 90000)
public void testResourceValidation() throws Exception {
assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
.length());
SliderFileSystem sfs = initMock(null);
Application app = new Application();
// no name
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no name");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
}
// bad format name
String[] badNames = {"4finance", "Finance", "finance@home"};
String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
for (String badName : badNames) {
app.setName(badName);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
} catch (IllegalArgumentException e) {
assertEquals(String.format(
@ -89,9 +108,20 @@ public class TestServiceApiUtil {
}
// launch command not specified
app.setName("finance_home");
app.setName(LEN_64_STR);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
} catch (IllegalArgumentException e) {
assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
e.getMessage());
}
// launch command not specified
app.setName(LEN_64_STR.substring(0, RegistryConstants
.MAX_FQDN_LABEL_LENGTH));
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
} catch (IllegalArgumentException e) {
assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
@ -101,7 +131,7 @@ public class TestServiceApiUtil {
// resource not specified
app.setLaunchCommand("sleep 3600");
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no resource");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
@ -113,7 +143,7 @@ public class TestServiceApiUtil {
Resource res = new Resource();
app.setResource(res);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no memory");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
@ -125,7 +155,7 @@ public class TestServiceApiUtil {
res.setMemory("100mb");
res.setCpus(-2);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(
EXCEPTION_PREFIX + "application with invalid no of cpus");
} catch (IllegalArgumentException e) {
@ -137,7 +167,7 @@ public class TestServiceApiUtil {
// number of containers not specified
res.setCpus(2);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no container count");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage()
@ -147,7 +177,7 @@ public class TestServiceApiUtil {
// specifying profile along with cpus/memory raises exception
res.setProfile("hbase_finance_large");
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX
+ "application with resource profile along with cpus/memory");
} catch (IllegalArgumentException e) {
@ -162,7 +192,7 @@ public class TestServiceApiUtil {
res.setCpus(null);
res.setMemory(null);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
@ -176,7 +206,7 @@ public class TestServiceApiUtil {
// null number of containers
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "null number of containers");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage()
@ -186,7 +216,7 @@ public class TestServiceApiUtil {
// negative number of containers
app.setNumberOfContainers(-1L);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage()
@ -196,7 +226,7 @@ public class TestServiceApiUtil {
// everything valid here
app.setNumberOfContainers(5L);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
LOG.error("application attributes specified should be valid here", e);
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
@ -218,7 +248,7 @@ public class TestServiceApiUtil {
Artifact artifact = new Artifact();
app.setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@ -227,7 +257,7 @@ public class TestServiceApiUtil {
// no artifact id fails with APPLICATION type
artifact.setType(Artifact.TypeEnum.APPLICATION);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@ -236,7 +266,7 @@ public class TestServiceApiUtil {
// no artifact id fails with TARBALL type
artifact.setType(Artifact.TypeEnum.TARBALL);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@ -246,7 +276,7 @@ public class TestServiceApiUtil {
artifact.setType(Artifact.TypeEnum.DOCKER);
artifact.setId("docker.io/centos:centos7");
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
LOG.error("application attributes specified should be valid here", e);
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
@ -314,7 +344,7 @@ public class TestServiceApiUtil {
app.setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -333,7 +363,7 @@ public class TestServiceApiUtil {
// duplicate component name fails
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with component collision");
} catch (IllegalArgumentException e) {
assertEquals("Component name collision: " + compName, e.getMessage());
@ -353,7 +383,7 @@ public class TestServiceApiUtil {
// duplicate component name okay in the case of APPLICATION component
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -371,7 +401,7 @@ public class TestServiceApiUtil {
app.setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -384,7 +414,7 @@ public class TestServiceApiUtil {
app.getComponent("comp2").setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -441,4 +471,52 @@ public class TestServiceApiUtil {
.getMessage());
}
}
@Test
public void testInvalidComponent() throws IOException {
SliderFileSystem sfs = initMock(null);
testComponent(sfs, false);
testComponent(sfs, true);
}
private static void testComponent(SliderFileSystem sfs, boolean unique)
throws IOException {
int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
if (unique) {
assertEquals(19, Long.toString(Long.MAX_VALUE).length());
maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
}
String compName = LEN_64_STR.substring(0, maxLen + 1);
Application app = createValidApplication(null);
app.addComponent(createValidComponent(compName).uniqueComponentSupport(
unique));
// invalid component name fails if dns is enabled
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with invalid component name");
} catch (IllegalArgumentException e) {
assertEquals(String.format(RestApiErrorMessages
.ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
}
// does not fail if dns is disabled
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
compName = LEN_64_STR.substring(0, maxLen);
app = createValidApplication(null);
app.addComponent(createValidComponent(compName).uniqueComponentSupport(
unique));
// does not fail
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
}
}

View File

@ -77,6 +77,11 @@ public interface RegistryConstants {
*/
String KEY_DNS_DOMAIN = DNS_PREFIX + "domain-name";
/**
* Max length of a label (node delimited by a dot in the FQDN).
*/
int MAX_FQDN_LABEL_LENGTH = 63;
/**
* DNS bind address.
*/