YARN-1766. Fixed a bug in ResourceManager to use configuration loaded from the configuration-provider when booting up. Contributed by Xuan Gong.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1574252 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli
parent
1da076fd13
commit
82e8c8bd71
|
|
@ -392,6 +392,9 @@ Release 2.4.0 - UNRELEASED
|
|||
secondary-filters param into the JSON-compatible object. (Billie Rinaldi via
|
||||
zjshen)
|
||||
|
||||
YARN-1766. Fixed a bug in ResourceManager to use configuration loaded from the
|
||||
configuration-provider when booting up. (Xuan Gong via vinodkv)
|
||||
|
||||
|
||||
Release 2.3.1 - UNRELEASED
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
import org.apache.hadoop.security.Groups;
|
||||
import org.apache.hadoop.security.SecurityUtil;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.ProxyUsers;
|
||||
import org.apache.hadoop.service.AbstractService;
|
||||
import org.apache.hadoop.service.CompositeService;
|
||||
import org.apache.hadoop.service.Service;
|
||||
|
|
@ -181,7 +182,6 @@ protected static void setClusterTimeStamp(long timestamp) {
|
|||
|
||||
@Override
|
||||
protected void serviceInit(Configuration conf) throws Exception {
|
||||
validateConfigs(conf);
|
||||
this.conf = conf;
|
||||
this.rmContext = new RMContextImpl();
|
||||
|
||||
|
|
@ -190,13 +190,6 @@ protected void serviceInit(Configuration conf) throws Exception {
|
|||
this.configurationProvider.init(this.conf);
|
||||
rmContext.setConfigurationProvider(configurationProvider);
|
||||
|
||||
// load yarn-site.xml
|
||||
InputStream yarnSiteXMLInputStream =
|
||||
this.configurationProvider.getConfigurationInputStream(this.conf,
|
||||
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
|
||||
if (yarnSiteXMLInputStream != null) {
|
||||
this.conf.addResource(yarnSiteXMLInputStream);
|
||||
}
|
||||
// load core-site.xml
|
||||
InputStream coreSiteXMLInputStream =
|
||||
this.configurationProvider.getConfigurationInputStream(this.conf,
|
||||
|
|
@ -209,6 +202,19 @@ protected void serviceInit(Configuration conf) throws Exception {
|
|||
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
|
||||
.refresh();
|
||||
|
||||
// Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
|
||||
ProxyUsers.refreshSuperUserGroupsConfiguration(this.conf);
|
||||
|
||||
// load yarn-site.xml
|
||||
InputStream yarnSiteXMLInputStream =
|
||||
this.configurationProvider.getConfigurationInputStream(this.conf,
|
||||
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
|
||||
if (yarnSiteXMLInputStream != null) {
|
||||
this.conf.addResource(yarnSiteXMLInputStream);
|
||||
}
|
||||
|
||||
validateConfigs(this.conf);
|
||||
|
||||
// register the handlers for all AlwaysOn services using setupDispatcher().
|
||||
rmDispatcher = setupDispatcher();
|
||||
addIfService(rmDispatcher);
|
||||
|
|
@ -218,15 +224,15 @@ protected void serviceInit(Configuration conf) throws Exception {
|
|||
addService(adminService);
|
||||
rmContext.setRMAdminService(adminService);
|
||||
|
||||
this.rmContext.setHAEnabled(HAUtil.isHAEnabled(conf));
|
||||
this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf));
|
||||
if (this.rmContext.isHAEnabled()) {
|
||||
HAUtil.verifyAndSetConfiguration(conf);
|
||||
HAUtil.verifyAndSetConfiguration(this.conf);
|
||||
}
|
||||
createAndInitActiveServices();
|
||||
|
||||
webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(conf);
|
||||
webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(this.conf);
|
||||
|
||||
super.serviceInit(conf);
|
||||
super.serviceInit(this.conf);
|
||||
}
|
||||
|
||||
protected QueueACLsManager createQueueACLsManager(ResourceScheduler scheduler,
|
||||
|
|
|
|||
|
|
@ -590,6 +590,140 @@ public void testRMStartsWithoutConfigurationFilesProvided() {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRMInitialsWithFileSystemBasedConfigurationProvider()
|
||||
throws Exception {
|
||||
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
|
||||
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
|
||||
|
||||
// upload configurations
|
||||
final File excludeHostsFile = new File(tmpDir.toString(), "excludeHosts");
|
||||
if (excludeHostsFile.exists()) {
|
||||
excludeHostsFile.delete();
|
||||
}
|
||||
if (!excludeHostsFile.createNewFile()) {
|
||||
Assert.fail("Can not create " + "excludeHosts");
|
||||
}
|
||||
PrintWriter fileWriter = new PrintWriter(excludeHostsFile);
|
||||
fileWriter.write("0.0.0.0:123");
|
||||
fileWriter.close();
|
||||
uploadToRemoteFileSystem(new Path(excludeHostsFile.getAbsolutePath()));
|
||||
|
||||
YarnConfiguration yarnConf = new YarnConfiguration();
|
||||
yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda");
|
||||
yarnConf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, this.workingPath
|
||||
+ "/excludeHosts");
|
||||
uploadConfiguration(yarnConf, "yarn-site.xml");
|
||||
|
||||
CapacitySchedulerConfiguration csConf =
|
||||
new CapacitySchedulerConfiguration();
|
||||
csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
|
||||
uploadConfiguration(csConf, "capacity-scheduler.xml");
|
||||
|
||||
String aclsString = "alice,bob users,wheel";
|
||||
Configuration newConf = new Configuration();
|
||||
newConf.set("security.applicationclient.protocol.acl", aclsString);
|
||||
uploadConfiguration(newConf, "hadoop-policy.xml");
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(
|
||||
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
|
||||
conf.set("hadoop.proxyuser.test.groups", "test_groups");
|
||||
conf.set("hadoop.proxyuser.test.hosts", "test_hosts");
|
||||
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
|
||||
MockUnixGroupsMapping.class,
|
||||
GroupMappingServiceProvider.class);
|
||||
uploadConfiguration(conf, "core-site.xml");
|
||||
|
||||
// update the groups
|
||||
MockUnixGroupsMapping.updateGroups();
|
||||
|
||||
ResourceManager resourceManager = null;
|
||||
try {
|
||||
try {
|
||||
resourceManager = new ResourceManager();
|
||||
resourceManager.init(configuration);
|
||||
resourceManager.start();
|
||||
} catch (Exception ex) {
|
||||
fail("Should not get any exceptions");
|
||||
}
|
||||
|
||||
// validate values for excludeHosts
|
||||
Set<String> excludeHosts =
|
||||
resourceManager.getRMContext().getNodesListManager()
|
||||
.getHostsReader().getExcludedHosts();
|
||||
Assert.assertTrue(excludeHosts.size() == 1);
|
||||
Assert.assertTrue(excludeHosts.contains("0.0.0.0:123"));
|
||||
|
||||
// validate values for admin-acls
|
||||
String aclStringAfter =
|
||||
resourceManager.adminService.getAccessControlList()
|
||||
.getAclString().trim();
|
||||
Assert.assertEquals(aclStringAfter, "world:anyone:rwcda");
|
||||
|
||||
// validate values for queue configuration
|
||||
CapacityScheduler cs =
|
||||
(CapacityScheduler) resourceManager.getRMContext().getScheduler();
|
||||
int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
|
||||
Assert.assertEquals(maxAppsAfter, 5000);
|
||||
|
||||
// verify service Acls for AdminService
|
||||
ServiceAuthorizationManager adminServiceServiceManager =
|
||||
resourceManager.adminService.getServer()
|
||||
.getServiceAuthorizationManager();
|
||||
verifyServiceACLsRefresh(adminServiceServiceManager,
|
||||
org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
|
||||
aclsString);
|
||||
|
||||
// verify service ACLs for ClientRMService
|
||||
ServiceAuthorizationManager clientRMServiceServiceManager =
|
||||
resourceManager.getRMContext().getClientRMService().getServer()
|
||||
.getServiceAuthorizationManager();
|
||||
verifyServiceACLsRefresh(clientRMServiceServiceManager,
|
||||
org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
|
||||
aclsString);
|
||||
|
||||
// verify service ACLs for ApplicationMasterService
|
||||
ServiceAuthorizationManager appMasterService =
|
||||
resourceManager.getRMContext().getApplicationMasterService()
|
||||
.getServer().getServiceAuthorizationManager();
|
||||
verifyServiceACLsRefresh(appMasterService,
|
||||
org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
|
||||
aclsString);
|
||||
|
||||
// verify service ACLs for ResourceTrackerService
|
||||
ServiceAuthorizationManager RTService =
|
||||
resourceManager.getRMContext().getResourceTrackerService()
|
||||
.getServer().getServiceAuthorizationManager();
|
||||
verifyServiceACLsRefresh(RTService,
|
||||
org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
|
||||
aclsString);
|
||||
|
||||
// verify ProxyUsers and ProxyHosts
|
||||
Assert.assertTrue(ProxyUsers.getProxyGroups()
|
||||
.get("hadoop.proxyuser.test.groups").size() == 1);
|
||||
Assert.assertTrue(ProxyUsers.getProxyGroups()
|
||||
.get("hadoop.proxyuser.test.groups").contains("test_groups"));
|
||||
|
||||
Assert.assertTrue(ProxyUsers.getProxyHosts()
|
||||
.get("hadoop.proxyuser.test.hosts").size() == 1);
|
||||
Assert.assertTrue(ProxyUsers.getProxyHosts()
|
||||
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
|
||||
|
||||
// verify UserToGroupsMappings
|
||||
List<String> groupAfter =
|
||||
Groups.getUserToGroupsMappingService(configuration).getGroups(
|
||||
UserGroupInformation.getCurrentUser().getUserName());
|
||||
Assert.assertTrue(groupAfter.contains("test_group_D")
|
||||
&& groupAfter.contains("test_group_E")
|
||||
&& groupAfter.contains("test_group_F") && groupAfter.size() == 3);
|
||||
} finally {
|
||||
if (resourceManager != null) {
|
||||
resourceManager.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String writeConfigurationXML(Configuration conf, String confXMLName)
|
||||
throws IOException {
|
||||
DataOutputStream output = null;
|
||||
|
|
|
|||
Loading…
Reference in New Issue