YARN-1676. Modified RM HA handling of user-to-group mappings to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1568041 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
8ffd3a91a5
commit
2b916db60b
|
@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||
|
@ -240,4 +241,18 @@ public class Groups {
|
|||
}
|
||||
return GROUPS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create new groups used to map user-to-groups with loaded configuration.
|
||||
* @param conf
|
||||
* @return the groups being used to map user-to-groups.
|
||||
*/
|
||||
@Private
|
||||
public static synchronized Groups
|
||||
getUserToGroupsMappingServiceWithLoadedConfiguration(
|
||||
Configuration conf) {
|
||||
|
||||
GROUPS = new Groups(conf);
|
||||
return GROUPS;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -184,6 +184,10 @@ Release 2.4.0 - UNRELEASED
|
|||
YARN-1345. Remove FINAL_SAVING state from YarnApplicationAttemptState
|
||||
(Zhijie Shen via jianhe)
|
||||
|
||||
YARN-1676. Modified RM HA handling of user-to-group mappings to
|
||||
be available across RM failover by making using of a remote
|
||||
configuration-provider. (Xuan Gong via vinodkv)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -381,21 +381,22 @@ public class AdminService extends CompositeService implements
|
|||
@Override
|
||||
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
|
||||
RefreshUserToGroupsMappingsRequest request)
|
||||
throws YarnException, StandbyException {
|
||||
UserGroupInformation user = checkAcls("refreshUserToGroupsMappings");
|
||||
throws YarnException, IOException {
|
||||
String argName = "refreshUserToGroupsMappings";
|
||||
UserGroupInformation user = checkAcls(argName);
|
||||
|
||||
// TODO (YARN-1459): Revisit handling user-groups on Standby RM
|
||||
if (!isRMActive()) {
|
||||
RMAuditLogger.logFailure(user.getShortUserName(),
|
||||
"refreshUserToGroupsMapping",
|
||||
RMAuditLogger.logFailure(user.getShortUserName(), argName,
|
||||
adminAcl.toString(), "AdminService",
|
||||
"ResourceManager is not active. Can not refresh user-groups.");
|
||||
throwStandbyException();
|
||||
}
|
||||
|
||||
Groups.getUserToGroupsMappingService().refresh();
|
||||
RMAuditLogger.logSuccess(user.getShortUserName(),
|
||||
"refreshUserToGroupsMappings", "AdminService");
|
||||
Groups.getUserToGroupsMappingService(
|
||||
getConfiguration(getConfig(),
|
||||
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();
|
||||
|
||||
RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");
|
||||
|
||||
return recordFactory.newRecordInstance(
|
||||
RefreshUserToGroupsMappingsResponse.class);
|
||||
|
|
|
@ -33,6 +33,7 @@ import org.apache.hadoop.http.HttpConfig;
|
|||
import org.apache.hadoop.http.HttpConfig.Policy;
|
||||
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
|
||||
import org.apache.hadoop.metrics2.source.JvmMetrics;
|
||||
import org.apache.hadoop.security.Groups;
|
||||
import org.apache.hadoop.security.SecurityUtil;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.service.AbstractService;
|
||||
|
@ -198,6 +199,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
|
|||
this.conf =
|
||||
this.configurationProvider.getConfiguration(this.conf,
|
||||
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
|
||||
// Do refreshUserToGroupsMappings with loaded core-site.xml
|
||||
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
|
||||
.refresh();
|
||||
}
|
||||
|
||||
// register the handlers for all AlwaysOn services using setupDispatcher().
|
||||
|
|
|
@ -24,10 +24,17 @@ import java.io.DataOutputStream;
|
|||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.security.GroupMappingServiceProvider;
|
||||
import org.apache.hadoop.security.Groups;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||
import org.apache.hadoop.security.authorize.ProxyUsers;
|
||||
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
|
||||
|
@ -37,6 +44,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest
|
|||
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
|
||||
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
|
||||
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
|
||||
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.junit.After;
|
||||
|
@ -366,6 +374,84 @@ public class TestRMAdminService {
|
|||
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() {
|
||||
rm = new MockRM(configuration);
|
||||
rm.init(configuration);
|
||||
rm.start();
|
||||
try {
|
||||
rm.adminService
|
||||
.refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
|
||||
.newInstance());
|
||||
} catch (Exception ex) {
|
||||
fail("Using localConfigurationProvider. Should not get any exception.");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
testRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider()
|
||||
throws IOException, YarnException {
|
||||
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
|
||||
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
|
||||
try {
|
||||
rm = new MockRM(configuration);
|
||||
rm.init(configuration);
|
||||
rm.start();
|
||||
fail("Should throw an exception");
|
||||
} catch (Exception ex) {
|
||||
// Expect exception here
|
||||
}
|
||||
|
||||
String user = UserGroupInformation.getCurrentUser().getUserName();
|
||||
List<String> groupWithInit =
|
||||
new ArrayList<String>(Groups.getUserToGroupsMappingService(
|
||||
configuration).getGroups(user));
|
||||
|
||||
// upload default configurations
|
||||
uploadDefaultConfiguration();
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
|
||||
MockUnixGroupsMapping.class,
|
||||
GroupMappingServiceProvider.class);
|
||||
uploadConfiguration(conf, "core-site.xml");
|
||||
|
||||
try {
|
||||
rm = new MockRM(configuration);
|
||||
rm.init(configuration);
|
||||
rm.start();
|
||||
} catch (Exception ex) {
|
||||
fail("Should not get any exceptions");
|
||||
}
|
||||
|
||||
// Make sure RM will use the updated GroupMappingServiceProvider
|
||||
List<String> groupBefore =
|
||||
new ArrayList<String>(Groups.getUserToGroupsMappingService(
|
||||
configuration).getGroups(user));
|
||||
Assert.assertTrue(groupBefore.contains("test_group_A")
|
||||
&& groupBefore.contains("test_group_B")
|
||||
&& groupBefore.contains("test_group_C") && groupBefore.size() == 3);
|
||||
Assert.assertTrue(groupWithInit.size() != groupBefore.size());
|
||||
Assert.assertFalse(groupWithInit.contains("test_group_A")
|
||||
|| groupWithInit.contains("test_group_B")
|
||||
|| groupWithInit.contains("test_group_C"));
|
||||
|
||||
// update the groups
|
||||
MockUnixGroupsMapping.updateGroups();
|
||||
|
||||
rm.adminService
|
||||
.refreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest
|
||||
.newInstance());
|
||||
List<String> groupAfter =
|
||||
Groups.getUserToGroupsMappingService(configuration).getGroups(user);
|
||||
|
||||
// should get the updated groups
|
||||
Assert.assertTrue(groupAfter.contains("test_group_D")
|
||||
&& groupAfter.contains("test_group_E")
|
||||
&& groupAfter.contains("test_group_F") && groupAfter.size() == 3);
|
||||
|
||||
}
|
||||
|
||||
private String writeConfigurationXML(Configuration conf, String confXMLName)
|
||||
throws IOException {
|
||||
DataOutputStream output = null;
|
||||
|
@ -418,4 +504,38 @@ public class TestRMAdminService {
|
|||
.addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
|
||||
uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
|
||||
}
|
||||
|
||||
private static class MockUnixGroupsMapping implements
|
||||
GroupMappingServiceProvider {
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
private static List<String> group = new ArrayList<String>() {{
|
||||
add("test_group_A");
|
||||
add("test_group_B");
|
||||
add("test_group_C");
|
||||
}};
|
||||
|
||||
@Override
|
||||
public List<String> getGroups(String user) throws IOException {
|
||||
return group;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cacheGroupsRefresh() throws IOException {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cacheGroupsAdd(List<String> groups) throws IOException {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
public static void updateGroups() {
|
||||
group.clear();
|
||||
group.add("test_group_D");
|
||||
group.add("test_group_E");
|
||||
group.add("test_group_F");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue