HADOOP-8943. Merging change r1605857 from trunk

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1605884 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Brandon Li 2014-06-26 19:21:00 +00:00
parent d2163883f3
commit aad8ea5896
5 changed files with 447 additions and 0 deletions

View File

@ -10,6 +10,8 @@ Release 2.5.0 - UNRELEASED
HADOOP-9704. Write metrics sink plugin for Hadoop/Graphite (Chu Tong, Alex Newman and Babak Behzad via raviprak) HADOOP-9704. Write metrics sink plugin for Hadoop/Graphite (Chu Tong, Alex Newman and Babak Behzad via raviprak)
HADOOP-8943. Support multiple group mapping providers. (Kai Zheng via brandonli)
IMPROVEMENTS IMPROVEMENTS
HADOOP-10451. Remove unused field and imports from SaslRpcServer. HADOOP-10451. Remove unused field and imports from SaslRpcServer.

View File

@ -0,0 +1,166 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.security;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
/**
* An implementation of {@link GroupMappingServiceProvider} which
* composites other group mapping providers for determining group membership.
* This allows to combine existing provider implementations and composite
* a virtually new provider without customized development to deal with complex situation.
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
public class CompositeGroupsMapping
implements GroupMappingServiceProvider, Configurable {
public static final String MAPPING_PROVIDERS_CONFIG_KEY = GROUP_MAPPING_CONFIG_PREFIX + ".providers";
public static final String MAPPING_PROVIDERS_COMBINED_CONFIG_KEY = MAPPING_PROVIDERS_CONFIG_KEY + ".combined";
public static final String MAPPING_PROVIDER_CONFIG_PREFIX = GROUP_MAPPING_CONFIG_PREFIX + ".provider";
private static final Log LOG = LogFactory.getLog(CompositeGroupsMapping.class);
private List<GroupMappingServiceProvider> providersList =
new ArrayList<GroupMappingServiceProvider>();
private Configuration conf;
private boolean combined;
/**
* Returns list of groups for a user.
*
* @param user get groups for this user
* @return list of groups for a given user
*/
@Override
public synchronized List<String> getGroups(String user) throws IOException {
Set<String> groupSet = new TreeSet<String>();
List<String> groups = null;
for (GroupMappingServiceProvider provider : providersList) {
try {
groups = provider.getGroups(user);
} catch (Exception e) {
//LOG.warn("Exception trying to get groups for user " + user, e);
}
if (groups != null && ! groups.isEmpty()) {
groupSet.addAll(groups);
if (!combined) break;
}
}
List<String> results = new ArrayList<String>(groupSet.size());
results.addAll(groupSet);
return results;
}
/**
* Caches groups, no need to do that for this provider
*/
@Override
public void cacheGroupsRefresh() throws IOException {
// does nothing in this provider of user to groups mapping
}
/**
* Adds groups to cache, no need to do that for this provider
*
* @param groups unused
*/
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
// does nothing in this provider of user to groups mapping
}
@Override
public synchronized Configuration getConf() {
return conf;
}
@Override
public synchronized void setConf(Configuration conf) {
this.conf = conf;
this.combined = conf.getBoolean(MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, true);
loadMappingProviders();
}
private void loadMappingProviders() {
String[] providerNames = conf.getStrings(MAPPING_PROVIDERS_CONFIG_KEY, new String[]{});
String providerKey;
for (String name : providerNames) {
providerKey = MAPPING_PROVIDER_CONFIG_PREFIX + "." + name;
Class<?> providerClass = conf.getClass(providerKey, null);
if (providerClass == null) {
LOG.error("The mapping provider, " + name + " does not have a valid class");
} else {
addMappingProvider(name, providerClass);
}
}
}
private void addMappingProvider(String providerName, Class<?> providerClass) {
Configuration newConf = prepareConf(providerName);
GroupMappingServiceProvider provider =
(GroupMappingServiceProvider) ReflectionUtils.newInstance(providerClass, newConf);
providersList.add(provider);
}
/*
* For any provider specific configuration properties, such as "hadoop.security.group.mapping.ldap.url"
* and the like, allow them to be configured as "hadoop.security.group.mapping.provider.PROVIDER-X.ldap.url",
* so that a provider such as LdapGroupsMapping can be used to composite a complex one with other providers.
*/
private Configuration prepareConf(String providerName) {
Configuration newConf = new Configuration();
Iterator<Map.Entry<String, String>> entries = conf.iterator();
String providerKey = MAPPING_PROVIDER_CONFIG_PREFIX + "." + providerName;
while (entries.hasNext()) {
Map.Entry<String, String> entry = entries.next();
String key = entry.getKey();
// get a property like "hadoop.security.group.mapping.provider.PROVIDER-X.ldap.url"
if (key.startsWith(providerKey) && !key.equals(providerKey)) {
// restore to be the one like "hadoop.security.group.mapping.ldap.url"
// so that can be used by original provider.
key = key.replace(".provider." + providerName, "");
newConf.set(key, entry.getValue());
}
}
return newConf;
}
}

View File

@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/** /**
* An interface for the implementation of a user-to-groups mapping service * An interface for the implementation of a user-to-groups mapping service
@ -30,6 +31,7 @@
@InterfaceAudience.Public @InterfaceAudience.Public
@InterfaceStability.Evolving @InterfaceStability.Evolving
public interface GroupMappingServiceProvider { public interface GroupMappingServiceProvider {
public static final String GROUP_MAPPING_CONFIG_PREFIX = CommonConfigurationKeysPublic.HADOOP_SECURITY_GROUP_MAPPING;
/** /**
* Get all various group memberships of a given user. * Get all various group memberships of a given user.

View File

@ -94,6 +94,98 @@
</description> </description>
</property> </property>
<!--
=== Multiple group mapping providers configuration sample ===
This sample illustrates a typical use case for CompositeGroupsMapping where
Hadoop authentication uses MIT Kerberos which trusts an AD realm. In this case, service
principals such as hdfs, mapred, hbase, hive, oozie and etc can be placed in In MIT Kerberos,
but end users are just from the trusted AD. For the service principals, ShellBasedUnixGroupsMapping
provider can be used to query their groups for efficiency, and for end users, LdapGroupsMapping
provider can be used. This avoids to add group entries in AD for service principals when only using
LdapGroupsMapping provider.
In case multiple ADs are involved and trusted by the MIT Kerberos in this use case, LdapGroupsMapping
provider can be used more times with different AD specific configurations. This sample also shows how
to do that. Here are the necessary configurations.
<property>
<name>hadoop.security.group.mapping</name>
<value>org.apache.hadoop.security.CompositeGroupsMapping</value>
<description>
Class for user to group mapping (get groups for a given user) for ACL, which
makes use of other multiple providers to provide the service.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.providers</name>
<value>shell4services,ad4usersX,ad4usersY</value>
<description>
Comma separated of names of other providers to provide user to group mapping.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.providers.combined</name>
<value>true</value>
<description>
true or false to indicate whether groups from the providers are combined or not. The default value is true
If true, then all the providers will be tried to get groups and all the groups are combined to return as
the final results. Otherwise, providers are tried one by one in the configured list order, and if any
groups are retrieved from any provider, then the groups will be returned without trying the left ones.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.provider.shell4services</name>
<value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
<description>
Class for group mapping provider named by 'shell4services'. The name can then be referenced
by hadoop.security.group.mapping.providers property.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.provider.ad4usersX</name>
<value>org.apache.hadoop.security.LdapGroupsMapping</value>
<description>
Class for group mapping provider named by 'ad4usersX'. The name can then be referenced
by hadoop.security.group.mapping.providers property.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.provider.ad4usersY</name>
<value>org.apache.hadoop.security.LdapGroupsMapping</value>
<description>
Class for group mapping provider named by 'ad4usersY'. The name can then be referenced
by hadoop.security.group.mapping.providers property.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.provider.ad4usersX.ldap.url</name>
<value>ldap://ad-host-for-users-X:389</value>
<description>
ldap url for the provider named by 'ad4usersX'. Note this property comes from
'hadoop.security.group.mapping.ldap.url'.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.provider.ad4usersY.ldap.url</name>
<value>ldap://ad-host-for-users-Y:389</value>
<description>
ldap url for the provider named by 'ad4usersY'. Note this property comes from
'hadoop.security.group.mapping.ldap.url'.
</description>
</property>
You also need to configure other properties like
hadoop.security.group.mapping.ldap.bind.password.file and etc.
for ldap providers in the same way as above does.
-->
<property> <property>
<name>hadoop.security.groups.cache.secs</name> <name>hadoop.security.groups.cache.secs</name>
<value>300</value> <value>300</value>

View File

@ -0,0 +1,185 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.security;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.junit.Test;
public class TestCompositeGroupMapping {
public static final Log LOG = LogFactory.getLog(TestCompositeGroupMapping.class);
private static Configuration conf = new Configuration();
private static class TestUser {
String name;
String group;
String group2;
public TestUser(String name, String group) {
this.name = name;
this.group = group;
}
public TestUser(String name, String group, String group2) {
this(name, group);
this.group2 = group2;
}
};
private static TestUser john = new TestUser("John", "user-group");
private static TestUser hdfs = new TestUser("hdfs", "supergroup");
private static TestUser jack = new TestUser("Jack", "user-group", "dev-group-1");
private static final String PROVIDER_SPECIFIC_CONF = ".test.prop";
private static final String PROVIDER_SPECIFIC_CONF_KEY =
GroupMappingServiceProvider.GROUP_MAPPING_CONFIG_PREFIX + PROVIDER_SPECIFIC_CONF;
private static final String PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER = "value-for-user";
private static final String PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER = "value-for-cluster";
private static abstract class GroupMappingProviderBase
implements GroupMappingServiceProvider, Configurable {
private Configuration conf;
@Override
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Configuration getConf() {
return this.conf;
}
@Override
public void cacheGroupsRefresh() throws IOException {
}
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
protected List<String> toList(String group) {
if (group != null) {
return Arrays.asList(new String[] {group});
}
return new ArrayList<String>();
}
protected void checkTestConf(String expectedValue) {
String configValue = getConf().get(PROVIDER_SPECIFIC_CONF_KEY);
if (configValue == null || !configValue.equals(expectedValue)) {
throw new RuntimeException("Failed to find mandatory configuration of " + PROVIDER_SPECIFIC_CONF_KEY);
}
}
};
private static class UserProvider extends GroupMappingProviderBase {
@Override
public List<String> getGroups(String user) throws IOException {
checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER);
String group = null;
if (user.equals(john.name)) {
group = john.group;
} else if (user.equals(jack.name)) {
group = jack.group;
}
return toList(group);
}
}
private static class ClusterProvider extends GroupMappingProviderBase {
@Override
public List<String> getGroups(String user) throws IOException {
checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER);
String group = null;
if (user.equals(hdfs.name)) {
group = hdfs.group;
} else if (user.equals(jack.name)) { // jack has another group from clusterProvider
group = jack.group2;
}
return toList(group);
}
}
static {
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
CompositeGroupsMapping.class, GroupMappingServiceProvider.class);
conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_CONFIG_KEY, "userProvider,clusterProvider");
conf.setClass(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + ".userProvider",
UserProvider.class, GroupMappingServiceProvider.class);
conf.setClass(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + ".clusterProvider",
ClusterProvider.class, GroupMappingServiceProvider.class);
conf.set(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX +
".clusterProvider" + PROVIDER_SPECIFIC_CONF, PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER);
conf.set(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX +
".userProvider" + PROVIDER_SPECIFIC_CONF, PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER);
}
@Test
public void TestMultipleGroupsMapping() throws Exception {
Groups groups = new Groups(conf);
assertTrue(groups.getGroups(john.name).get(0).equals(john.group));
assertTrue(groups.getGroups(hdfs.name).get(0).equals(hdfs.group));
}
@Test
public void TestMultipleGroupsMappingWithCombined() throws Exception {
conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, "true");
Groups groups = new Groups(conf);
assertTrue(groups.getGroups(jack.name).size() == 2);
// the configured providers list in order is "userProvider,clusterProvider"
// group -> userProvider, group2 -> clusterProvider
assertTrue(groups.getGroups(jack.name).contains(jack.group));
assertTrue(groups.getGroups(jack.name).contains(jack.group2));
}
@Test
public void TestMultipleGroupsMappingWithoutCombined() throws Exception {
conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, "false");
Groups groups = new Groups(conf);
// the configured providers list in order is "userProvider,clusterProvider"
// group -> userProvider, group2 -> clusterProvider
assertTrue(groups.getGroups(jack.name).size() == 1);
assertTrue(groups.getGroups(jack.name).get(0).equals(jack.group));
}
}