diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c3d50dc3ca8..bea94261d87 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -10,6 +10,8 @@ Release 2.5.0 - UNRELEASED HADOOP-9704. Write metrics sink plugin for Hadoop/Graphite (Chu Tong, Alex Newman and Babak Behzad via raviprak) + HADOOP-8943. Support multiple group mapping providers. (Kai Zheng via brandonli) + IMPROVEMENTS HADOOP-10451. Remove unused field and imports from SaslRpcServer. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java new file mode 100644 index 00000000000..ffa7e2bdbbd --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * An implementation of {@link GroupMappingServiceProvider} which + * composites other group mapping providers for determining group membership. + * This allows to combine existing provider implementations and composite + * a virtually new provider without customized development to deal with complex situation. + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Evolving +public class CompositeGroupsMapping + implements GroupMappingServiceProvider, Configurable { + + public static final String MAPPING_PROVIDERS_CONFIG_KEY = GROUP_MAPPING_CONFIG_PREFIX + ".providers"; + public static final String MAPPING_PROVIDERS_COMBINED_CONFIG_KEY = MAPPING_PROVIDERS_CONFIG_KEY + ".combined"; + public static final String MAPPING_PROVIDER_CONFIG_PREFIX = GROUP_MAPPING_CONFIG_PREFIX + ".provider"; + + private static final Log LOG = LogFactory.getLog(CompositeGroupsMapping.class); + + private List providersList = + new ArrayList(); + + private Configuration conf; + private boolean combined; + + + + /** + * Returns list of groups for a user. + * + * @param user get groups for this user + * @return list of groups for a given user + */ + @Override + public synchronized List getGroups(String user) throws IOException { + Set groupSet = new TreeSet(); + + List groups = null; + for (GroupMappingServiceProvider provider : providersList) { + try { + groups = provider.getGroups(user); + } catch (Exception e) { + //LOG.warn("Exception trying to get groups for user " + user, e); + } + if (groups != null && ! groups.isEmpty()) { + groupSet.addAll(groups); + if (!combined) break; + } + } + + List results = new ArrayList(groupSet.size()); + results.addAll(groupSet); + return results; + } + + /** + * Caches groups, no need to do that for this provider + */ + @Override + public void cacheGroupsRefresh() throws IOException { + // does nothing in this provider of user to groups mapping + } + + /** + * Adds groups to cache, no need to do that for this provider + * + * @param groups unused + */ + @Override + public void cacheGroupsAdd(List groups) throws IOException { + // does nothing in this provider of user to groups mapping + } + + @Override + public synchronized Configuration getConf() { + return conf; + } + + @Override + public synchronized void setConf(Configuration conf) { + this.conf = conf; + + this.combined = conf.getBoolean(MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, true); + + loadMappingProviders(); + } + + private void loadMappingProviders() { + String[] providerNames = conf.getStrings(MAPPING_PROVIDERS_CONFIG_KEY, new String[]{}); + + String providerKey; + for (String name : providerNames) { + providerKey = MAPPING_PROVIDER_CONFIG_PREFIX + "." + name; + Class providerClass = conf.getClass(providerKey, null); + if (providerClass == null) { + LOG.error("The mapping provider, " + name + " does not have a valid class"); + } else { + addMappingProvider(name, providerClass); + } + } + } + + private void addMappingProvider(String providerName, Class providerClass) { + Configuration newConf = prepareConf(providerName); + GroupMappingServiceProvider provider = + (GroupMappingServiceProvider) ReflectionUtils.newInstance(providerClass, newConf); + providersList.add(provider); + + } + + /* + * For any provider specific configuration properties, such as "hadoop.security.group.mapping.ldap.url" + * and the like, allow them to be configured as "hadoop.security.group.mapping.provider.PROVIDER-X.ldap.url", + * so that a provider such as LdapGroupsMapping can be used to composite a complex one with other providers. + */ + private Configuration prepareConf(String providerName) { + Configuration newConf = new Configuration(); + Iterator> entries = conf.iterator(); + String providerKey = MAPPING_PROVIDER_CONFIG_PREFIX + "." + providerName; + while (entries.hasNext()) { + Map.Entry entry = entries.next(); + String key = entry.getKey(); + // get a property like "hadoop.security.group.mapping.provider.PROVIDER-X.ldap.url" + if (key.startsWith(providerKey) && !key.equals(providerKey)) { + // restore to be the one like "hadoop.security.group.mapping.ldap.url" + // so that can be used by original provider. + key = key.replace(".provider." + providerName, ""); + newConf.set(key, entry.getValue()); + } + } + return newConf; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java index ef26b5c5223..8b90f5bc7af 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; /** * An interface for the implementation of a user-to-groups mapping service @@ -30,6 +31,7 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public interface GroupMappingServiceProvider { + public static final String GROUP_MAPPING_CONFIG_PREFIX = CommonConfigurationKeysPublic.HADOOP_SECURITY_GROUP_MAPPING; /** * Get all various group memberships of a given user. diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 07f27b72672..25600561b37 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -94,6 +94,98 @@ + + hadoop.security.groups.cache.secs 300 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java new file mode 100644 index 00000000000..79f56e065a9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.junit.Test; + + +public class TestCompositeGroupMapping { + public static final Log LOG = LogFactory.getLog(TestCompositeGroupMapping.class); + private static Configuration conf = new Configuration(); + + private static class TestUser { + String name; + String group; + String group2; + + public TestUser(String name, String group) { + this.name = name; + this.group = group; + } + + public TestUser(String name, String group, String group2) { + this(name, group); + this.group2 = group2; + } + }; + + private static TestUser john = new TestUser("John", "user-group"); + private static TestUser hdfs = new TestUser("hdfs", "supergroup"); + private static TestUser jack = new TestUser("Jack", "user-group", "dev-group-1"); + + private static final String PROVIDER_SPECIFIC_CONF = ".test.prop"; + private static final String PROVIDER_SPECIFIC_CONF_KEY = + GroupMappingServiceProvider.GROUP_MAPPING_CONFIG_PREFIX + PROVIDER_SPECIFIC_CONF; + private static final String PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER = "value-for-user"; + private static final String PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER = "value-for-cluster"; + + private static abstract class GroupMappingProviderBase + implements GroupMappingServiceProvider, Configurable { + + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + + } + + protected List toList(String group) { + if (group != null) { + return Arrays.asList(new String[] {group}); + } + return new ArrayList(); + } + + protected void checkTestConf(String expectedValue) { + String configValue = getConf().get(PROVIDER_SPECIFIC_CONF_KEY); + if (configValue == null || !configValue.equals(expectedValue)) { + throw new RuntimeException("Failed to find mandatory configuration of " + PROVIDER_SPECIFIC_CONF_KEY); + } + } + }; + + private static class UserProvider extends GroupMappingProviderBase { + @Override + public List getGroups(String user) throws IOException { + checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER); + + String group = null; + if (user.equals(john.name)) { + group = john.group; + } else if (user.equals(jack.name)) { + group = jack.group; + } + + return toList(group); + } + } + + private static class ClusterProvider extends GroupMappingProviderBase { + @Override + public List getGroups(String user) throws IOException { + checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER); + + String group = null; + if (user.equals(hdfs.name)) { + group = hdfs.group; + } else if (user.equals(jack.name)) { // jack has another group from clusterProvider + group = jack.group2; + } + + return toList(group); + } + } + + static { + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + CompositeGroupsMapping.class, GroupMappingServiceProvider.class); + conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_CONFIG_KEY, "userProvider,clusterProvider"); + + conf.setClass(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + ".userProvider", + UserProvider.class, GroupMappingServiceProvider.class); + + conf.setClass(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + ".clusterProvider", + ClusterProvider.class, GroupMappingServiceProvider.class); + + conf.set(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + + ".clusterProvider" + PROVIDER_SPECIFIC_CONF, PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER); + + conf.set(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + + ".userProvider" + PROVIDER_SPECIFIC_CONF, PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER); + } + + @Test + public void TestMultipleGroupsMapping() throws Exception { + Groups groups = new Groups(conf); + + assertTrue(groups.getGroups(john.name).get(0).equals(john.group)); + assertTrue(groups.getGroups(hdfs.name).get(0).equals(hdfs.group)); + } + + @Test + public void TestMultipleGroupsMappingWithCombined() throws Exception { + conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, "true"); + Groups groups = new Groups(conf); + + assertTrue(groups.getGroups(jack.name).size() == 2); + // the configured providers list in order is "userProvider,clusterProvider" + // group -> userProvider, group2 -> clusterProvider + assertTrue(groups.getGroups(jack.name).contains(jack.group)); + assertTrue(groups.getGroups(jack.name).contains(jack.group2)); + } + + @Test + public void TestMultipleGroupsMappingWithoutCombined() throws Exception { + conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, "false"); + Groups groups = new Groups(conf); + + // the configured providers list in order is "userProvider,clusterProvider" + // group -> userProvider, group2 -> clusterProvider + assertTrue(groups.getGroups(jack.name).size() == 1); + assertTrue(groups.getGroups(jack.name).get(0).equals(jack.group)); + } +}