diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9241783c090..84cc137d394 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED OPTIMIZATIONS + HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who" + (vinay via cmccabe) + BUG FIXES HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index e52659dd6e7..5e7fe93a7c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -204,6 +204,14 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { public static final String DEFAULT_HADOOP_HTTP_STATIC_USER = "dr.who"; + /** + * User->groups static mapping to override the groups lookup + */ + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES = + "hadoop.user.group.static.mapping.overrides"; + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT = + "dr.who=;"; + /** Enable/Disable aliases serving from jetty */ public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES = "hadoop.jetty.logs.serve.aliases"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index d9d8781245d..33659c6fada 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -18,15 +18,20 @@ package org.apache.hadoop.security; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.commons.logging.Log; @@ -49,6 +54,8 @@ public class Groups { private final Map userToGroupsMap = new ConcurrentHashMap(); + private final Map> staticUserToGroupsMap = + new HashMap>(); private final long cacheTimeout; private final long warningDeltaMs; @@ -66,12 +73,43 @@ public class Groups { warningDeltaMs = conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS, CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT); - + parseStaticMapping(conf); + if(LOG.isDebugEnabled()) LOG.debug("Group mapping impl=" + impl.getClass().getName() + "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" + warningDeltaMs); } + + /* + * Parse the hadoop.user.group.static.mapping.overrides configuration to + * staticUserToGroupsMap + */ + private void parseStaticMapping(Configuration conf) { + String staticMapping = conf.get( + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT); + Collection mappings = StringUtils.getStringCollection( + staticMapping, ";"); + for (String users : mappings) { + Collection userToGroups = StringUtils.getStringCollection(users, + "="); + if (userToGroups.size() < 1 || userToGroups.size() > 2) { + throw new HadoopIllegalArgumentException("Configuration " + + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES + + " is invalid"); + } + String[] userToGroupsArray = userToGroups.toArray(new String[userToGroups + .size()]); + String user = userToGroupsArray[0]; + List groups = Collections.emptyList(); + if (userToGroupsArray.length == 2) { + groups = (List) StringUtils + .getStringCollection(userToGroupsArray[1]); + } + staticUserToGroupsMap.put(user, groups); + } + } /** * Get the group memberships of a given user. @@ -80,6 +118,11 @@ public class Groups { * @throws IOException */ public List getGroups(String user) throws IOException { + // No need to lookup for groups of static users + List staticMapping = staticUserToGroupsMap.get(user); + if (staticMapping != null) { + return staticMapping; + } // Return cached value if available CachedGroups groups = userToGroupsMap.get(user); long startMs = Time.monotonicNow(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index 32e5572c2e9..d1c428e0f42 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -325,10 +325,24 @@ public class StringUtils { * @return an ArrayList of string values */ public static Collection getStringCollection(String str){ + String delim = ","; + return getStringCollection(str, delim); + } + + /** + * Returns a collection of strings. + * + * @param str + * String to parse + * @param delim + * delimiter to separate the values + * @return Collection of parsed elements. + */ + public static Collection getStringCollection(String str, String delim) { List values = new ArrayList(); if (str == null) return values; - StringTokenizer tokenizer = new StringTokenizer (str,","); + StringTokenizer tokenizer = new StringTokenizer(str, delim); values = new ArrayList(); while (tokenizer.hasMoreTokens()) { values.add(tokenizer.nextToken()); diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index dba417341ef..69607009a45 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1261,4 +1261,18 @@ Specify the port number used by Hadoop mount daemon. + + + hadoop.user.group.static.mapping.overrides + dr.who=; + + Static mapping of user to groups. This will override the groups if + available in the system for the specified user. In otherwords, groups + look-up will not happen for these users, instead groups mapped in this + configuration will be used. + Mapping should be in this format. + user1=group1,group2;user2=;user3=group2; + Default, "dr.who=;" will consider "dr.who" as user without groups. + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java index 48627276f85..44134cc755d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java @@ -19,14 +19,17 @@ package org.apache.hadoop.security; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; +import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import org.apache.commons.logging.Log; @@ -40,10 +43,12 @@ import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; public class TestGroupsCaching { public static final Log LOG = LogFactory.getLog(TestGroupsCaching.class); - private static Configuration conf = new Configuration(); private static String[] myGroups = {"grp1", "grp2"}; + private Configuration conf; - static { + @Before + public void setup() { + conf = new Configuration(); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, FakeGroupMapping.class, ShellBasedUnixGroupsMapping.class); @@ -88,7 +93,7 @@ public class TestGroupsCaching { } @Test - public void TestGroupsCaching() throws Exception { + public void testGroupsCaching() throws Exception { Groups groups = new Groups(conf); groups.cacheGroupsAdd(Arrays.asList(myGroups)); groups.refresh(); @@ -117,4 +122,45 @@ public class TestGroupsCaching { FakeGroupMapping.clearBlackList(); assertTrue(groups.getGroups("user1").size() == 2); } + + public static class FakeunPrivilegedGroupMapping extends FakeGroupMapping { + private static boolean invoked = false; + @Override + public List getGroups(String user) throws IOException { + invoked = true; + return super.getGroups(user); + } + } + + /* + * Group lookup should not happen for static users + */ + @Test + public void testGroupLookupForStaticUsers() throws Exception { + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class); + conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, "me=;user1=group1;user2=group1,group2"); + Groups groups = new Groups(conf); + List userGroups = groups.getGroups("me"); + assertTrue("non-empty groups for static user", userGroups.isEmpty()); + assertFalse("group lookup done for static user", + FakeunPrivilegedGroupMapping.invoked); + + List expected = new ArrayList(); + expected.add("group1"); + + FakeunPrivilegedGroupMapping.invoked = false; + userGroups = groups.getGroups("user1"); + assertTrue("groups not correct", expected.equals(userGroups)); + assertFalse("group lookup done for unprivileged user", + FakeunPrivilegedGroupMapping.invoked); + + expected.add("group2"); + FakeunPrivilegedGroupMapping.invoked = false; + userGroups = groups.getGroups("user2"); + assertTrue("groups not correct", expected.equals(userGroups)); + assertFalse("group lookup done for unprivileged user", + FakeunPrivilegedGroupMapping.invoked); + + } }