diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java index 0502c74291c..768294d707b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java @@ -40,6 +40,23 @@ import com.google.common.collect.HashBiMap; * A simple shell-based implementation of {@link IdMappingServiceProvider} * Map id to user name or group name. It does update every 15 minutes. Only a * single instance of this class is expected to be on the server. + * + * The maps are incrementally updated as described below: + * 1. Initialize the maps as empty. + * 2. Incrementally update the maps + * - When ShellBasedIdMapping is requested for user or group name given + * an ID, or for ID given a user or group name, do look up in the map + * first, if it doesn't exist, find the corresponding entry with shell + * command, and insert the entry to the maps. + * - When group ID is requested for a given group name, and if the + * group name is numerical, the full group map is loaded. Because we + * don't have a good way to find the entry for a numerical group name, + * loading the full map helps to get in all entries. + * 3. Periodically refresh the maps for both user and group, e.g, + * do step 1. + * Note: for testing purpose, step 1 may initial the maps with full mapping + * when using constructor + * {@link ShellBasedIdMapping#ShellBasedIdMapping(Configuration, boolean)}. */ public class ShellBasedIdMapping implements IdMappingServiceProvider { @@ -55,6 +72,8 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID"; private final File staticMappingFile; + private StaticMapping staticMapping = null; + private boolean constructFullMapAtInit = false; // Used for parsing the static mapping file. private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$"); @@ -69,9 +88,18 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { private BiMap gidNameMap = HashBiMap.create(); private long lastUpdateTime = 0; // Last time maps were updated - + + /* + * Constructor + * @param conf the configuration + * @param constructFullMapAtInit initialize the maps with full mapping when + * true, otherwise initialize the maps to empty. This parameter is + * intended for testing only, its default is false. + */ + @VisibleForTesting public ShellBasedIdMapping(Configuration conf, - final String defaultStaticIdMappingFile) throws IOException { + boolean constructFullMapAtInit) throws IOException { + this.constructFullMapAtInit = constructFullMapAtInit; long updateTime = conf.getLong( IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT); @@ -84,22 +112,45 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { timeout = updateTime; } - String staticFilePath = conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY, - defaultStaticIdMappingFile); + String staticFilePath = + conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY, + IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT); staticMappingFile = new File(staticFilePath); - + updateMaps(); } + /* + * Constructor + * initialize user and group maps to empty + * @param conf the configuration + */ public ShellBasedIdMapping(Configuration conf) throws IOException { - this(conf, IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT); + this(conf, false); } @VisibleForTesting public long getTimeout() { return timeout; } - + + @VisibleForTesting + public BiMap getUidNameMap() { + return uidNameMap; + } + + @VisibleForTesting + public BiMap getGidNameMap() { + return gidNameMap; + } + + @VisibleForTesting + synchronized public void clearNameMaps() { + uidNameMap.clear(); + gidNameMap.clear(); + lastUpdateTime = Time.monotonicNow(); + } + synchronized private boolean isExpired() { return Time.monotonicNow() - lastUpdateTime > timeout; } @@ -153,13 +204,15 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { } /** - * Get the whole list of users and groups and save them in the maps. + * Get the list of users or groups returned by the specified command, + * and save them in the corresponding map. * @throws IOException */ @VisibleForTesting - public static void updateMapInternal(BiMap map, String mapName, - String command, String regex, Map staticMapping) - throws IOException { + public static boolean updateMapInternal(BiMap map, + String mapName, String command, String regex, + Map staticMapping) throws IOException { + boolean updated = false; BufferedReader br = null; try { Process process = Runtime.getRuntime().exec( @@ -194,8 +247,9 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { continue; } map.put(key, value); + updated = true; } - LOG.info("Updated " + mapName + " map size: " + map.size()); + LOG.debug("Updated " + mapName + " map size: " + map.size()); } catch (IOException e) { LOG.error("Can't update " + mapName + " map"); @@ -209,20 +263,31 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { } } } + return updated; } - synchronized public void updateMaps() throws IOException { - BiMap uMap = HashBiMap.create(); - BiMap gMap = HashBiMap.create(); - + private boolean checkSupportedPlatform() { if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) { LOG.error("Platform is not supported:" + OS + ". Can't update user map and group map and" + " 'nobody' will be used for any user and group."); - return; + return false; } - - StaticMapping staticMapping = new StaticMapping( + return true; + } + + private static boolean isInteger(final String s) { + try { + Integer.parseInt(s); + } catch(NumberFormatException e) { + return false; + } + // only got here if we didn't return false + return true; + } + + private void initStaticMapping() throws IOException { + staticMapping = new StaticMapping( new HashMap(), new HashMap()); if (staticMappingFile.exists()) { LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping..."); @@ -231,24 +296,218 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile + "' does not exist."); } + } + /* + * Reset the maps to empty. + * For testing code, a full map may be re-constructed here when the object + * was created with constructFullMapAtInit being set to true. + */ + synchronized public void updateMaps() throws IOException { + if (!checkSupportedPlatform()) { + return; + } + + if (constructFullMapAtInit) { + loadFullMaps(); + } else { + clearNameMaps(); + } + } + + synchronized private void loadFullUserMap() throws IOException { + if (staticMapping == null) { + initStaticMapping(); + } + BiMap uMap = HashBiMap.create(); if (OS.startsWith("Mac")) { updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+", staticMapping.uidMapping); - updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+", - staticMapping.gidMapping); } else { updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", staticMapping.uidMapping); + } + uidNameMap = uMap; + lastUpdateTime = Time.monotonicNow(); + } + + synchronized private void loadFullGroupMap() throws IOException { + if (staticMapping == null) { + initStaticMapping(); + } + BiMap gMap = HashBiMap.create(); + + if (OS.startsWith("Mac")) { + updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+", + staticMapping.gidMapping); + } else { updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", staticMapping.gidMapping); } - - uidNameMap = uMap; gidNameMap = gMap; lastUpdateTime = Time.monotonicNow(); } + + synchronized private void loadFullMaps() throws IOException { + initStaticMapping(); + loadFullUserMap(); + loadFullGroupMap(); + } + + // search for id with given name, return ":" + // return + // getent group | cut -d: -f1,3 + // OR + // id -u | awk '{print ":"$1 }' + // + private String getName2IdCmdLinux(final String name, final boolean isGrp) { + String cmd; + if (isGrp) { + cmd = "getent group " + name + " | cut -d: -f1,3"; + } else { + cmd = "id -u " + name + " | awk '{print \"" + name + ":\"$1 }'"; + } + return cmd; + } + // search for name with given id, return ":" + private String getId2NameCmdLinux(final int id, final boolean isGrp) { + String cmd = "getent "; + cmd += isGrp? "group " : "passwd "; + cmd += String.valueOf(id) + " | cut -d: -f1,3"; + return cmd; + } + + // "dscl . -read /Users/ | grep UniqueID" returns "UniqueId: ", + // "dscl . -read /Groups/ | grep PrimaryGroupID" returns "PrimaryGoupID: " + // The following method returns a command that uses awk to process the result, + // of these commands, and returns " ", to simulate one entry returned by + // MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD. + // Specificially, this method returns: + // id -u | awk '{print ":"$1 }' + // OR + // dscl . -read /Groups/ | grep PrimaryGroupID | awk '($1 == "PrimaryGroupID:") { print " " $2 }' + // + private String getName2IdCmdMac(final String name, final boolean isGrp) { + String cmd; + if (isGrp) { + cmd = "dscl . -read /Groups/" + name; + cmd += " | grep PrimaryGroupID | awk '($1 == \"PrimaryGroupID:\") "; + cmd += "{ print \"" + name + " \" $2 }'"; + } else { + cmd = "id -u " + name + " | awk '{print \"" + name + " \"$1 }'"; + } + return cmd; + } + + // "dscl . -search /Users UniqueID " returns + // UniqueID = ( + // + // ) + // "dscl . -search /Groups PrimaryGroupID " returns + // PrimaryGroupID = ( + // + // ) + // The following method returns a command that uses sed to process the + // the result and returns " " to simulate one entry returned + // by MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD. + // For certain negative id case like nfsnobody, the is quoted as + // "", added one sed section to remove the quote. + // Specifically, the method returns: + // dscl . -search /Users UniqueID | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/UniqueID =//g' | sed 's/)//g' | sed 's/\"//g' + // OR + // dscl . -search /Groups PrimaryGroupID | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/PrimaryGroupID =//g' | sed 's/)//g' | sed 's/\"//g' + // + private String getId2NameCmdMac(final int id, final boolean isGrp) { + String cmd = "dscl . -search /"; + cmd += isGrp? "Groups PrimaryGroupID " : "Users UniqueID "; + cmd += String.valueOf(id); + cmd += " | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/"; + cmd += isGrp? "PrimaryGroupID" : "UniqueID"; + cmd += " = (//g' | sed 's/)//g' | sed 's/\\\"//g'"; + return cmd; + } + + synchronized private void updateMapIncr(final String name, + final boolean isGrp) throws IOException { + if (!checkSupportedPlatform()) { + return; + } + if (isInteger(name) && isGrp) { + loadFullGroupMap(); + return; + } + + boolean updated = false; + if (staticMapping == null) { + initStaticMapping(); + } + + if (OS.startsWith("Linux")) { + if (isGrp) { + updated = updateMapInternal(gidNameMap, "group", + getName2IdCmdLinux(name, true), ":", + staticMapping.gidMapping); + } else { + updated = updateMapInternal(uidNameMap, "user", + getName2IdCmdLinux(name, false), ":", + staticMapping.uidMapping); + } + } else { + // Mac + if (isGrp) { + updated = updateMapInternal(gidNameMap, "group", + getName2IdCmdMac(name, true), "\\s+", + staticMapping.gidMapping); + } else { + updated = updateMapInternal(uidNameMap, "user", + getName2IdCmdMac(name, false), "\\s+", + staticMapping.uidMapping); + } + } + if (updated) { + lastUpdateTime = Time.monotonicNow(); + } + } + + synchronized private void updateMapIncr(final int id, + final boolean isGrp) throws IOException { + if (!checkSupportedPlatform()) { + return; + } + + boolean updated = false; + if (staticMapping == null) { + initStaticMapping(); + } + + if (OS.startsWith("Linux")) { + if (isGrp) { + updated = updateMapInternal(gidNameMap, "group", + getId2NameCmdLinux(id, true), ":", + staticMapping.gidMapping); + } else { + updated = updateMapInternal(uidNameMap, "user", + getId2NameCmdLinux(id, false), ":", + staticMapping.uidMapping); + } + } else { + // Mac + if (isGrp) { + updated = updateMapInternal(gidNameMap, "group", + getId2NameCmdMac(id, true), "\\s+", + staticMapping.gidMapping); + } else { + updated = updateMapInternal(uidNameMap, "user", + getId2NameCmdMac(id, false), "\\s+", + staticMapping.uidMapping); + } + } + if (updated) { + lastUpdateTime = Time.monotonicNow(); + } + } + @SuppressWarnings("serial") static final class PassThroughMap extends HashMap { @@ -335,7 +594,11 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { Integer id = uidNameMap.inverse().get(user); if (id == null) { - throw new IOException("User just deleted?:" + user); + updateMapIncr(user, false); + id = uidNameMap.inverse().get(user); + if (id == null) { + throw new IOException("User just deleted?:" + user); + } } return id.intValue(); } @@ -345,8 +608,11 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { Integer id = gidNameMap.inverse().get(group); if (id == null) { - throw new IOException("No such group:" + group); - + updateMapIncr(group, true); + id = gidNameMap.inverse().get(group); + if (id == null) { + throw new IOException("No such group:" + group); + } } return id.intValue(); } @@ -355,9 +621,16 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { checkAndUpdateMaps(); String uname = uidNameMap.get(uid); if (uname == null) { - LOG.warn("Can't find user name for uid " + uid - + ". Use default user name " + unknown); - uname = unknown; + try { + updateMapIncr(uid, false); + } catch (Exception e) { + } + uname = uidNameMap.get(uid); + if (uname == null) { + LOG.warn("Can't find user name for uid " + uid + + ". Use default user name " + unknown); + uname = unknown; + } } return uname; } @@ -366,9 +639,16 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider { checkAndUpdateMaps(); String gname = gidNameMap.get(gid); if (gname == null) { - LOG.warn("Can't find group name for gid " + gid - + ". Use default group name " + unknown); - gname = unknown; + try { + updateMapIncr(gid, true); + } catch (Exception e) { + } + gname = gidNameMap.get(gid); + if (gname == null) { + LOG.warn("Can't find group name for gid " + gid + + ". Use default group name " + unknown); + gname = unknown; + } } return gname; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java index 808c3fd2a69..ec8ac1d4971 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java @@ -219,4 +219,65 @@ public class TestShellBasedIdMapping { assertEquals(iug.getTimeout(), IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT * 2); } + + @Test + public void testUpdateMapIncr() throws IOException { + Configuration conf = new Configuration(); + conf.setLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, 600000); + ShellBasedIdMapping refIdMapping = + new ShellBasedIdMapping(conf, true); + ShellBasedIdMapping incrIdMapping = new ShellBasedIdMapping(conf); + + // Command such as "getent passwd " will return empty string if + // is numerical, remove them from the map for testing purpose. + BiMap uidNameMap = refIdMapping.getUidNameMap(); + BiMap gidNameMap = refIdMapping.getGidNameMap(); + + // Force empty map, to see effect of incremental map update of calling + // getUserName() + incrIdMapping.clearNameMaps(); + uidNameMap = refIdMapping.getUidNameMap(); + for (BiMap.Entry me : uidNameMap.entrySet()) { + Integer id = me.getKey(); + String name = me.getValue(); + String tname = incrIdMapping.getUserName(id, null); + assertEquals(name, tname); + } + assertEquals(uidNameMap.size(), incrIdMapping.getUidNameMap().size()); + + // Force empty map, to see effect of incremental map update of calling + // getUid() + incrIdMapping.clearNameMaps(); + for (BiMap.Entry me : uidNameMap.entrySet()) { + Integer id = me.getKey(); + String name = me.getValue(); + Integer tid = incrIdMapping.getUid(name); + assertEquals(id, tid); + } + assertEquals(uidNameMap.size(), incrIdMapping.getUidNameMap().size()); + + // Force empty map, to see effect of incremental map update of calling + // getGroupName() + incrIdMapping.clearNameMaps(); + gidNameMap = refIdMapping.getGidNameMap(); + for (BiMap.Entry me : gidNameMap.entrySet()) { + Integer id = me.getKey(); + String name = me.getValue(); + String tname = incrIdMapping.getGroupName(id, null); + assertEquals(name, tname); + } + assertEquals(gidNameMap.size(), incrIdMapping.getGidNameMap().size()); + + // Force empty map, to see effect of incremental map update of calling + // getGid() + incrIdMapping.clearNameMaps(); + gidNameMap = refIdMapping.getGidNameMap(); + for (BiMap.Entry me : gidNameMap.entrySet()) { + Integer id = me.getKey(); + String name = me.getValue(); + Integer tid = incrIdMapping.getGid(name); + assertEquals(id, tid); + } + assertEquals(gidNameMap.size(), incrIdMapping.getGidNameMap().size()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index d96babfdb41..f86dbecd44c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -173,8 +173,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { this.config = config; config.set(FsPermission.UMASK_LABEL, "000"); - iug = new ShellBasedIdMapping(config, - IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT); + iug = new ShellBasedIdMapping(config); aixCompatMode = config.getBoolean( NfsConfigKeys.AIX_COMPAT_MODE_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 799f37f4e9d..c5e9f821ff4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -183,6 +183,9 @@ Release 2.7.0 - UNRELEASED HDFS-7399. Lack of synchronization in DFSOutputStream#Packet#getLastByteOffsetBlock() (vinayakumarb) + HDFS-7146. NFS ID/Group lookup requires SSSD enumeration on the server + (Yongjun Zhang via brandonli) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES