Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-7146. NFS ID/Group lookup requires SSSD enumeration on the server. Contributed by Yongjun Zhang

This commit is contained in:
Brandon Li 2014-11-17 13:16:43 -08:00
parent 81c9d17af8
commit 351c5561c2
4 changed files with 377 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

320
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
View File

@ -40,6 +40,23 @@ import com.google.common.collect.HashBiMap;
* A simple shell-based implementation of {@link IdMappingServiceProvider}
* Map id to user name or group name. It does update every 15 minutes. Only a
* single instance of this class is expected to be on the server.
*
* The maps are incrementally updated as described below:
* 1. Initialize the maps as empty.
* 2. Incrementally update the maps
* - When ShellBasedIdMapping is requested for user or group name given
* an ID, or for ID given a user or group name, do look up in the map
* first, if it doesn't exist, find the corresponding entry with shell
* command, and insert the entry to the maps.
* - When group ID is requested for a given group name, and if the
* group name is numerical, the full group map is loaded. Because we
* don't have a good way to find the entry for a numerical group name,
* loading the full map helps to get in all entries.
* 3. Periodically refresh the maps for both user and group, e.g,
* do step 1.
* Note: for testing purpose, step 1 may initial the maps with full mapping
* when using constructor
* {@link ShellBasedIdMapping#ShellBasedIdMapping(Configuration, boolean)}.
*/
public class ShellBasedIdMapping implements IdMappingServiceProvider {
@ -55,6 +72,8 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID";
private final File staticMappingFile;
private StaticMapping staticMapping = null;
private boolean constructFullMapAtInit = false;
// Used for parsing the static mapping file.
private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$");
@ -70,8 +89,17 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
private long lastUpdateTime = 0; // Last time maps were updated
/*
* Constructor
* @param conf the configuration
* @param constructFullMapAtInit initialize the maps with full mapping when
* true, otherwise initialize the maps to empty. This parameter is
* intended for testing only, its default is false.
*/
@VisibleForTesting
public ShellBasedIdMapping(Configuration conf,
final String defaultStaticIdMappingFile) throws IOException {
boolean constructFullMapAtInit) throws IOException {
this.constructFullMapAtInit = constructFullMapAtInit;
long updateTime = conf.getLong(
IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY,
IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT);
@ -84,15 +112,21 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
timeout = updateTime;
}
String staticFilePath = conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
defaultStaticIdMappingFile);
String staticFilePath =
conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
staticMappingFile = new File(staticFilePath);
updateMaps();
}
/*
* Constructor
* initialize user and group maps to empty
* @param conf the configuration
*/
public ShellBasedIdMapping(Configuration conf) throws IOException {
this(conf, IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
this(conf, false);
}
@VisibleForTesting
@ -100,6 +134,23 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
return timeout;
}
@VisibleForTesting
public BiMap<Integer, String> getUidNameMap() {
return uidNameMap;
}
@VisibleForTesting
public BiMap<Integer, String> getGidNameMap() {
return gidNameMap;
}
@VisibleForTesting
synchronized public void clearNameMaps() {
uidNameMap.clear();
gidNameMap.clear();
lastUpdateTime = Time.monotonicNow();
}
synchronized private boolean isExpired() {
return Time.monotonicNow() - lastUpdateTime > timeout;
}
@ -153,13 +204,15 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
}
/**
* Get the whole list of users and groups and save them in the maps.
* Get the list of users or groups returned by the specified command,
* and save them in the corresponding map.
* @throws IOException
*/
@VisibleForTesting
public static void updateMapInternal(BiMap<Integer, String> map, String mapName,
String command, String regex, Map<Integer, Integer> staticMapping)
throws IOException {
public static boolean updateMapInternal(BiMap<Integer, String> map,
String mapName, String command, String regex,
Map<Integer, Integer> staticMapping) throws IOException {
boolean updated = false;
BufferedReader br = null;
try {
Process process = Runtime.getRuntime().exec(
@ -194,8 +247,9 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
continue;
}
map.put(key, value);
updated = true;
}
LOG.info("Updated " + mapName + " map size: " + map.size());
LOG.debug("Updated " + mapName + " map size: " + map.size());
} catch (IOException e) {
LOG.error("Can't update " + mapName + " map");
@ -209,20 +263,31 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
}
}
}
return updated;
}
synchronized public void updateMaps() throws IOException {
BiMap<Integer, String> uMap = HashBiMap.create();
BiMap<Integer, String> gMap = HashBiMap.create();
private boolean checkSupportedPlatform() {
if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) {
LOG.error("Platform is not supported:" + OS
+ ". Can't update user map and group map and"
+ " 'nobody' will be used for any user and group.");
return;
return false;
}
return true;
}
StaticMapping staticMapping = new StaticMapping(
private static boolean isInteger(final String s) {
try {
Integer.parseInt(s);
} catch(NumberFormatException e) {
return false;
}
// only got here if we didn't return false
return true;
}
private void initStaticMapping() throws IOException {
staticMapping = new StaticMapping(
new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>());
if (staticMappingFile.exists()) {
LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping...");
@ -231,24 +296,218 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile
+ "' does not exist.");
}
}
/*
* Reset the maps to empty.
* For testing code, a full map may be re-constructed here when the object
* was created with constructFullMapAtInit being set to true.
*/
synchronized public void updateMaps() throws IOException {
if (!checkSupportedPlatform()) {
return;
}
if (constructFullMapAtInit) {
loadFullMaps();
} else {
clearNameMaps();
}
}
synchronized private void loadFullUserMap() throws IOException {
if (staticMapping == null) {
initStaticMapping();
}
BiMap<Integer, String> uMap = HashBiMap.create();
if (OS.startsWith("Mac")) {
updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
staticMapping.uidMapping);
updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
staticMapping.gidMapping);
} else {
updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":",
staticMapping.uidMapping);
}
uidNameMap = uMap;
lastUpdateTime = Time.monotonicNow();
}
synchronized private void loadFullGroupMap() throws IOException {
if (staticMapping == null) {
initStaticMapping();
}
BiMap<Integer, String> gMap = HashBiMap.create();
if (OS.startsWith("Mac")) {
updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
staticMapping.gidMapping);
} else {
updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":",
staticMapping.gidMapping);
}
uidNameMap = uMap;
gidNameMap = gMap;
lastUpdateTime = Time.monotonicNow();
}
synchronized private void loadFullMaps() throws IOException {
initStaticMapping();
loadFullUserMap();
loadFullGroupMap();
}
// search for id with given name, return "<name>:<id>"
// return
// getent group <name> | cut -d: -f1,3
// OR
// id -u <name> | awk '{print "<name>:"$1 }'
//
private String getName2IdCmdLinux(final String name, final boolean isGrp) {
String cmd;
if (isGrp) {
cmd = "getent group " + name + " | cut -d: -f1,3";
} else {
cmd = "id -u " + name + " | awk '{print \"" + name + ":\"$1 }'";
}
return cmd;
}
// search for name with given id, return "<name>:<id>"
private String getId2NameCmdLinux(final int id, final boolean isGrp) {
String cmd = "getent ";
cmd += isGrp? "group " : "passwd ";
cmd += String.valueOf(id) + " | cut -d: -f1,3";
return cmd;
}
// "dscl . -read /Users/<name> | grep UniqueID" returns "UniqueId: <id>",
// "dscl . -read /Groups/<name> | grep PrimaryGroupID" returns "PrimaryGoupID: <id>"
// The following method returns a command that uses awk to process the result,
// of these commands, and returns "<name> <id>", to simulate one entry returned by
// MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD.
// Specificially, this method returns:
// id -u <name> | awk '{print "<name>:"$1 }'
// OR
// dscl . -read /Groups/<name> | grep PrimaryGroupID | awk '($1 == "PrimaryGroupID:") { print "<name> " $2 }'
//
private String getName2IdCmdMac(final String name, final boolean isGrp) {
String cmd;
if (isGrp) {
cmd = "dscl . -read /Groups/" + name;
cmd += " | grep PrimaryGroupID | awk '($1 == \"PrimaryGroupID:\") ";
cmd += "{ print \"" + name + " \" $2 }'";
} else {
cmd = "id -u " + name + " | awk '{print \"" + name + " \"$1 }'";
}
return cmd;
}
// "dscl . -search /Users UniqueID <id>" returns
// <name> UniqueID = (
// <id>
// )
// "dscl . -search /Groups PrimaryGroupID <id>" returns
// <name> PrimaryGroupID = (
// <id>
// )
// The following method returns a command that uses sed to process the
// the result and returns "<name> <id>" to simulate one entry returned
// by MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD.
// For certain negative id case like nfsnobody, the <id> is quoted as
// "<id>", added one sed section to remove the quote.
// Specifically, the method returns:
// dscl . -search /Users UniqueID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/UniqueID =//g' | sed 's/)//g' | sed 's/\"//g'
// OR
// dscl . -search /Groups PrimaryGroupID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/PrimaryGroupID =//g' | sed 's/)//g' | sed 's/\"//g'
//
private String getId2NameCmdMac(final int id, final boolean isGrp) {
String cmd = "dscl . -search /";
cmd += isGrp? "Groups PrimaryGroupID " : "Users UniqueID ";
cmd += String.valueOf(id);
cmd += " | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/";
cmd += isGrp? "PrimaryGroupID" : "UniqueID";
cmd += " = (//g' | sed 's/)//g' | sed 's/\\\"//g'";
return cmd;
}
synchronized private void updateMapIncr(final String name,
final boolean isGrp) throws IOException {
if (!checkSupportedPlatform()) {
return;
}
if (isInteger(name) && isGrp) {
loadFullGroupMap();
return;
}
boolean updated = false;
if (staticMapping == null) {
initStaticMapping();
}
if (OS.startsWith("Linux")) {
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getName2IdCmdLinux(name, true), ":",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getName2IdCmdLinux(name, false), ":",
staticMapping.uidMapping);
}
} else {
// Mac
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getName2IdCmdMac(name, true), "\\s+",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getName2IdCmdMac(name, false), "\\s+",
staticMapping.uidMapping);
}
}
if (updated) {
lastUpdateTime = Time.monotonicNow();
}
}
synchronized private void updateMapIncr(final int id,
final boolean isGrp) throws IOException {
if (!checkSupportedPlatform()) {
return;
}
boolean updated = false;
if (staticMapping == null) {
initStaticMapping();
}
if (OS.startsWith("Linux")) {
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getId2NameCmdLinux(id, true), ":",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getId2NameCmdLinux(id, false), ":",
staticMapping.uidMapping);
}
} else {
// Mac
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
getId2NameCmdMac(id, true), "\\s+",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
getId2NameCmdMac(id, false), "\\s+",
staticMapping.uidMapping);
}
}
if (updated) {
lastUpdateTime = Time.monotonicNow();
}
}
@SuppressWarnings("serial")
static final class PassThroughMap<K> extends HashMap<K, K> {
@ -334,9 +593,13 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
checkAndUpdateMaps();
Integer id = uidNameMap.inverse().get(user);
if (id == null) {
updateMapIncr(user, false);
id = uidNameMap.inverse().get(user);
if (id == null) {
throw new IOException("User just deleted?:" + user);
}
}
return id.intValue();
}
@ -344,9 +607,12 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
checkAndUpdateMaps();
Integer id = gidNameMap.inverse().get(group);
if (id == null) {
updateMapIncr(group, true);
id = gidNameMap.inverse().get(group);
if (id == null) {
throw new IOException("No such group:" + group);
}
}
return id.intValue();
}
@ -354,22 +620,36 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
synchronized public String getUserName(int uid, String unknown) {
checkAndUpdateMaps();
String uname = uidNameMap.get(uid);
if (uname == null) {
try {
updateMapIncr(uid, false);
} catch (Exception e) {
}
uname = uidNameMap.get(uid);
if (uname == null) {
LOG.warn("Can't find user name for uid " + uid
+ ". Use default user name " + unknown);
uname = unknown;
}
}
return uname;
}
synchronized public String getGroupName(int gid, String unknown) {
checkAndUpdateMaps();
String gname = gidNameMap.get(gid);
if (gname == null) {
try {
updateMapIncr(gid, true);
} catch (Exception e) {
}
gname = gidNameMap.get(gid);
if (gname == null) {
LOG.warn("Can't find group name for gid " + gid
+ ". Use default group name " + unknown);
gname = unknown;
}
}
return gname;
}

61
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
View File

@ -219,4 +219,65 @@ public class TestShellBasedIdMapping {
assertEquals(iug.getTimeout(),
IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT * 2);
}
@Test
public void testUpdateMapIncr() throws IOException {
Configuration conf = new Configuration();
conf.setLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, 600000);
ShellBasedIdMapping refIdMapping =
new ShellBasedIdMapping(conf, true);
ShellBasedIdMapping incrIdMapping = new ShellBasedIdMapping(conf);
// Command such as "getent passwd <userName>" will return empty string if
// <username> is numerical, remove them from the map for testing purpose.
BiMap<Integer, String> uidNameMap = refIdMapping.getUidNameMap();
BiMap<Integer, String> gidNameMap = refIdMapping.getGidNameMap();
// Force empty map, to see effect of incremental map update of calling
// getUserName()
incrIdMapping.clearNameMaps();
uidNameMap = refIdMapping.getUidNameMap();
for (BiMap.Entry<Integer, String> me : uidNameMap.entrySet()) {
Integer id = me.getKey();
String name = me.getValue();
String tname = incrIdMapping.getUserName(id, null);
assertEquals(name, tname);
}
assertEquals(uidNameMap.size(), incrIdMapping.getUidNameMap().size());
// Force empty map, to see effect of incremental map update of calling
// getUid()
incrIdMapping.clearNameMaps();
for (BiMap.Entry<Integer, String> me : uidNameMap.entrySet()) {
Integer id = me.getKey();
String name = me.getValue();
Integer tid = incrIdMapping.getUid(name);
assertEquals(id, tid);
}
assertEquals(uidNameMap.size(), incrIdMapping.getUidNameMap().size());
// Force empty map, to see effect of incremental map update of calling
// getGroupName()
incrIdMapping.clearNameMaps();
gidNameMap = refIdMapping.getGidNameMap();
for (BiMap.Entry<Integer, String> me : gidNameMap.entrySet()) {
Integer id = me.getKey();
String name = me.getValue();
String tname = incrIdMapping.getGroupName(id, null);
assertEquals(name, tname);
}
assertEquals(gidNameMap.size(), incrIdMapping.getGidNameMap().size());
// Force empty map, to see effect of incremental map update of calling
// getGid()
incrIdMapping.clearNameMaps();
gidNameMap = refIdMapping.getGidNameMap();
for (BiMap.Entry<Integer, String> me : gidNameMap.entrySet()) {
Integer id = me.getKey();
String name = me.getValue();
Integer tid = incrIdMapping.getGid(name);
assertEquals(id, tid);
}
assertEquals(gidNameMap.size(), incrIdMapping.getGidNameMap().size());
}
}

3
hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
View File

@ -173,8 +173,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
this.config = config;
config.set(FsPermission.UMASK_LABEL, "000");
iug = new ShellBasedIdMapping(config,
IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
iug = new ShellBasedIdMapping(config);
aixCompatMode = config.getBoolean(
NfsConfigKeys.AIX_COMPAT_MODE_KEY,

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -438,6 +438,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7399. Lack of synchronization in
DFSOutputStream#Packet#getLastByteOffsetBlock() (vinayakumarb)
HDFS-7146. NFS ID/Group lookup requires SSSD enumeration on the server
(Yongjun Zhang via brandonli)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES