HDFS-7146. NFS ID/Group lookup requires SSSD enumeration on the server. Contributed by Yongjun Zhang

(cherry picked from commit 351c5561c2)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java

@@ -40,6 +40,23 @@ import com.google.common.collect.HashBiMap;
  * A simple shell-based implementation of {@link IdMappingServiceProvider} 
  * Map id to user name or group name. It does update every 15 minutes. Only a
  * single instance of this class is expected to be on the server.
+ * 
+ * The maps are incrementally updated as described below:
+ *   1. Initialize the maps as empty. 
+ *   2. Incrementally update the maps
+ *      - When ShellBasedIdMapping is requested for user or group name given 
+ *        an ID, or for ID given a user or group name, do look up in the map
+ *        first, if it doesn't exist, find the corresponding entry with shell
+ *        command, and insert the entry to the maps.
+ *      - When group ID is requested for a given group name, and if the
+ *        group name is numerical, the full group map is loaded. Because we
+ *        don't have a good way to find the entry for a numerical group name,
+ *        loading the full map helps to get in all entries.
+ *   3. Periodically refresh the maps for both user and group, e.g,
+ *      do step 1.
+ *   Note: for testing purpose, step 1 may initial the maps with full mapping
+ *   when using constructor
+ *   {@link ShellBasedIdMapping#ShellBasedIdMapping(Configuration, boolean)}.
  */
 public class ShellBasedIdMapping implements IdMappingServiceProvider {
 
@@ -55,6 +72,8 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID";
 
   private final File staticMappingFile;
+  private StaticMapping staticMapping = null;
+  private boolean constructFullMapAtInit = false;
 
   // Used for parsing the static mapping file.
   private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$");
@@ -70,8 +89,17 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
 
   private long lastUpdateTime = 0; // Last time maps were updated
 
+  /*
+   * Constructor
+   * @param conf the configuration
+   * @param constructFullMapAtInit initialize the maps with full mapping when
+   *        true, otherwise initialize the maps to empty. This parameter is
+   *        intended for testing only, its default is false.
+   */
+  @VisibleForTesting
   public ShellBasedIdMapping(Configuration conf,
-      final String defaultStaticIdMappingFile) throws IOException {
+      boolean constructFullMapAtInit) throws IOException {
+    this.constructFullMapAtInit = constructFullMapAtInit;
     long updateTime = conf.getLong(
         IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY,
         IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT);
@@ -84,15 +112,21 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
       timeout = updateTime;
     }
     
-    String staticFilePath = conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
+    String staticFilePath = 
-        defaultStaticIdMappingFile);
+        conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
+            IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
     staticMappingFile = new File(staticFilePath);
 
     updateMaps();
   }
 
+  /*
+   * Constructor
+   * initialize user and group maps to empty
+   * @param conf the configuration
+   */
   public ShellBasedIdMapping(Configuration conf) throws IOException {
-    this(conf, IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
+    this(conf, false);
   }
 
   @VisibleForTesting
@@ -100,6 +134,23 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
     return timeout;
   }
 
+  @VisibleForTesting
+  public BiMap<Integer, String> getUidNameMap() {
+    return uidNameMap;
+  }
+
+  @VisibleForTesting
+  public BiMap<Integer, String> getGidNameMap() {
+    return gidNameMap;
+  }
+
+  @VisibleForTesting  
+  synchronized public void clearNameMaps() {
+    uidNameMap.clear();
+    gidNameMap.clear();
+    lastUpdateTime = Time.monotonicNow();
+  }  
+
   synchronized private boolean isExpired() {
     return Time.monotonicNow() - lastUpdateTime > timeout;
   }
@@ -153,13 +204,15 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   }
   
   /**
-   * Get the whole list of users and groups and save them in the maps.
+   * Get the list of users or groups returned by the specified command,
+   * and save them in the corresponding map.
    * @throws IOException 
    */
   @VisibleForTesting
-  public static void updateMapInternal(BiMap<Integer, String> map, String mapName,
+  public static boolean updateMapInternal(BiMap<Integer, String> map,
-      String command, String regex, Map<Integer, Integer> staticMapping)
+      String mapName, String command, String regex,
-      throws IOException  {
+      Map<Integer, Integer> staticMapping) throws IOException  {
+    boolean updated = false;
     BufferedReader br = null;
     try {
       Process process = Runtime.getRuntime().exec(
@@ -194,8 +247,9 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
           continue;
         }
         map.put(key, value);
+        updated = true;
       }
-      LOG.info("Updated " + mapName + " map size: " + map.size());
+      LOG.debug("Updated " + mapName + " map size: " + map.size());
       
     } catch (IOException e) {
       LOG.error("Can't update " + mapName + " map");
@@ -209,20 +263,31 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
         }
       }
     }
+    return updated;
   }
 
-  synchronized public void updateMaps() throws IOException {
+  private boolean checkSupportedPlatform() {
-    BiMap<Integer, String> uMap = HashBiMap.create();
-    BiMap<Integer, String> gMap = HashBiMap.create();
-
     if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) {
       LOG.error("Platform is not supported:" + OS
           + ". Can't update user map and group map and"
           + " 'nobody' will be used for any user and group.");
-      return;
+      return false;
+    }
+    return true;
   }
 
-    StaticMapping staticMapping = new StaticMapping(
+  private static boolean isInteger(final String s) {
+    try { 
+      Integer.parseInt(s); 
+    } catch(NumberFormatException e) { 
+      return false; 
+    }
+    // only got here if we didn't return false
+    return true;
+  }
+
+  private void initStaticMapping() throws IOException {
+    staticMapping = new StaticMapping(
         new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>());
     if (staticMappingFile.exists()) {
       LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping...");
@@ -231,24 +296,218 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
       LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile
           + "' does not exist.");
     }
+  }  
 
+  /*
+   * Reset the maps to empty.
+   * For testing code, a full map may be re-constructed here when the object
+   * was created with constructFullMapAtInit being set to true.
+   */
+  synchronized public void updateMaps() throws IOException {
+    if (!checkSupportedPlatform()) {
+      return;
+    }
+
+    if (constructFullMapAtInit) {
+      loadFullMaps();
+    } else {
+      clearNameMaps();
+    }
+  }
+  
+  synchronized private void loadFullUserMap() throws IOException {
+    if (staticMapping == null) {
+      initStaticMapping();
+    }
+    BiMap<Integer, String> uMap = HashBiMap.create();
     if (OS.startsWith("Mac")) {
       updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
           staticMapping.uidMapping);
-      updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
-          staticMapping.gidMapping);
     } else {
       updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":",
           staticMapping.uidMapping);
+    }
+    uidNameMap = uMap;
+    lastUpdateTime = Time.monotonicNow();
+  }
+
+  synchronized private void loadFullGroupMap() throws IOException {
+    if (staticMapping == null) {
+      initStaticMapping();
+    }
+    BiMap<Integer, String> gMap = HashBiMap.create();
+
+    if (OS.startsWith("Mac")) {
+      updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
+          staticMapping.gidMapping);
+    } else {
       updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":",
           staticMapping.gidMapping);
     }
-
-    uidNameMap = uMap;
     gidNameMap = gMap;
     lastUpdateTime = Time.monotonicNow();
   }
 
+  synchronized private void loadFullMaps() throws IOException {
+    initStaticMapping();
+    loadFullUserMap();
+    loadFullGroupMap();
+  }
+
+  // search for id with given name, return "<name>:<id>"
+  // return
+  //     getent group <name> | cut -d: -f1,3
+  // OR
+  //     id -u <name> | awk '{print "<name>:"$1 }'
+  //
+  private String getName2IdCmdLinux(final String name, final boolean isGrp) {
+    String cmd;
+    if (isGrp) {
+      cmd = "getent group " + name + " | cut -d: -f1,3";   
+    } else {
+      cmd = "id -u " + name + " | awk '{print \"" + name + ":\"$1 }'";
+    }
+    return cmd;
+  }
+  
+  // search for name with given id, return "<name>:<id>"
+  private String getId2NameCmdLinux(final int id, final boolean isGrp) {
+    String cmd = "getent ";
+    cmd += isGrp? "group " : "passwd ";
+    cmd += String.valueOf(id) + " | cut -d: -f1,3";
+    return cmd;
+  }
+
+  // "dscl . -read /Users/<name> | grep UniqueID" returns "UniqueId: <id>",
+  // "dscl . -read /Groups/<name> | grep PrimaryGroupID" returns "PrimaryGoupID: <id>"
+  // The following method returns a command that uses awk to process the result,
+  // of these commands, and returns "<name> <id>", to simulate one entry returned by 
+  // MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD.
+  // Specificially, this method returns:
+  // id -u <name> | awk '{print "<name>:"$1 }'
+  // OR
+  // dscl . -read /Groups/<name> | grep PrimaryGroupID | awk '($1 == "PrimaryGroupID:") { print "<name> " $2 }'
+  //
+  private String getName2IdCmdMac(final String name, final boolean isGrp) {
+    String cmd;
+    if (isGrp) {
+      cmd = "dscl . -read /Groups/" + name;
+      cmd += " | grep PrimaryGroupID | awk '($1 == \"PrimaryGroupID:\") ";
+      cmd += "{ print \"" + name + "  \" $2 }'";
+    } else {
+      cmd = "id -u " + name + " | awk '{print \"" + name + "  \"$1 }'";
+    }
+    return cmd;
+  }
+
+  // "dscl . -search /Users UniqueID <id>" returns 
+  //    <name> UniqueID = (
+  //      <id>
+  //    )
+  // "dscl . -search /Groups PrimaryGroupID <id>" returns
+  //    <name> PrimaryGroupID = (
+  //      <id>
+  //    )
+  // The following method returns a command that uses sed to process the
+  // the result and returns "<name> <id>" to simulate one entry returned
+  // by MAC_GET_ALL_USERS_CMD or MAC_GET_ALL_GROUPS_CMD.
+  // For certain negative id case like nfsnobody, the <id> is quoted as
+  // "<id>", added one sed section to remove the quote.
+  // Specifically, the method returns:
+  // dscl . -search /Users UniqueID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/UniqueID =//g' | sed 's/)//g' | sed 's/\"//g'
+  // OR
+  // dscl . -search /Groups PrimaryGroupID <id> | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/PrimaryGroupID =//g' | sed 's/)//g' | sed 's/\"//g'
+  //
+  private String getId2NameCmdMac(final int id, final boolean isGrp) {
+    String cmd = "dscl . -search /";
+    cmd += isGrp? "Groups PrimaryGroupID " : "Users UniqueID ";
+    cmd += String.valueOf(id);
+    cmd += " | sed 'N;s/\\n//g;N;s/\\n//g' | sed 's/";
+    cmd += isGrp? "PrimaryGroupID" : "UniqueID";
+    cmd += " = (//g' | sed 's/)//g' | sed 's/\\\"//g'";
+    return cmd;
+  }
+
+  synchronized private void updateMapIncr(final String name,
+      final boolean isGrp) throws IOException {
+    if (!checkSupportedPlatform()) {
+      return;
+    }
+    if (isInteger(name) && isGrp) {
+      loadFullGroupMap();
+      return;
+    }
+
+    boolean updated = false;
+    if (staticMapping == null) {
+      initStaticMapping();
+    }
+
+    if (OS.startsWith("Linux")) {
+      if (isGrp) {
+        updated = updateMapInternal(gidNameMap, "group",
+            getName2IdCmdLinux(name, true), ":",
+            staticMapping.gidMapping);
+      } else {
+        updated = updateMapInternal(uidNameMap, "user",
+            getName2IdCmdLinux(name, false), ":",
+            staticMapping.uidMapping);
+      }
+    } else {
+      // Mac
+      if (isGrp) {        
+        updated = updateMapInternal(gidNameMap, "group",
+            getName2IdCmdMac(name, true), "\\s+",
+            staticMapping.gidMapping);
+      } else {
+        updated = updateMapInternal(uidNameMap, "user",
+            getName2IdCmdMac(name, false), "\\s+",
+            staticMapping.uidMapping);
+      }
+    }
+    if (updated) {
+      lastUpdateTime = Time.monotonicNow();
+    }
+  }
+
+  synchronized private void updateMapIncr(final int id,
+      final boolean isGrp) throws IOException {
+    if (!checkSupportedPlatform()) {
+      return;
+    }
+    
+    boolean updated = false;
+    if (staticMapping == null) {
+      initStaticMapping();
+    }
+
+    if (OS.startsWith("Linux")) {
+      if (isGrp) {
+        updated = updateMapInternal(gidNameMap, "group",
+            getId2NameCmdLinux(id, true), ":",
+            staticMapping.gidMapping);
+      } else {
+        updated = updateMapInternal(uidNameMap, "user",
+            getId2NameCmdLinux(id, false), ":",
+            staticMapping.uidMapping);
+      }
+    } else {
+      // Mac
+      if (isGrp) {
+        updated = updateMapInternal(gidNameMap, "group",
+            getId2NameCmdMac(id, true), "\\s+",
+            staticMapping.gidMapping);
+      } else {
+        updated = updateMapInternal(uidNameMap, "user",
+            getId2NameCmdMac(id, false), "\\s+",
+            staticMapping.uidMapping);
+      }
+    }
+    if (updated) {
+      lastUpdateTime = Time.monotonicNow();
+    }
+  }
+
   @SuppressWarnings("serial")
   static final class PassThroughMap<K> extends HashMap<K, K> {
     
@@ -334,9 +593,13 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
     checkAndUpdateMaps();
 
     Integer id = uidNameMap.inverse().get(user);
+    if (id == null) {
+      updateMapIncr(user, false);
+      id = uidNameMap.inverse().get(user);
       if (id == null) {
         throw new IOException("User just deleted?:" + user);
       }
+    }
     return id.intValue();
   }
 
@@ -344,9 +607,12 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
     checkAndUpdateMaps();
 
     Integer id = gidNameMap.inverse().get(group);
+    if (id == null) {
+      updateMapIncr(group, true);
+      id = gidNameMap.inverse().get(group);
       if (id == null) {
         throw new IOException("No such group:" + group);
-
+      }
     }
     return id.intValue();
   }
@@ -354,22 +620,36 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   synchronized public String getUserName(int uid, String unknown) {
     checkAndUpdateMaps();
     String uname = uidNameMap.get(uid);
+    if (uname == null) {
+      try {
+        updateMapIncr(uid, false);
+      } catch (Exception e) {        
+      }
+      uname = uidNameMap.get(uid);
       if (uname == null) {     
         LOG.warn("Can't find user name for uid " + uid
             + ". Use default user name " + unknown);
         uname = unknown;
       }
+    }
     return uname;
   }
 
   synchronized public String getGroupName(int gid, String unknown) {
     checkAndUpdateMaps();
     String gname = gidNameMap.get(gid);
+    if (gname == null) {
+      try {
+        updateMapIncr(gid, true);
+      } catch (Exception e) {        
+      }
+      gname = gidNameMap.get(gid);
       if (gname == null) {
         LOG.warn("Can't find group name for gid " + gid
             + ". Use default group name " + unknown);
         gname = unknown;
       }
+    }
     return gname;
   }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java

@@ -219,4 +219,65 @@ public class TestShellBasedIdMapping {
     assertEquals(iug.getTimeout(),
         IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT * 2);
   }
+  
+  @Test
+  public void testUpdateMapIncr() throws IOException {
+    Configuration conf = new Configuration();
+    conf.setLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, 600000);
+    ShellBasedIdMapping refIdMapping =
+        new ShellBasedIdMapping(conf, true);
+    ShellBasedIdMapping incrIdMapping = new ShellBasedIdMapping(conf);
+
+    // Command such as "getent passwd <userName>" will return empty string if
+    // <username> is numerical, remove them from the map for testing purpose.
+    BiMap<Integer, String> uidNameMap = refIdMapping.getUidNameMap();
+    BiMap<Integer, String> gidNameMap = refIdMapping.getGidNameMap();
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getUserName()
+    incrIdMapping.clearNameMaps();
+    uidNameMap = refIdMapping.getUidNameMap();
+    for (BiMap.Entry<Integer, String> me : uidNameMap.entrySet()) {
+      Integer id = me.getKey();
+      String name = me.getValue();
+      String tname = incrIdMapping.getUserName(id, null);
+      assertEquals(name, tname);
+    }
+    assertEquals(uidNameMap.size(), incrIdMapping.getUidNameMap().size());
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getUid()
+    incrIdMapping.clearNameMaps();
+    for (BiMap.Entry<Integer, String> me : uidNameMap.entrySet()) {
+      Integer id = me.getKey();
+      String name = me.getValue();
+      Integer tid = incrIdMapping.getUid(name);
+      assertEquals(id, tid);
+    }
+    assertEquals(uidNameMap.size(), incrIdMapping.getUidNameMap().size());
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getGroupName()
+    incrIdMapping.clearNameMaps();
+    gidNameMap = refIdMapping.getGidNameMap();
+    for (BiMap.Entry<Integer, String> me : gidNameMap.entrySet()) {
+      Integer id = me.getKey();
+      String name = me.getValue();
+      String tname = incrIdMapping.getGroupName(id, null);
+      assertEquals(name, tname);
+    }
+    assertEquals(gidNameMap.size(), incrIdMapping.getGidNameMap().size());
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getGid()
+    incrIdMapping.clearNameMaps();
+    gidNameMap = refIdMapping.getGidNameMap();
+    for (BiMap.Entry<Integer, String> me : gidNameMap.entrySet()) {
+      Integer id = me.getKey();
+      String name = me.getValue();
+      Integer tid = incrIdMapping.getGid(name);
+      assertEquals(id, tid);
+    }
+    assertEquals(gidNameMap.size(), incrIdMapping.getGidNameMap().size());
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java

@@ -173,8 +173,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
 
     this.config = config;
     config.set(FsPermission.UMASK_LABEL, "000");
-    iug = new ShellBasedIdMapping(config,
+    iug = new ShellBasedIdMapping(config);
-        IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
 
     aixCompatMode = config.getBoolean(
         NfsConfigKeys.AIX_COMPAT_MODE_KEY,

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -183,6 +183,9 @@ Release 2.7.0 - UNRELEASED
     HDFS-7399. Lack of synchronization in
     DFSOutputStream#Packet#getLastByteOffsetBlock() (vinayakumarb)
 
+    HDFS-7146. NFS ID/Group lookup requires SSSD enumeration on the server
+    (Yongjun Zhang via brandonli)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES