MAPREDUCE-7010. Make Job History File Permissions configurable. Contributed by Gergely Novák

This commit is contained in:
Billie Rinaldi 2018-02-26 14:32:46 -08:00
parent 78a10029ec
commit 7dd385098c
5 changed files with 70 additions and 9 deletions

View File

@ -231,8 +231,8 @@ public class JobHistoryEventHandler extends AbstractService
try {
doneDirPrefixPath =
FileContext.getFileContext(conf).makeQualified(new Path(userDoneDirStr));
mkdir(doneDirFS, doneDirPrefixPath, new FsPermission(
JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS));
mkdir(doneDirFS, doneDirPrefixPath, JobHistoryUtils.
getConfiguredHistoryIntermediateUserDoneDirPermissions(conf));
} catch (IOException e) {
LOG.error("Error creating user intermediate history done directory: [ "
+ doneDirPrefixPath + "]", e);

View File

@ -93,6 +93,10 @@ public class JHAdminConfig {
**/
public static final String MR_HISTORY_INTERMEDIATE_DONE_DIR =
MR_HISTORY_PREFIX + "intermediate-done-dir";
public static final String MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS =
MR_HISTORY_PREFIX + "intermediate-user-done-dir.permissions";
public static final short
DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = 0770;
/** Size of the job list cache.*/
public static final String MR_HISTORY_JOBLIST_CACHE_SIZE =

View File

@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
@ -89,13 +90,7 @@ public class JobHistoryUtils {
*/
public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 01777);
/**
* Permissions for the user directory under the intermediate done directory.
*/
public static final FsPermission HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 0770);
public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS =
FsPermission.createImmutable((short) 0770); // rwx------
@ -208,6 +203,35 @@ public class JobHistoryUtils {
}
return ensurePathInDefaultFileSystem(doneDirPrefix, conf);
}
/**
* Gets the configured directory permissions for the user directories in the
* directory of the intermediate done history files. The user and the group
* both need full permissions, this is enforced by this method.
* @param conf The configuration object
* @return FsPermission of the user directories
*/
public static FsPermission
getConfiguredHistoryIntermediateUserDoneDirPermissions(
Configuration conf) {
String userDoneDirPermissions = conf.get(
JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS);
if (userDoneDirPermissions == null) {
return new FsPermission(
JHAdminConfig.DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS);
}
FsPermission permission = new FsPermission(userDoneDirPermissions);
if (permission.getUserAction() != FsAction.ALL ||
permission.getGroupAction() != FsAction.ALL) {
permission = new FsPermission(FsAction.ALL, FsAction.ALL,
permission.getOtherAction(), permission.getStickyBit());
LOG.warn("Unsupported permission configured in " +
JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS +
", the user and the group permission must be 7 (rwx). " +
"The permission was set to " + permission.toString());
}
return permission;
}
/**
* Gets the configured directory prefix for Done history files.

View File

@ -24,6 +24,10 @@ import java.io.IOException;
import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@ -31,6 +35,9 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.Assert;
import org.junit.Test;
import static org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions;
public class TestJobHistoryUtils {
final static String TEST_DIR = new File(System.getProperty("test.build.data"))
@ -140,4 +147,21 @@ public class TestJobHistoryUtils {
fc.mkdir(path, FsPermission.getDirDefault(), true);
return path;
}
@Test
public void testGetConfiguredHistoryIntermediateUserDoneDirPermissions() {
Configuration conf = new Configuration();
Map<String, FsPermission> parameters = ImmutableMap.of(
"775", new FsPermission(0775),
"123", new FsPermission(0773),
"-rwx", new FsPermission(0770) ,
"+rwx", new FsPermission(0777)
);
for (Map.Entry<String, FsPermission> entry : parameters.entrySet()) {
conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS,
entry.getKey());
Assert.assertEquals(entry.getValue(),
getConfiguredHistoryIntermediateUserDoneDirPermissions(conf));
}
}
}

View File

@ -1730,6 +1730,15 @@
<description></description>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-user-done-dir.permissions</name>
<value>770</value>
<description>The permissions of the user directories in
${mapreduce.jobhistory.intermediate-done-dir}. The user and the group
permission must be 7, this is enforced.
</description>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>${yarn.app.mapreduce.am.staging-dir}/history/done</value>