MAPREDUCE-7010. Make Job History File Permissions configurable. Contributed by Gergely Novák
This commit is contained in:
parent
78a10029ec
commit
7dd385098c
|
@ -231,8 +231,8 @@ public class JobHistoryEventHandler extends AbstractService
|
|||
try {
|
||||
doneDirPrefixPath =
|
||||
FileContext.getFileContext(conf).makeQualified(new Path(userDoneDirStr));
|
||||
mkdir(doneDirFS, doneDirPrefixPath, new FsPermission(
|
||||
JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS));
|
||||
mkdir(doneDirFS, doneDirPrefixPath, JobHistoryUtils.
|
||||
getConfiguredHistoryIntermediateUserDoneDirPermissions(conf));
|
||||
} catch (IOException e) {
|
||||
LOG.error("Error creating user intermediate history done directory: [ "
|
||||
+ doneDirPrefixPath + "]", e);
|
||||
|
|
|
@ -93,6 +93,10 @@ public class JHAdminConfig {
|
|||
**/
|
||||
public static final String MR_HISTORY_INTERMEDIATE_DONE_DIR =
|
||||
MR_HISTORY_PREFIX + "intermediate-done-dir";
|
||||
public static final String MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS =
|
||||
MR_HISTORY_PREFIX + "intermediate-user-done-dir.permissions";
|
||||
public static final short
|
||||
DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = 0770;
|
||||
|
||||
/** Size of the job list cache.*/
|
||||
public static final String MR_HISTORY_JOBLIST_CACHE_SIZE =
|
||||
|
|
|
@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path;
|
|||
import org.apache.hadoop.fs.PathFilter;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
||||
import org.apache.hadoop.fs.permission.FsAction;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.mapreduce.JobID;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
|
@ -89,13 +90,7 @@ public class JobHistoryUtils {
|
|||
*/
|
||||
public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS =
|
||||
FsPermission.createImmutable((short) 01777);
|
||||
|
||||
/**
|
||||
* Permissions for the user directory under the intermediate done directory.
|
||||
*/
|
||||
public static final FsPermission HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS =
|
||||
FsPermission.createImmutable((short) 0770);
|
||||
|
||||
|
||||
public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS =
|
||||
FsPermission.createImmutable((short) 0770); // rwx------
|
||||
|
||||
|
@ -208,6 +203,35 @@ public class JobHistoryUtils {
|
|||
}
|
||||
return ensurePathInDefaultFileSystem(doneDirPrefix, conf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the configured directory permissions for the user directories in the
|
||||
* directory of the intermediate done history files. The user and the group
|
||||
* both need full permissions, this is enforced by this method.
|
||||
* @param conf The configuration object
|
||||
* @return FsPermission of the user directories
|
||||
*/
|
||||
public static FsPermission
|
||||
getConfiguredHistoryIntermediateUserDoneDirPermissions(
|
||||
Configuration conf) {
|
||||
String userDoneDirPermissions = conf.get(
|
||||
JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS);
|
||||
if (userDoneDirPermissions == null) {
|
||||
return new FsPermission(
|
||||
JHAdminConfig.DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS);
|
||||
}
|
||||
FsPermission permission = new FsPermission(userDoneDirPermissions);
|
||||
if (permission.getUserAction() != FsAction.ALL ||
|
||||
permission.getGroupAction() != FsAction.ALL) {
|
||||
permission = new FsPermission(FsAction.ALL, FsAction.ALL,
|
||||
permission.getOtherAction(), permission.getStickyBit());
|
||||
LOG.warn("Unsupported permission configured in " +
|
||||
JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS +
|
||||
", the user and the group permission must be 7 (rwx). " +
|
||||
"The permission was set to " + permission.toString());
|
||||
}
|
||||
return permission;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the configured directory prefix for Done history files.
|
||||
|
|
|
@ -24,6 +24,10 @@ import java.io.IOException;
|
|||
import java.util.Calendar;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
@ -31,6 +35,9 @@ import org.apache.hadoop.fs.permission.FsPermission;
|
|||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions;
|
||||
|
||||
|
||||
public class TestJobHistoryUtils {
|
||||
|
||||
final static String TEST_DIR = new File(System.getProperty("test.build.data"))
|
||||
|
@ -140,4 +147,21 @@ public class TestJobHistoryUtils {
|
|||
fc.mkdir(path, FsPermission.getDirDefault(), true);
|
||||
return path;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetConfiguredHistoryIntermediateUserDoneDirPermissions() {
|
||||
Configuration conf = new Configuration();
|
||||
Map<String, FsPermission> parameters = ImmutableMap.of(
|
||||
"775", new FsPermission(0775),
|
||||
"123", new FsPermission(0773),
|
||||
"-rwx", new FsPermission(0770) ,
|
||||
"+rwx", new FsPermission(0777)
|
||||
);
|
||||
for (Map.Entry<String, FsPermission> entry : parameters.entrySet()) {
|
||||
conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS,
|
||||
entry.getKey());
|
||||
Assert.assertEquals(entry.getValue(),
|
||||
getConfiguredHistoryIntermediateUserDoneDirPermissions(conf));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1730,6 +1730,15 @@
|
|||
<description></description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jobhistory.intermediate-user-done-dir.permissions</name>
|
||||
<value>770</value>
|
||||
<description>The permissions of the user directories in
|
||||
${mapreduce.jobhistory.intermediate-done-dir}. The user and the group
|
||||
permission must be 7, this is enforced.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>mapreduce.jobhistory.done-dir</name>
|
||||
<value>${yarn.app.mapreduce.am.staging-dir}/history/done</value>
|
||||
|
|
Loading…
Reference in New Issue