diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index fc9254af97a..9023f6a11ad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -1481,7 +1481,7 @@ public class JobHistoryEventHandler extends AbstractService summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString()); summaryFileOut.close(); doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission( - JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS)); + JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions(getConfig()))); } catch (IOException e) { LOG.info("Unable to write out JobSummaryInfo to [" + qualifiedSummaryDoneFile + "]", e); @@ -1738,8 +1738,9 @@ public class JobHistoryEventHandler extends AbstractService boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath, false, getConfig()); - doneDirFS.setPermission(toPath, new FsPermission( - JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS)); + doneDirFS.setPermission(toPath, new FsPermission(JobHistoryUtils. + getConfiguredHistoryIntermediateUserDoneDirPermissions( + getConfig()))); if (copied) { LOG.info("Copied from: " + fromPath.toString() + " to done location: " + toPath.toString()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index fbeb94a2a16..8159bc2456c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.jobhistory; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -43,6 +44,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapreduce.CounterGroup; @@ -1036,6 +1038,48 @@ public class TestJobHistoryEventHandler { jheh.stop(); } } + + @Test(timeout = 50000) + public void testJobHistoryFilePermissions() throws Exception { + TestParams t = new TestParams(true); + Configuration conf = new Configuration(); + String setFilePermission = "777"; + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, setFilePermission); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString()); + + JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, + 0, false); + JHEvenHandlerForTest jheh = spy(realJheh); + jheh.init(conf); + + try { + jheh.start(); + handleEvent(jheh, new JobHistoryEvent(t.jobId, + new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost", + 3000, 4000, -1))); + + // Job finishes and successfully writes history + handleEvent(jheh, new JobHistoryEvent(t.jobId, + new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0, + 0, 0, 0, 0, 0, + new Counters(), + new Counters(), new Counters()))); + + verify(jheh, times(1)).processDoneFiles(any(JobId.class)); + + String intermediateSummaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(t.jobId); + String doneDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf); + FileSystem fs = FileSystem.get(dfsCluster.getConfiguration(0)); + Path intermediateSummaryFileNamePath = new Path(doneDir, intermediateSummaryFileName); + FsPermission getIntermediateSummaryFilePermission = + fs.getFileStatus(intermediateSummaryFileNamePath).getPermission(); + assertEquals(setFilePermission, + String.valueOf(getIntermediateSummaryFilePermission.toOctal())); + } finally { + jheh.stop(); + } + } } class JHEvenHandlerForTest extends JobHistoryEventHandler { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index e22726c303a..1a1df010dd8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -91,9 +91,6 @@ public class JobHistoryUtils { public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = FsPermission.createImmutable((short) 01777); - public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = - FsPermission.createImmutable((short) 0770); // rwx------ - /** * Suffix for configuration files. */ @@ -206,7 +203,7 @@ public class JobHistoryUtils { /** * Gets the configured directory permissions for the user directories in the - * directory of the intermediate done history files. The user and the group + * Gets the configured permissions for the user directories and files in the * both need full permissions, this is enforced by this method. * @param conf The configuration object * @return FsPermission of the user directories