Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3269. Yarn.nodemanager.remote-app-log-dir could not be configured to fully qualified path. Contributed by Xuan Gong 

(cherry picked from commit d81109e588)

(cherry picked from commit f40f17489c)
This commit is contained in:
Junping Du 2015-03-20 13:41:22 -07:00 committed by Zhe Zhang
parent f5a4f69e9d
commit 086ee0adf8
5 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -67,6 +67,9 @@ Release 2.7.4 - UNRELEASED
YARN-4355. NPE while processing localizer heartbeat. Contributed by YARN-4355. NPE while processing localizer heartbeat. Contributed by
Varun Saxena & Jonathan Hung Varun Saxena & Jonathan Hung
YARN-3269. Yarn.nodemanager.remote-app-log-dir could not be configured to
fully qualified path. (Xuan Gong via junping_du)
Release 2.7.3 - 2016-08-25 Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -379,7 +379,7 @@ public class AggregatedLogFormat {
userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() { userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() {
@Override @Override
public FSDataOutputStream run() throws Exception { public FSDataOutputStream run() throws Exception {
fc = FileContext.getFileContext(conf); fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
fc.setUMask(APP_LOG_FILE_UMASK); fc.setUMask(APP_LOG_FILE_UMASK);
return fc.create( return fc.create(
remoteAppLogFile, remoteAppLogFile,
@ -471,7 +471,8 @@ public class AggregatedLogFormat {
public LogReader(Configuration conf, Path remoteAppLogFile) public LogReader(Configuration conf, Path remoteAppLogFile)
throws IOException { throws IOException {
FileContext fileContext = FileContext.getFileContext(conf); FileContext fileContext =
FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
this.fsDataIStream = fileContext.open(remoteAppLogFile); this.fsDataIStream = fileContext.open(remoteAppLogFile);
reader = reader =
new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus(

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
View File

@ -304,7 +304,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
userUgi.doAs(new PrivilegedExceptionAction<Object>() { userUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override @Override
public Object run() throws Exception { public Object run() throws Exception {
FileSystem remoteFS = FileSystem.get(conf); FileSystem remoteFS = remoteNodeLogFileForApp.getFileSystem(conf);
if (remoteFS.exists(remoteNodeTmpLogFileForApp)) { if (remoteFS.exists(remoteNodeTmpLogFileForApp)) {
if (rename) { if (rename) {
remoteFS.rename(remoteNodeTmpLogFileForApp, renamedPath); remoteFS.rename(remoteNodeTmpLogFileForApp, renamedPath);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
View File

@ -176,7 +176,7 @@ public class LogAggregationService extends AbstractService implements
} }
protected FileSystem getFileSystem(Configuration conf) throws IOException { protected FileSystem getFileSystem(Configuration conf) throws IOException {
return FileSystem.get(conf); return this.remoteRootLogDir.getFileSystem(conf);
} }
void verifyAndCreateRemoteLogDir(Configuration conf) { void verifyAndCreateRemoteLogDir(Configuration conf) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
View File

@ -1392,8 +1392,10 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
logAggregationContextWithInterval.setRolledLogsIncludePattern(".*"); logAggregationContextWithInterval.setRolledLogsIncludePattern(".*");
logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final"); logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final");
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
//configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
//have fully qualified path
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
this.remoteRootLogDir.getAbsolutePath()); "file://" + this.remoteRootLogDir.getAbsolutePath());
this.conf.setLong( this.conf.setLong(
YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
3600); 3600);