Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3269. Yarn.nodemanager.remote-app-log-dir could not be configured to fully qualified path. Contributed by Xuan Gong 

(cherry picked from commit d81109e588)
This commit is contained in:
Junping Du 2015-03-20 13:41:22 -07:00
parent 308460654a
commit f40f17489c
5 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -35,6 +35,9 @@ Release 2.8.0 - UNRELEASED
YARN-3351. AppMaster tracking URL is broken in HA. (Anubhav Dhoot via kasha) YARN-3351. AppMaster tracking URL is broken in HA. (Anubhav Dhoot via kasha)
YARN-3269. Yarn.nodemanager.remote-app-log-dir could not be configured to
fully qualified path. (Xuan Gong via junping_du)
Release 2.7.0 - UNRELEASED Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -379,7 +379,7 @@ public class AggregatedLogFormat {
userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() { userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() {
@Override @Override
public FSDataOutputStream run() throws Exception { public FSDataOutputStream run() throws Exception {
fc = FileContext.getFileContext(conf); fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
fc.setUMask(APP_LOG_FILE_UMASK); fc.setUMask(APP_LOG_FILE_UMASK);
return fc.create( return fc.create(
remoteAppLogFile, remoteAppLogFile,
@ -471,7 +471,8 @@ public class AggregatedLogFormat {
public LogReader(Configuration conf, Path remoteAppLogFile) public LogReader(Configuration conf, Path remoteAppLogFile)
throws IOException { throws IOException {
FileContext fileContext = FileContext.getFileContext(conf); FileContext fileContext =
FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
this.fsDataIStream = fileContext.open(remoteAppLogFile); this.fsDataIStream = fileContext.open(remoteAppLogFile);
reader = reader =
new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus(

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
View File

@ -303,7 +303,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
userUgi.doAs(new PrivilegedExceptionAction<Object>() { userUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override @Override
public Object run() throws Exception { public Object run() throws Exception {
FileSystem remoteFS = FileSystem.get(conf); FileSystem remoteFS = remoteNodeLogFileForApp.getFileSystem(conf);
if (remoteFS.exists(remoteNodeTmpLogFileForApp)) { if (remoteFS.exists(remoteNodeTmpLogFileForApp)) {
if (rename) { if (rename) {
remoteFS.rename(remoteNodeTmpLogFileForApp, renamedPath); remoteFS.rename(remoteNodeTmpLogFileForApp, renamedPath);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
View File

@ -176,7 +176,7 @@ public class LogAggregationService extends AbstractService implements
} }
protected FileSystem getFileSystem(Configuration conf) throws IOException { protected FileSystem getFileSystem(Configuration conf) throws IOException {
return FileSystem.get(conf); return this.remoteRootLogDir.getFileSystem(conf);
} }
void verifyAndCreateRemoteLogDir(Configuration conf) { void verifyAndCreateRemoteLogDir(Configuration conf) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
View File

@ -1299,8 +1299,10 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
logAggregationContextWithInterval.setRolledLogsIncludePattern(".*"); logAggregationContextWithInterval.setRolledLogsIncludePattern(".*");
logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final"); logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final");
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
//configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
//have fully qualified path
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
this.remoteRootLogDir.getAbsolutePath()); "file://" + this.remoteRootLogDir.getAbsolutePath());
this.conf.setLong( this.conf.setLong(
YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
3600); 3600);