Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-22. Fix ContainerLogs to work if the log-dir is specified as a URI. (Contributed by Mayank Bansal) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375829 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Siddharth Seth 2012-08-21 22:27:11 +00:00
parent 557b937f47
commit 67673227f2
3 changed files with 102 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -34,6 +34,9 @@ Release 2.1.0-alpha - Unreleased
YARN-12. Fix findbugs warnings in FairScheduler. (Junping Du via acmurthy)
YARN-22. Fix ContainerLogs to work if the log-dir is specified as a URI.
(Mayank Bansal via sseth)
Release 0.23.3 - Unreleased
INCOMPATIBLE CHANGES

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
View File

@ -29,6 +29,8 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -53,6 +55,7 @@
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.mortbay.log.Log;
import com.google.inject.Inject;
@ -198,12 +201,14 @@ private void printLogs(Block html, ContainerId containerId,
if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
File logFile = null;
try {
logFile =
new File(this.dirsHandler.getLogPathToRead(
ContainerLaunch.getRelativeContainerLogDir(
URI logPathURI = new URI(this.dirsHandler.getLogPathToRead(
ContainerLaunch.getRelativeContainerLogDir(
applicationId.toString(), containerId.toString())
+ Path.SEPARATOR + $(CONTAINER_LOG_TYPE))
.toUri().getPath());
+ Path.SEPARATOR + $(CONTAINER_LOG_TYPE)).toString());
logFile = new File(logPathURI.getPath());
} catch (URISyntaxException e) {
html.h1("Cannot find this log on the local disk.");
return;
} catch (Exception e) {
html.h1("Cannot find this log on the local disk.");
return;
@ -278,14 +283,16 @@ private void printLogs(Block html, ContainerId containerId,
boolean foundLogFile = false;
for (File containerLogsDir : containerLogsDirs) {
File[] logFiles = containerLogsDir.listFiles();
Arrays.sort(logFiles);
for (File logFile : logFiles) {
foundLogFile = true;
html.p()
.a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
logFile.getName(), "?start=-4096"),
logFile.getName() + " : Total file length is "
+ logFile.length() + " bytes.")._();
if (logFiles != null) {
Arrays.sort(logFiles);
for (File logFile : logFiles) {
foundLogFile = true;
html.p()
.a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
logFile.getName(), "?start=-4096"),
logFile.getName() + " : Total file length is "
+ logFile.length() + " bytes.")._();
}
}
}
if (!foundLogFile) {
@ -297,13 +304,17 @@ private void printLogs(Block html, ContainerId containerId,
}
static List<File> getContainerLogDirs(ContainerId containerId,
LocalDirsHandlerService dirsHandler) {
LocalDirsHandlerService dirsHandler) {
List<String> logDirs = dirsHandler.getLogDirs();
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
for (String logDir : logDirs) {
String appIdStr =
ConverterUtils.toString(
containerId.getApplicationAttemptId().getApplicationId());
try {
logDir = new URI(logDir).getPath();
} catch (URISyntaxException e) {
Log.warn(e.getMessage());
}
String appIdStr = ConverterUtils.toString(containerId
.getApplicationAttemptId().getApplicationId());
File appLogDir = new File(logDir, appIdStr);
String containerIdStr = ConverterUtils.toString(containerId);
containerLogDirs.add(new File(appLogDir, containerIdStr));

71
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java Normal file
View File

@ -0,0 +1,71 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
public class TestContainerLogsPage {
@Test
public void testContainerLogDirs() throws IOException {
String logdirwithFile = "file:///target/"
+ TestNMWebServer.class.getSimpleName() + "LogDir";
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
String user = "nobody";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.newApplicationId(recordFactory,
clusterTimeStamp, 1);
Application app = mock(Application.class);
when(app.getUser()).thenReturn(user);
when(app.getAppId()).thenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
appId, 1);
ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
appAttemptId, 0);
List<File> files = null;
files = ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(
container1, dirsHandler);
Assert.assertTrue(!(files.get(0).toString().contains("file:")));
}
}