merge YARN-22 from trunk. Fix ContainerLogs to work if the log-dir is specified as a URI. (Contributed by Mayank Bansal)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1375831 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
63810d26ae
commit
d5190bbe00
|
@ -36,6 +36,9 @@ Release 2.1.0-alpha - Unreleased
|
||||||
|
|
||||||
YARN-12. Fix findbugs warnings in FairScheduler. (Junping Du via acmurthy)
|
YARN-12. Fix findbugs warnings in FairScheduler. (Junping Du via acmurthy)
|
||||||
|
|
||||||
|
YARN-22. Fix ContainerLogs to work if the log-dir is specified as a URI.
|
||||||
|
(Mayank Bansal via sseth)
|
||||||
|
|
||||||
Release 0.23.3 - Unreleased
|
Release 0.23.3 - Unreleased
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -29,6 +29,8 @@ import java.io.File;
|
||||||
import java.io.FileReader;
|
import java.io.FileReader;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
||||||
import org.apache.hadoop.yarn.webapp.SubView;
|
import org.apache.hadoop.yarn.webapp.SubView;
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||||
|
import org.mortbay.log.Log;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import com.google.inject.Inject;
|
||||||
|
|
||||||
|
@ -198,12 +201,14 @@ public class ContainerLogsPage extends NMView {
|
||||||
if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
|
if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
|
||||||
File logFile = null;
|
File logFile = null;
|
||||||
try {
|
try {
|
||||||
logFile =
|
URI logPathURI = new URI(this.dirsHandler.getLogPathToRead(
|
||||||
new File(this.dirsHandler.getLogPathToRead(
|
ContainerLaunch.getRelativeContainerLogDir(
|
||||||
ContainerLaunch.getRelativeContainerLogDir(
|
|
||||||
applicationId.toString(), containerId.toString())
|
applicationId.toString(), containerId.toString())
|
||||||
+ Path.SEPARATOR + $(CONTAINER_LOG_TYPE))
|
+ Path.SEPARATOR + $(CONTAINER_LOG_TYPE)).toString());
|
||||||
.toUri().getPath());
|
logFile = new File(logPathURI.getPath());
|
||||||
|
} catch (URISyntaxException e) {
|
||||||
|
html.h1("Cannot find this log on the local disk.");
|
||||||
|
return;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
html.h1("Cannot find this log on the local disk.");
|
html.h1("Cannot find this log on the local disk.");
|
||||||
return;
|
return;
|
||||||
|
@ -278,14 +283,16 @@ public class ContainerLogsPage extends NMView {
|
||||||
boolean foundLogFile = false;
|
boolean foundLogFile = false;
|
||||||
for (File containerLogsDir : containerLogsDirs) {
|
for (File containerLogsDir : containerLogsDirs) {
|
||||||
File[] logFiles = containerLogsDir.listFiles();
|
File[] logFiles = containerLogsDir.listFiles();
|
||||||
Arrays.sort(logFiles);
|
if (logFiles != null) {
|
||||||
for (File logFile : logFiles) {
|
Arrays.sort(logFiles);
|
||||||
foundLogFile = true;
|
for (File logFile : logFiles) {
|
||||||
html.p()
|
foundLogFile = true;
|
||||||
.a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
|
html.p()
|
||||||
logFile.getName(), "?start=-4096"),
|
.a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
|
||||||
logFile.getName() + " : Total file length is "
|
logFile.getName(), "?start=-4096"),
|
||||||
+ logFile.length() + " bytes.")._();
|
logFile.getName() + " : Total file length is "
|
||||||
|
+ logFile.length() + " bytes.")._();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!foundLogFile) {
|
if (!foundLogFile) {
|
||||||
|
@ -297,13 +304,17 @@ public class ContainerLogsPage extends NMView {
|
||||||
}
|
}
|
||||||
|
|
||||||
static List<File> getContainerLogDirs(ContainerId containerId,
|
static List<File> getContainerLogDirs(ContainerId containerId,
|
||||||
LocalDirsHandlerService dirsHandler) {
|
LocalDirsHandlerService dirsHandler) {
|
||||||
List<String> logDirs = dirsHandler.getLogDirs();
|
List<String> logDirs = dirsHandler.getLogDirs();
|
||||||
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
|
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
|
||||||
for (String logDir : logDirs) {
|
for (String logDir : logDirs) {
|
||||||
String appIdStr =
|
try {
|
||||||
ConverterUtils.toString(
|
logDir = new URI(logDir).getPath();
|
||||||
containerId.getApplicationAttemptId().getApplicationId());
|
} catch (URISyntaxException e) {
|
||||||
|
Log.warn(e.getMessage());
|
||||||
|
}
|
||||||
|
String appIdStr = ConverterUtils.toString(containerId
|
||||||
|
.getApplicationAttemptId().getApplicationId());
|
||||||
File appLogDir = new File(logDir, appIdStr);
|
File appLogDir = new File(logDir, appIdStr);
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = ConverterUtils.toString(containerId);
|
||||||
containerLogDirs.add(new File(appLogDir, containerIdStr));
|
containerLogDirs.add(new File(appLogDir, containerIdStr));
|
||||||
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.server.nodemanager.webapp;
|
||||||
|
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
|
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
|
||||||
|
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
|
||||||
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
|
||||||
|
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestContainerLogsPage {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testContainerLogDirs() throws IOException {
|
||||||
|
String logdirwithFile = "file:///target/"
|
||||||
|
+ TestNMWebServer.class.getSimpleName() + "LogDir";
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
|
||||||
|
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
|
||||||
|
healthChecker.init(conf);
|
||||||
|
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
|
||||||
|
// Add an application and the corresponding containers
|
||||||
|
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
|
||||||
|
String user = "nobody";
|
||||||
|
long clusterTimeStamp = 1234;
|
||||||
|
ApplicationId appId = BuilderUtils.newApplicationId(recordFactory,
|
||||||
|
clusterTimeStamp, 1);
|
||||||
|
Application app = mock(Application.class);
|
||||||
|
when(app.getUser()).thenReturn(user);
|
||||||
|
when(app.getAppId()).thenReturn(appId);
|
||||||
|
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
||||||
|
appId, 1);
|
||||||
|
ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
|
||||||
|
appAttemptId, 0);
|
||||||
|
List<File> files = null;
|
||||||
|
files = ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(
|
||||||
|
container1, dirsHandler);
|
||||||
|
Assert.assertTrue(!(files.get(0).toString().contains("file:")));
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue