Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7148. TestLogsCLI fails in trunk and branch-2 and javadoc error. Contributed by Xuan Gong.

This commit is contained in:
Junping Du 2017-09-06 13:19:34 -07:00
parent 1f3bc63e67
commit 63231a2a30
6 changed files with 103 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
View File

@ -285,7 +285,7 @@ public void testHelpMessage() throws Exception {
@Test (timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -646,7 +646,7 @@ public void testGetRunningContainerLogs() throws Exception {
any(ContainerId.class));
// create local logs
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration);
String rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
@ -790,7 +790,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser);
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -921,7 +921,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
public void testLogsCLIWithInvalidArgs() throws Exception {
String localDir = "target/SaveLogs";
Path localPath = new Path(localDir);
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration);
ApplicationId appId = ApplicationId.newInstance(0, 1);
YarnClient mockYarnClient =
@ -992,7 +992,7 @@ public void testSaveContainerLogsLocally() throws Exception {
String localDir = "target/SaveLogs";
Path localPath = new Path(localDir);
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -1083,7 +1083,7 @@ private String readContainerContent(Path containerPath,
@Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -1188,7 +1188,7 @@ public void testPrintContainerLogMetadata() throws Exception {
@Test (timeout = 15000)
public void testListNodeInfo() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -1239,7 +1239,7 @@ public void testListNodeInfo() throws Exception {
@Test (timeout = 15000)
public void testFetchApplictionLogsHar() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);

78
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -583,9 +583,9 @@ public LogReader(Configuration conf, Path remoteAppLogFile)
/**
* Returns the owner of the application.
*
*
* @return the application owner.
* @throws IOException
* @throws IOException if we can not get the application owner.
*/
public String getApplicationOwner() throws IOException {
TFile.Reader.Scanner ownerScanner = null;
@ -610,9 +610,9 @@ public String getApplicationOwner() throws IOException {
/**
* Returns ACLs for the application. An empty map is returned if no ACLs are
* found.
*
*
* @return a map of the Application ACLs.
* @throws IOException
* @throws IOException if we can not get the application acls.
*/
public Map<ApplicationAccessType, String> getApplicationAcls()
throws IOException {
@ -656,9 +656,10 @@ public Map<ApplicationAccessType, String> getApplicationAcls()
/**
* Read the next key and return the value-stream.
*
* @param key
* @return the valueStream if there are more keys or null otherwise.
* @throws IOException
* @param key the log key
* @return the valueStream if there are more keys or null otherwise
* @throws IOException if we can not get the dataInputStream
* for the next key
*/
public DataInputStream next(LogKey key) throws IOException {
if (!this.atBeginning) {
@ -683,10 +684,10 @@ public DataInputStream next(LogKey key) throws IOException {
* Get a ContainerLogsReader to read the logs for
* the specified container.
*
* @param containerId
* @param containerId the containerId
* @return object to read the container's logs or null if the
* logs could not be found
* @throws IOException
* @throws IOException if we can not get the container log reader.
*/
@Private
public ContainerLogsReader getContainerLogsReader(
@ -752,10 +753,10 @@ public ContainerLogsReader getContainerLogsReader(
/**
* Writes all logs for a single container to the provided writer.
* @param valueStream
* @param writer
* @param logUploadedTime
* @throws IOException
* @param valueStream the valueStream
* @param writer the log writer
* @param logUploadedTime the time stamp
* @throws IOException if we can not read the container logs.
*/
public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer, long logUploadedTime) throws IOException {
@ -780,9 +781,9 @@ public static void readAcontainerLogs(DataInputStream valueStream,
/**
* Writes all logs for a single container to the provided writer.
* @param valueStream
* @param writer
* @throws IOException
* @param valueStream the value stream
* @param writer the log writer
* @throws IOException if we can not read the container logs.
*/
public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer) throws IOException {
@ -849,10 +850,11 @@ private static void readContainerLogs(DataInputStream valueStream,
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container.
*
* @param valueStream
* @param out
* @param logUploadedTime
* @throws IOException
* @param valueStream the value stream
* @param out the print stream
* @param logUploadedTime the time stamp
* @throws IOException if we can not read the container log by specifying
* the container log type.
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime)
@ -864,11 +866,11 @@ public static void readAContainerLogsForALogType(
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container for the specific bytes.
*
* @param valueStream
* @param out
* @param logUploadedTime
* @param bytes
* @throws IOException
* @param valueStream the value stream
* @param out the output print stream
* @param logUploadedTime the log upload time stamp
* @param bytes the output size of the log
* @throws IOException if we can not read the container log
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
@ -880,9 +882,9 @@ public static void readAContainerLogsForALogType(
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container.
*
* @param valueStream
* @param out
* @throws IOException
* @param valueStream the value stream
* @param out the output print stream
* @throws IOException if we can not read the container log
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out)
@ -893,11 +895,11 @@ public static void readAContainerLogsForALogType(
/**
* Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container.
* @param valueStream
* @param out
* @param logUploadedTime
* @param logType
* @throws IOException
* @param valueStream the value stream
* @param out the output print stream
* @param logUploadedTime the log uploaded time stamp
* @param logType the given log type
* @throws IOException if we can not read the container logs
*/
public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
@ -909,11 +911,11 @@ public static int readContainerLogsForALogType(
/**
* Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container.
* @param valueStream
* @param out
* @param logUploadedTime
* @param logType
* @throws IOException
* @param valueStream the value stream
* @param out the output print stream
* @param logUploadedTime the log uploaded time stamp
* @param logType the given log type
* @throws IOException if we can not read the container logs
*/
public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
View File

@ -38,11 +38,11 @@ public class LogAggregationUtils {
/**
* Constructs the full filename for an application's log file per node.
* @param remoteRootLogDir
* @param appId
* @param user
* @param nodeId
* @param suffix
* @param remoteRootLogDir the aggregated remote root log dir
* @param appId the application Id
* @param user the application owner
* @param nodeId the node id
* @param suffix the log dir suffix
* @return the remote log file.
*/
public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir,
@ -53,10 +53,10 @@ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir,
/**
* Gets the remote app log dir.
* @param remoteRootLogDir
* @param appId
* @param user
* @param suffix
* @param remoteRootLogDir the aggregated log remote root log dir
* @param appId the application id
* @param user the application owner
* @param suffix the log directory suffix
* @return the remote application specific log dir.
*/
public static Path getRemoteAppLogDir(Path remoteRootLogDir,
@ -67,9 +67,9 @@ public static Path getRemoteAppLogDir(Path remoteRootLogDir,
/**
* Gets the remote suffixed log dir for the user.
* @param remoteRootLogDir
* @param user
* @param suffix
* @param remoteRootLogDir the aggregated log remote root log dir
* @param user the application owner
* @param suffix the log dir suffix
* @return the remote suffixed log dir.
*/
public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir,
@ -83,8 +83,8 @@ public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir,
/**
* Gets the remote log user dir.
* @param remoteRootLogDir
* @param user
* @param remoteRootLogDir the aggregated log remote root log dir
* @param user the application owner
* @return the remote per user log dir.
*/
public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) {
@ -93,7 +93,7 @@ public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) {
/**
* Returns the suffix component of the log dir.
* @param conf
* @param conf the configuration
* @return the suffix which will be appended to the user log dir.
*/
public static String getRemoteNodeLogDirSuffix(Configuration conf) {
@ -104,7 +104,7 @@ public static String getRemoteNodeLogDirSuffix(Configuration conf) {
/**
* Converts a nodeId to a form used in the app log file name.
* @param nodeId
* @param nodeId the nodeId
* @return the node string to be used to construct the file name.
*/
@VisibleForTesting

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
View File

@ -128,8 +128,17 @@ public int dumpAContainerLogsForLogType(ContainerLogsRequest options)
@VisibleForTesting
public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
boolean outputFailure) throws IOException {
boolean foundAnyLogs = this.getFileController(options.getAppId(),
options.getAppOwner()).readAggregatedLogs(options, null);
LogAggregationFileController fc = null;
try {
fc = this.getFileController(
options.getAppId(), options.getAppOwner());
} catch (IOException ex) {
System.err.println(ex);
}
boolean foundAnyLogs = false;
if (fc != null) {
foundAnyLogs = fc.readAggregatedLogs(options, null);
}
if (!foundAnyLogs) {
if (outputFailure) {
containerLogNotFound(options.getContainerId());
@ -142,9 +151,17 @@ public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
@Private
public int dumpAContainerLogsForLogTypeWithoutNodeId(
ContainerLogsRequest options) throws IOException {
boolean foundAnyLogs = getFileController(options.getAppId(),
options.getAppOwner()).readAggregatedLogs(
options, null);
LogAggregationFileController fc = null;
try {
fc = this.getFileController(
options.getAppId(), options.getAppOwner());
} catch (IOException ex) {
System.err.println(ex);
}
boolean foundAnyLogs = false;
if (fc != null) {
foundAnyLogs = fc.readAggregatedLogs(options, null);
}
if (!foundAnyLogs) {
containerLogNotFound(options.getContainerId());
return -1;
@ -155,9 +172,17 @@ public int dumpAContainerLogsForLogTypeWithoutNodeId(
@Private
public int dumpAllContainersLogs(ContainerLogsRequest options)
throws IOException {
boolean foundAnyLogs = getFileController(options.getAppId(),
options.getAppOwner()).readAggregatedLogs(
options, null);
LogAggregationFileController fc = null;
try {
fc = this.getFileController(
options.getAppId(), options.getAppOwner());
} catch (IOException ex) {
System.err.println(ex);
}
boolean foundAnyLogs = false;
if (fc != null) {
foundAnyLogs = fc.readAggregatedLogs(options, null);
}
if (!foundAnyLogs) {
emptyLogDir(LogAggregationUtils.getRemoteAppLogDir(
conf, options.getAppId(), options.getAppOwner())

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
View File

@ -201,6 +201,7 @@ protected void closePrintStream(OutputStream out) {
* Output container log.
* @param logRequest {@link ContainerLogsRequest}
* @param os the output stream
* @return true if we can read the aggregated logs successfully
* @throws IOException if we can not access the log file.
*/
public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest,
@ -228,9 +229,9 @@ public abstract void renderAggregatedLogsBlock(Block html,
/**
* Returns the owner of the application.
*
* @param the aggregatedLog path.
* @return the application owner.
* @throws IOException
* @param aggregatedLogPath the aggregatedLog path
* @return the application owner
* @throws IOException if we can not get the application owner
*/
public abstract String getApplicationOwner(Path aggregatedLogPath)
throws IOException;
@ -239,9 +240,9 @@ public abstract String getApplicationOwner(Path aggregatedLogPath)
* Returns ACLs for the application. An empty map is returned if no ACLs are
* found.
*
* @param the aggregatedLog path.
* @param aggregatedLogPath the aggregatedLog path.
* @return a map of the Application ACLs.
* @throws IOException
* @throws IOException if we can not get the application acls
*/
public abstract Map<ApplicationAccessType, String> getApplicationAcls(
Path aggregatedLogPath) throws IOException;

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
View File

@ -172,7 +172,6 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest,
byte[] buf = new byte[65535];
while (nodeFiles != null && nodeFiles.hasNext()) {
final FileStatus thisNodeFile = nodeFiles.next();
LOG.error(thisNodeFile.getPath().toString());
String nodeName = thisNodeFile.getPath().getName();
if (nodeName.equals(appId + ".har")) {
Path p = new Path("har:///"