YARN-7148. TestLogsCLI fails in trunk and branch-2 and javadoc error. Contributed by Xuan Gong.

This commit is contained in:
Junping Du 2017-09-06 13:19:34 -07:00
parent 1f3bc63e67
commit 63231a2a30
6 changed files with 103 additions and 76 deletions

View File

@ -285,7 +285,7 @@ public class TestLogsCLI {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception { public void testFetchFinishedApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -646,7 +646,7 @@ public class TestLogsCLI {
any(ContainerId.class)); any(ContainerId.class));
// create local logs // create local logs
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(configuration);
String rootLogDir = "target/LocalLogs"; String rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir); Path rootLogDirPath = new Path(rootLogDir);
@ -790,7 +790,7 @@ public class TestLogsCLI {
UserGroupInformation testUgi = UserGroupInformation UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser); .createRemoteUser(testUser);
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -921,7 +921,7 @@ public class TestLogsCLI {
public void testLogsCLIWithInvalidArgs() throws Exception { public void testLogsCLIWithInvalidArgs() throws Exception {
String localDir = "target/SaveLogs"; String localDir = "target/SaveLogs";
Path localPath = new Path(localDir); Path localPath = new Path(localDir);
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration); FileSystem fs = FileSystem.get(configuration);
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
YarnClient mockYarnClient = YarnClient mockYarnClient =
@ -992,7 +992,7 @@ public class TestLogsCLI {
String localDir = "target/SaveLogs"; String localDir = "target/SaveLogs";
Path localPath = new Path(localDir); Path localPath = new Path(localDir);
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -1083,7 +1083,7 @@ public class TestLogsCLI {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception { public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -1188,7 +1188,7 @@ public class TestLogsCLI {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testListNodeInfo() throws Exception { public void testListNodeInfo() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@ -1239,7 +1239,7 @@ public class TestLogsCLI {
@Test (timeout = 15000) @Test (timeout = 15000)
public void testFetchApplictionLogsHar() throws Exception { public void testFetchApplictionLogsHar() throws Exception {
String remoteLogRootDir = "target/logs/"; String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration(); Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);

View File

@ -583,9 +583,9 @@ public class AggregatedLogFormat {
/** /**
* Returns the owner of the application. * Returns the owner of the application.
* *
* @return the application owner. * @return the application owner.
* @throws IOException * @throws IOException if we can not get the application owner.
*/ */
public String getApplicationOwner() throws IOException { public String getApplicationOwner() throws IOException {
TFile.Reader.Scanner ownerScanner = null; TFile.Reader.Scanner ownerScanner = null;
@ -610,9 +610,9 @@ public class AggregatedLogFormat {
/** /**
* Returns ACLs for the application. An empty map is returned if no ACLs are * Returns ACLs for the application. An empty map is returned if no ACLs are
* found. * found.
* *
* @return a map of the Application ACLs. * @return a map of the Application ACLs.
* @throws IOException * @throws IOException if we can not get the application acls.
*/ */
public Map<ApplicationAccessType, String> getApplicationAcls() public Map<ApplicationAccessType, String> getApplicationAcls()
throws IOException { throws IOException {
@ -656,9 +656,10 @@ public class AggregatedLogFormat {
/** /**
* Read the next key and return the value-stream. * Read the next key and return the value-stream.
* *
* @param key * @param key the log key
* @return the valueStream if there are more keys or null otherwise. * @return the valueStream if there are more keys or null otherwise
* @throws IOException * @throws IOException if we can not get the dataInputStream
* for the next key
*/ */
public DataInputStream next(LogKey key) throws IOException { public DataInputStream next(LogKey key) throws IOException {
if (!this.atBeginning) { if (!this.atBeginning) {
@ -683,10 +684,10 @@ public class AggregatedLogFormat {
* Get a ContainerLogsReader to read the logs for * Get a ContainerLogsReader to read the logs for
* the specified container. * the specified container.
* *
* @param containerId * @param containerId the containerId
* @return object to read the container's logs or null if the * @return object to read the container's logs or null if the
* logs could not be found * logs could not be found
* @throws IOException * @throws IOException if we can not get the container log reader.
*/ */
@Private @Private
public ContainerLogsReader getContainerLogsReader( public ContainerLogsReader getContainerLogsReader(
@ -752,10 +753,10 @@ public class AggregatedLogFormat {
/** /**
* Writes all logs for a single container to the provided writer. * Writes all logs for a single container to the provided writer.
* @param valueStream * @param valueStream the valueStream
* @param writer * @param writer the log writer
* @param logUploadedTime * @param logUploadedTime the time stamp
* @throws IOException * @throws IOException if we can not read the container logs.
*/ */
public static void readAcontainerLogs(DataInputStream valueStream, public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer, long logUploadedTime) throws IOException { Writer writer, long logUploadedTime) throws IOException {
@ -780,9 +781,9 @@ public class AggregatedLogFormat {
/** /**
* Writes all logs for a single container to the provided writer. * Writes all logs for a single container to the provided writer.
* @param valueStream * @param valueStream the value stream
* @param writer * @param writer the log writer
* @throws IOException * @throws IOException if we can not read the container logs.
*/ */
public static void readAcontainerLogs(DataInputStream valueStream, public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer) throws IOException { Writer writer) throws IOException {
@ -849,10 +850,11 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of * Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container. * all types for a single container.
* *
* @param valueStream * @param valueStream the value stream
* @param out * @param out the print stream
* @param logUploadedTime * @param logUploadedTime the time stamp
* @throws IOException * @throws IOException if we can not read the container log by specifying
* the container log type.
*/ */
public static void readAContainerLogsForALogType( public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime) DataInputStream valueStream, PrintStream out, long logUploadedTime)
@ -864,11 +866,11 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of * Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container for the specific bytes. * all types for a single container for the specific bytes.
* *
* @param valueStream * @param valueStream the value stream
* @param out * @param out the output print stream
* @param logUploadedTime * @param logUploadedTime the log upload time stamp
* @param bytes * @param bytes the output size of the log
* @throws IOException * @throws IOException if we can not read the container log
*/ */
public static void readAContainerLogsForALogType( public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime, DataInputStream valueStream, PrintStream out, long logUploadedTime,
@ -880,9 +882,9 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of * Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container. * all types for a single container.
* *
* @param valueStream * @param valueStream the value stream
* @param out * @param out the output print stream
* @throws IOException * @throws IOException if we can not read the container log
*/ */
public static void readAContainerLogsForALogType( public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out) DataInputStream valueStream, PrintStream out)
@ -893,11 +895,11 @@ public class AggregatedLogFormat {
/** /**
* Keep calling this till you get a {@link EOFException} for getting logs of * Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container. * the specific types for a single container.
* @param valueStream * @param valueStream the value stream
* @param out * @param out the output print stream
* @param logUploadedTime * @param logUploadedTime the log uploaded time stamp
* @param logType * @param logType the given log type
* @throws IOException * @throws IOException if we can not read the container logs
*/ */
public static int readContainerLogsForALogType( public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime, DataInputStream valueStream, PrintStream out, long logUploadedTime,
@ -909,11 +911,11 @@ public class AggregatedLogFormat {
/** /**
* Keep calling this till you get a {@link EOFException} for getting logs of * Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container. * the specific types for a single container.
* @param valueStream * @param valueStream the value stream
* @param out * @param out the output print stream
* @param logUploadedTime * @param logUploadedTime the log uploaded time stamp
* @param logType * @param logType the given log type
* @throws IOException * @throws IOException if we can not read the container logs
*/ */
public static int readContainerLogsForALogType( public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime, DataInputStream valueStream, PrintStream out, long logUploadedTime,

View File

@ -38,11 +38,11 @@ public class LogAggregationUtils {
/** /**
* Constructs the full filename for an application's log file per node. * Constructs the full filename for an application's log file per node.
* @param remoteRootLogDir * @param remoteRootLogDir the aggregated remote root log dir
* @param appId * @param appId the application Id
* @param user * @param user the application owner
* @param nodeId * @param nodeId the node id
* @param suffix * @param suffix the log dir suffix
* @return the remote log file. * @return the remote log file.
*/ */
public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir,
@ -53,10 +53,10 @@ public class LogAggregationUtils {
/** /**
* Gets the remote app log dir. * Gets the remote app log dir.
* @param remoteRootLogDir * @param remoteRootLogDir the aggregated log remote root log dir
* @param appId * @param appId the application id
* @param user * @param user the application owner
* @param suffix * @param suffix the log directory suffix
* @return the remote application specific log dir. * @return the remote application specific log dir.
*/ */
public static Path getRemoteAppLogDir(Path remoteRootLogDir, public static Path getRemoteAppLogDir(Path remoteRootLogDir,
@ -67,9 +67,9 @@ public class LogAggregationUtils {
/** /**
* Gets the remote suffixed log dir for the user. * Gets the remote suffixed log dir for the user.
* @param remoteRootLogDir * @param remoteRootLogDir the aggregated log remote root log dir
* @param user * @param user the application owner
* @param suffix * @param suffix the log dir suffix
* @return the remote suffixed log dir. * @return the remote suffixed log dir.
*/ */
public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir,
@ -83,8 +83,8 @@ public class LogAggregationUtils {
/** /**
* Gets the remote log user dir. * Gets the remote log user dir.
* @param remoteRootLogDir * @param remoteRootLogDir the aggregated log remote root log dir
* @param user * @param user the application owner
* @return the remote per user log dir. * @return the remote per user log dir.
*/ */
public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) { public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) {
@ -93,7 +93,7 @@ public class LogAggregationUtils {
/** /**
* Returns the suffix component of the log dir. * Returns the suffix component of the log dir.
* @param conf * @param conf the configuration
* @return the suffix which will be appended to the user log dir. * @return the suffix which will be appended to the user log dir.
*/ */
public static String getRemoteNodeLogDirSuffix(Configuration conf) { public static String getRemoteNodeLogDirSuffix(Configuration conf) {
@ -104,7 +104,7 @@ public class LogAggregationUtils {
/** /**
* Converts a nodeId to a form used in the app log file name. * Converts a nodeId to a form used in the app log file name.
* @param nodeId * @param nodeId the nodeId
* @return the node string to be used to construct the file name. * @return the node string to be used to construct the file name.
*/ */
@VisibleForTesting @VisibleForTesting

View File

@ -128,8 +128,17 @@ public class LogCLIHelpers implements Configurable {
@VisibleForTesting @VisibleForTesting
public int dumpAContainerLogsForLogType(ContainerLogsRequest options, public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
boolean outputFailure) throws IOException { boolean outputFailure) throws IOException {
boolean foundAnyLogs = this.getFileController(options.getAppId(), LogAggregationFileController fc = null;
options.getAppOwner()).readAggregatedLogs(options, null); try {
fc = this.getFileController(
options.getAppId(), options.getAppOwner());
} catch (IOException ex) {
System.err.println(ex);
}
boolean foundAnyLogs = false;
if (fc != null) {
foundAnyLogs = fc.readAggregatedLogs(options, null);
}
if (!foundAnyLogs) { if (!foundAnyLogs) {
if (outputFailure) { if (outputFailure) {
containerLogNotFound(options.getContainerId()); containerLogNotFound(options.getContainerId());
@ -142,9 +151,17 @@ public class LogCLIHelpers implements Configurable {
@Private @Private
public int dumpAContainerLogsForLogTypeWithoutNodeId( public int dumpAContainerLogsForLogTypeWithoutNodeId(
ContainerLogsRequest options) throws IOException { ContainerLogsRequest options) throws IOException {
boolean foundAnyLogs = getFileController(options.getAppId(), LogAggregationFileController fc = null;
options.getAppOwner()).readAggregatedLogs( try {
options, null); fc = this.getFileController(
options.getAppId(), options.getAppOwner());
} catch (IOException ex) {
System.err.println(ex);
}
boolean foundAnyLogs = false;
if (fc != null) {
foundAnyLogs = fc.readAggregatedLogs(options, null);
}
if (!foundAnyLogs) { if (!foundAnyLogs) {
containerLogNotFound(options.getContainerId()); containerLogNotFound(options.getContainerId());
return -1; return -1;
@ -155,9 +172,17 @@ public class LogCLIHelpers implements Configurable {
@Private @Private
public int dumpAllContainersLogs(ContainerLogsRequest options) public int dumpAllContainersLogs(ContainerLogsRequest options)
throws IOException { throws IOException {
boolean foundAnyLogs = getFileController(options.getAppId(), LogAggregationFileController fc = null;
options.getAppOwner()).readAggregatedLogs( try {
options, null); fc = this.getFileController(
options.getAppId(), options.getAppOwner());
} catch (IOException ex) {
System.err.println(ex);
}
boolean foundAnyLogs = false;
if (fc != null) {
foundAnyLogs = fc.readAggregatedLogs(options, null);
}
if (!foundAnyLogs) { if (!foundAnyLogs) {
emptyLogDir(LogAggregationUtils.getRemoteAppLogDir( emptyLogDir(LogAggregationUtils.getRemoteAppLogDir(
conf, options.getAppId(), options.getAppOwner()) conf, options.getAppId(), options.getAppOwner())

View File

@ -201,6 +201,7 @@ public abstract class LogAggregationFileController {
* Output container log. * Output container log.
* @param logRequest {@link ContainerLogsRequest} * @param logRequest {@link ContainerLogsRequest}
* @param os the output stream * @param os the output stream
* @return true if we can read the aggregated logs successfully
* @throws IOException if we can not access the log file. * @throws IOException if we can not access the log file.
*/ */
public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest, public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest,
@ -228,9 +229,9 @@ public abstract class LogAggregationFileController {
/** /**
* Returns the owner of the application. * Returns the owner of the application.
* *
* @param the aggregatedLog path. * @param aggregatedLogPath the aggregatedLog path
* @return the application owner. * @return the application owner
* @throws IOException * @throws IOException if we can not get the application owner
*/ */
public abstract String getApplicationOwner(Path aggregatedLogPath) public abstract String getApplicationOwner(Path aggregatedLogPath)
throws IOException; throws IOException;
@ -239,9 +240,9 @@ public abstract class LogAggregationFileController {
* Returns ACLs for the application. An empty map is returned if no ACLs are * Returns ACLs for the application. An empty map is returned if no ACLs are
* found. * found.
* *
* @param the aggregatedLog path. * @param aggregatedLogPath the aggregatedLog path.
* @return a map of the Application ACLs. * @return a map of the Application ACLs.
* @throws IOException * @throws IOException if we can not get the application acls
*/ */
public abstract Map<ApplicationAccessType, String> getApplicationAcls( public abstract Map<ApplicationAccessType, String> getApplicationAcls(
Path aggregatedLogPath) throws IOException; Path aggregatedLogPath) throws IOException;

View File

@ -172,7 +172,6 @@ public class LogAggregationTFileController
byte[] buf = new byte[65535]; byte[] buf = new byte[65535];
while (nodeFiles != null && nodeFiles.hasNext()) { while (nodeFiles != null && nodeFiles.hasNext()) {
final FileStatus thisNodeFile = nodeFiles.next(); final FileStatus thisNodeFile = nodeFiles.next();
LOG.error(thisNodeFile.getPath().toString());
String nodeName = thisNodeFile.getPath().getName(); String nodeName = thisNodeFile.getPath().getName();
if (nodeName.equals(appId + ".har")) { if (nodeName.equals(appId + ".har")) {
Path p = new Path("har:///" Path p = new Path("har:///"