YARN-7148. TestLogsCLI fails in trunk and branch-2 and javadoc error. Contributed by Xuan Gong.
This commit is contained in:
parent
1f3bc63e67
commit
63231a2a30
|
@ -285,7 +285,7 @@ public class TestLogsCLI {
|
|||
@Test (timeout = 15000)
|
||||
public void testFetchFinishedApplictionLogs() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
|
@ -646,7 +646,7 @@ public class TestLogsCLI {
|
|||
any(ContainerId.class));
|
||||
|
||||
// create local logs
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
String rootLogDir = "target/LocalLogs";
|
||||
Path rootLogDirPath = new Path(rootLogDir);
|
||||
|
@ -790,7 +790,7 @@ public class TestLogsCLI {
|
|||
UserGroupInformation testUgi = UserGroupInformation
|
||||
.createRemoteUser(testUser);
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
|
@ -921,7 +921,7 @@ public class TestLogsCLI {
|
|||
public void testLogsCLIWithInvalidArgs() throws Exception {
|
||||
String localDir = "target/SaveLogs";
|
||||
Path localPath = new Path(localDir);
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||
YarnClient mockYarnClient =
|
||||
|
@ -992,7 +992,7 @@ public class TestLogsCLI {
|
|||
String localDir = "target/SaveLogs";
|
||||
Path localPath = new Path(localDir);
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
|
@ -1083,7 +1083,7 @@ public class TestLogsCLI {
|
|||
@Test (timeout = 15000)
|
||||
public void testPrintContainerLogMetadata() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
|
@ -1188,7 +1188,7 @@ public class TestLogsCLI {
|
|||
@Test (timeout = 15000)
|
||||
public void testListNodeInfo() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
|
@ -1239,7 +1239,7 @@ public class TestLogsCLI {
|
|||
@Test (timeout = 15000)
|
||||
public void testFetchApplictionLogsHar() throws Exception {
|
||||
String remoteLogRootDir = "target/logs/";
|
||||
Configuration configuration = new Configuration();
|
||||
Configuration configuration = new YarnConfiguration();
|
||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||
configuration
|
||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||
|
|
|
@ -583,9 +583,9 @@ public class AggregatedLogFormat {
|
|||
|
||||
/**
|
||||
* Returns the owner of the application.
|
||||
*
|
||||
*
|
||||
* @return the application owner.
|
||||
* @throws IOException
|
||||
* @throws IOException if we can not get the application owner.
|
||||
*/
|
||||
public String getApplicationOwner() throws IOException {
|
||||
TFile.Reader.Scanner ownerScanner = null;
|
||||
|
@ -610,9 +610,9 @@ public class AggregatedLogFormat {
|
|||
/**
|
||||
* Returns ACLs for the application. An empty map is returned if no ACLs are
|
||||
* found.
|
||||
*
|
||||
*
|
||||
* @return a map of the Application ACLs.
|
||||
* @throws IOException
|
||||
* @throws IOException if we can not get the application acls.
|
||||
*/
|
||||
public Map<ApplicationAccessType, String> getApplicationAcls()
|
||||
throws IOException {
|
||||
|
@ -656,9 +656,10 @@ public class AggregatedLogFormat {
|
|||
/**
|
||||
* Read the next key and return the value-stream.
|
||||
*
|
||||
* @param key
|
||||
* @return the valueStream if there are more keys or null otherwise.
|
||||
* @throws IOException
|
||||
* @param key the log key
|
||||
* @return the valueStream if there are more keys or null otherwise
|
||||
* @throws IOException if we can not get the dataInputStream
|
||||
* for the next key
|
||||
*/
|
||||
public DataInputStream next(LogKey key) throws IOException {
|
||||
if (!this.atBeginning) {
|
||||
|
@ -683,10 +684,10 @@ public class AggregatedLogFormat {
|
|||
* Get a ContainerLogsReader to read the logs for
|
||||
* the specified container.
|
||||
*
|
||||
* @param containerId
|
||||
* @param containerId the containerId
|
||||
* @return object to read the container's logs or null if the
|
||||
* logs could not be found
|
||||
* @throws IOException
|
||||
* @throws IOException if we can not get the container log reader.
|
||||
*/
|
||||
@Private
|
||||
public ContainerLogsReader getContainerLogsReader(
|
||||
|
@ -752,10 +753,10 @@ public class AggregatedLogFormat {
|
|||
|
||||
/**
|
||||
* Writes all logs for a single container to the provided writer.
|
||||
* @param valueStream
|
||||
* @param writer
|
||||
* @param logUploadedTime
|
||||
* @throws IOException
|
||||
* @param valueStream the valueStream
|
||||
* @param writer the log writer
|
||||
* @param logUploadedTime the time stamp
|
||||
* @throws IOException if we can not read the container logs.
|
||||
*/
|
||||
public static void readAcontainerLogs(DataInputStream valueStream,
|
||||
Writer writer, long logUploadedTime) throws IOException {
|
||||
|
@ -780,9 +781,9 @@ public class AggregatedLogFormat {
|
|||
|
||||
/**
|
||||
* Writes all logs for a single container to the provided writer.
|
||||
* @param valueStream
|
||||
* @param writer
|
||||
* @throws IOException
|
||||
* @param valueStream the value stream
|
||||
* @param writer the log writer
|
||||
* @throws IOException if we can not read the container logs.
|
||||
*/
|
||||
public static void readAcontainerLogs(DataInputStream valueStream,
|
||||
Writer writer) throws IOException {
|
||||
|
@ -849,10 +850,11 @@ public class AggregatedLogFormat {
|
|||
* Keep calling this till you get a {@link EOFException} for getting logs of
|
||||
* all types for a single container.
|
||||
*
|
||||
* @param valueStream
|
||||
* @param out
|
||||
* @param logUploadedTime
|
||||
* @throws IOException
|
||||
* @param valueStream the value stream
|
||||
* @param out the print stream
|
||||
* @param logUploadedTime the time stamp
|
||||
* @throws IOException if we can not read the container log by specifying
|
||||
* the container log type.
|
||||
*/
|
||||
public static void readAContainerLogsForALogType(
|
||||
DataInputStream valueStream, PrintStream out, long logUploadedTime)
|
||||
|
@ -864,11 +866,11 @@ public class AggregatedLogFormat {
|
|||
* Keep calling this till you get a {@link EOFException} for getting logs of
|
||||
* all types for a single container for the specific bytes.
|
||||
*
|
||||
* @param valueStream
|
||||
* @param out
|
||||
* @param logUploadedTime
|
||||
* @param bytes
|
||||
* @throws IOException
|
||||
* @param valueStream the value stream
|
||||
* @param out the output print stream
|
||||
* @param logUploadedTime the log upload time stamp
|
||||
* @param bytes the output size of the log
|
||||
* @throws IOException if we can not read the container log
|
||||
*/
|
||||
public static void readAContainerLogsForALogType(
|
||||
DataInputStream valueStream, PrintStream out, long logUploadedTime,
|
||||
|
@ -880,9 +882,9 @@ public class AggregatedLogFormat {
|
|||
* Keep calling this till you get a {@link EOFException} for getting logs of
|
||||
* all types for a single container.
|
||||
*
|
||||
* @param valueStream
|
||||
* @param out
|
||||
* @throws IOException
|
||||
* @param valueStream the value stream
|
||||
* @param out the output print stream
|
||||
* @throws IOException if we can not read the container log
|
||||
*/
|
||||
public static void readAContainerLogsForALogType(
|
||||
DataInputStream valueStream, PrintStream out)
|
||||
|
@ -893,11 +895,11 @@ public class AggregatedLogFormat {
|
|||
/**
|
||||
* Keep calling this till you get a {@link EOFException} for getting logs of
|
||||
* the specific types for a single container.
|
||||
* @param valueStream
|
||||
* @param out
|
||||
* @param logUploadedTime
|
||||
* @param logType
|
||||
* @throws IOException
|
||||
* @param valueStream the value stream
|
||||
* @param out the output print stream
|
||||
* @param logUploadedTime the log uploaded time stamp
|
||||
* @param logType the given log type
|
||||
* @throws IOException if we can not read the container logs
|
||||
*/
|
||||
public static int readContainerLogsForALogType(
|
||||
DataInputStream valueStream, PrintStream out, long logUploadedTime,
|
||||
|
@ -909,11 +911,11 @@ public class AggregatedLogFormat {
|
|||
/**
|
||||
* Keep calling this till you get a {@link EOFException} for getting logs of
|
||||
* the specific types for a single container.
|
||||
* @param valueStream
|
||||
* @param out
|
||||
* @param logUploadedTime
|
||||
* @param logType
|
||||
* @throws IOException
|
||||
* @param valueStream the value stream
|
||||
* @param out the output print stream
|
||||
* @param logUploadedTime the log uploaded time stamp
|
||||
* @param logType the given log type
|
||||
* @throws IOException if we can not read the container logs
|
||||
*/
|
||||
public static int readContainerLogsForALogType(
|
||||
DataInputStream valueStream, PrintStream out, long logUploadedTime,
|
||||
|
|
|
@ -38,11 +38,11 @@ public class LogAggregationUtils {
|
|||
|
||||
/**
|
||||
* Constructs the full filename for an application's log file per node.
|
||||
* @param remoteRootLogDir
|
||||
* @param appId
|
||||
* @param user
|
||||
* @param nodeId
|
||||
* @param suffix
|
||||
* @param remoteRootLogDir the aggregated remote root log dir
|
||||
* @param appId the application Id
|
||||
* @param user the application owner
|
||||
* @param nodeId the node id
|
||||
* @param suffix the log dir suffix
|
||||
* @return the remote log file.
|
||||
*/
|
||||
public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir,
|
||||
|
@ -53,10 +53,10 @@ public class LogAggregationUtils {
|
|||
|
||||
/**
|
||||
* Gets the remote app log dir.
|
||||
* @param remoteRootLogDir
|
||||
* @param appId
|
||||
* @param user
|
||||
* @param suffix
|
||||
* @param remoteRootLogDir the aggregated log remote root log dir
|
||||
* @param appId the application id
|
||||
* @param user the application owner
|
||||
* @param suffix the log directory suffix
|
||||
* @return the remote application specific log dir.
|
||||
*/
|
||||
public static Path getRemoteAppLogDir(Path remoteRootLogDir,
|
||||
|
@ -67,9 +67,9 @@ public class LogAggregationUtils {
|
|||
|
||||
/**
|
||||
* Gets the remote suffixed log dir for the user.
|
||||
* @param remoteRootLogDir
|
||||
* @param user
|
||||
* @param suffix
|
||||
* @param remoteRootLogDir the aggregated log remote root log dir
|
||||
* @param user the application owner
|
||||
* @param suffix the log dir suffix
|
||||
* @return the remote suffixed log dir.
|
||||
*/
|
||||
public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir,
|
||||
|
@ -83,8 +83,8 @@ public class LogAggregationUtils {
|
|||
|
||||
/**
|
||||
* Gets the remote log user dir.
|
||||
* @param remoteRootLogDir
|
||||
* @param user
|
||||
* @param remoteRootLogDir the aggregated log remote root log dir
|
||||
* @param user the application owner
|
||||
* @return the remote per user log dir.
|
||||
*/
|
||||
public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) {
|
||||
|
@ -93,7 +93,7 @@ public class LogAggregationUtils {
|
|||
|
||||
/**
|
||||
* Returns the suffix component of the log dir.
|
||||
* @param conf
|
||||
* @param conf the configuration
|
||||
* @return the suffix which will be appended to the user log dir.
|
||||
*/
|
||||
public static String getRemoteNodeLogDirSuffix(Configuration conf) {
|
||||
|
@ -104,7 +104,7 @@ public class LogAggregationUtils {
|
|||
|
||||
/**
|
||||
* Converts a nodeId to a form used in the app log file name.
|
||||
* @param nodeId
|
||||
* @param nodeId the nodeId
|
||||
* @return the node string to be used to construct the file name.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
|
|
|
@ -128,8 +128,17 @@ public class LogCLIHelpers implements Configurable {
|
|||
@VisibleForTesting
|
||||
public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
|
||||
boolean outputFailure) throws IOException {
|
||||
boolean foundAnyLogs = this.getFileController(options.getAppId(),
|
||||
options.getAppOwner()).readAggregatedLogs(options, null);
|
||||
LogAggregationFileController fc = null;
|
||||
try {
|
||||
fc = this.getFileController(
|
||||
options.getAppId(), options.getAppOwner());
|
||||
} catch (IOException ex) {
|
||||
System.err.println(ex);
|
||||
}
|
||||
boolean foundAnyLogs = false;
|
||||
if (fc != null) {
|
||||
foundAnyLogs = fc.readAggregatedLogs(options, null);
|
||||
}
|
||||
if (!foundAnyLogs) {
|
||||
if (outputFailure) {
|
||||
containerLogNotFound(options.getContainerId());
|
||||
|
@ -142,9 +151,17 @@ public class LogCLIHelpers implements Configurable {
|
|||
@Private
|
||||
public int dumpAContainerLogsForLogTypeWithoutNodeId(
|
||||
ContainerLogsRequest options) throws IOException {
|
||||
boolean foundAnyLogs = getFileController(options.getAppId(),
|
||||
options.getAppOwner()).readAggregatedLogs(
|
||||
options, null);
|
||||
LogAggregationFileController fc = null;
|
||||
try {
|
||||
fc = this.getFileController(
|
||||
options.getAppId(), options.getAppOwner());
|
||||
} catch (IOException ex) {
|
||||
System.err.println(ex);
|
||||
}
|
||||
boolean foundAnyLogs = false;
|
||||
if (fc != null) {
|
||||
foundAnyLogs = fc.readAggregatedLogs(options, null);
|
||||
}
|
||||
if (!foundAnyLogs) {
|
||||
containerLogNotFound(options.getContainerId());
|
||||
return -1;
|
||||
|
@ -155,9 +172,17 @@ public class LogCLIHelpers implements Configurable {
|
|||
@Private
|
||||
public int dumpAllContainersLogs(ContainerLogsRequest options)
|
||||
throws IOException {
|
||||
boolean foundAnyLogs = getFileController(options.getAppId(),
|
||||
options.getAppOwner()).readAggregatedLogs(
|
||||
options, null);
|
||||
LogAggregationFileController fc = null;
|
||||
try {
|
||||
fc = this.getFileController(
|
||||
options.getAppId(), options.getAppOwner());
|
||||
} catch (IOException ex) {
|
||||
System.err.println(ex);
|
||||
}
|
||||
boolean foundAnyLogs = false;
|
||||
if (fc != null) {
|
||||
foundAnyLogs = fc.readAggregatedLogs(options, null);
|
||||
}
|
||||
if (!foundAnyLogs) {
|
||||
emptyLogDir(LogAggregationUtils.getRemoteAppLogDir(
|
||||
conf, options.getAppId(), options.getAppOwner())
|
||||
|
|
|
@ -201,6 +201,7 @@ public abstract class LogAggregationFileController {
|
|||
* Output container log.
|
||||
* @param logRequest {@link ContainerLogsRequest}
|
||||
* @param os the output stream
|
||||
* @return true if we can read the aggregated logs successfully
|
||||
* @throws IOException if we can not access the log file.
|
||||
*/
|
||||
public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest,
|
||||
|
@ -228,9 +229,9 @@ public abstract class LogAggregationFileController {
|
|||
/**
|
||||
* Returns the owner of the application.
|
||||
*
|
||||
* @param the aggregatedLog path.
|
||||
* @return the application owner.
|
||||
* @throws IOException
|
||||
* @param aggregatedLogPath the aggregatedLog path
|
||||
* @return the application owner
|
||||
* @throws IOException if we can not get the application owner
|
||||
*/
|
||||
public abstract String getApplicationOwner(Path aggregatedLogPath)
|
||||
throws IOException;
|
||||
|
@ -239,9 +240,9 @@ public abstract class LogAggregationFileController {
|
|||
* Returns ACLs for the application. An empty map is returned if no ACLs are
|
||||
* found.
|
||||
*
|
||||
* @param the aggregatedLog path.
|
||||
* @param aggregatedLogPath the aggregatedLog path.
|
||||
* @return a map of the Application ACLs.
|
||||
* @throws IOException
|
||||
* @throws IOException if we can not get the application acls
|
||||
*/
|
||||
public abstract Map<ApplicationAccessType, String> getApplicationAcls(
|
||||
Path aggregatedLogPath) throws IOException;
|
||||
|
|
|
@ -172,7 +172,6 @@ public class LogAggregationTFileController
|
|||
byte[] buf = new byte[65535];
|
||||
while (nodeFiles != null && nodeFiles.hasNext()) {
|
||||
final FileStatus thisNodeFile = nodeFiles.next();
|
||||
LOG.error(thisNodeFile.getPath().toString());
|
||||
String nodeName = thisNodeFile.getPath().getName();
|
||||
if (nodeName.equals(appId + ".har")) {
|
||||
Path p = new Path("har:///"
|
||||
|
|
Loading…
Reference in New Issue