diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index a2354784d0f..537807227d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -249,6 +249,10 @@ src/test/resources/application_1440536969523_0001.har/part-0 src/test/resources/application_1440536969523_0001.har/_masterindex src/test/resources/application_1440536969523_0001.har/_SUCCESS + src/test/resources/application_123456_0001.har/_index + src/test/resources/application_123456_0001.har/part-0 + src/test/resources/application_123456_0001.har/_masterindex + src/test/resources/application_123456_0001.har/_SUCCESS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index 56bae26d939..5bba2e0a409 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -495,16 +495,21 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, boolean getAllContainers = (containerIdStr == null || containerIdStr.isEmpty()); long size = logRequest.getBytes(); - List nodeFiles = LogAggregationUtils - .getRemoteNodeFileList(conf, appId, logRequest.getAppOwner(), + RemoteIterator nodeFiles = LogAggregationUtils + .getRemoteNodeFileDir(conf, appId, logRequest.getAppOwner(), this.remoteRootLogDir, this.remoteRootLogDirSuffix); - if (nodeFiles.isEmpty()) { + if (!nodeFiles.hasNext()) { throw new IOException("There is no available log fils for " + "application:" + appId); } - Map checkSumFiles = parseCheckSumFiles(nodeFiles); + List allFiles = getAllNodeFiles(nodeFiles, appId); + if (allFiles.isEmpty()) { + throw new IOException("There is no available log fils for " + + "application:" + appId); + } + Map checkSumFiles = parseCheckSumFiles(allFiles); List fileToRead = getNodeLogFileToRead( - nodeFiles, nodeIdStr, appId); + allFiles, nodeIdStr, appId); byte[] buf = new byte[65535]; for (FileStatus thisNodeFile : fileToRead) { String nodeName = thisNodeFile.getPath().getName(); @@ -609,16 +614,21 @@ public List readAggregatedLogsMeta( containerIdStr.isEmpty()); String nodeIdStr = (nodeId == null || nodeId.isEmpty()) ? null : LogAggregationUtils.getNodeString(nodeId); - List nodeFiles = LogAggregationUtils - .getRemoteNodeFileList(conf, appId, appOwner, this.remoteRootLogDir, + RemoteIterator nodeFiles = LogAggregationUtils + .getRemoteNodeFileDir(conf, appId, appOwner, this.remoteRootLogDir, this.remoteRootLogDirSuffix); - if (nodeFiles.isEmpty()) { + if (!nodeFiles.hasNext()) { throw new IOException("There is no available log fils for " + "application:" + appId); } - Map checkSumFiles = parseCheckSumFiles(nodeFiles); + List allFiles = getAllNodeFiles(nodeFiles, appId); + if (allFiles.isEmpty()) { + throw new IOException("There is no available log fils for " + + "application:" + appId); + } + Map checkSumFiles = parseCheckSumFiles(allFiles); List fileToRead = getNodeLogFileToRead( - nodeFiles, nodeIdStr, appId); + allFiles, nodeIdStr, appId); for(FileStatus thisNodeFile : fileToRead) { try { Long checkSumIndex = checkSumFiles.get( @@ -727,25 +737,37 @@ public List getNodeLogFileToRead( List nodeFiles, String nodeId, ApplicationId appId) throws IOException { List listOfFiles = new ArrayList<>(); - List files = new ArrayList<>(nodeFiles); - for (FileStatus file : files) { - String nodeName = file.getPath().getName(); + for (FileStatus thisNodeFile : nodeFiles) { + String nodeName = thisNodeFile.getPath().getName(); if ((nodeId == null || nodeId.isEmpty() || nodeName.contains(LogAggregationUtils .getNodeString(nodeId))) && !nodeName.endsWith( LogAggregationUtils.TMP_FILE_SUFFIX) && !nodeName.endsWith(CHECK_SUM_FILE_SUFFIX)) { - if (nodeName.equals(appId + ".har")) { - Path p = new Path("har:///" + file.getPath().toUri().getRawPath()); - files = Arrays.asList(HarFs.get(p.toUri(), conf).listStatus(p)); - continue; - } - listOfFiles.add(file); + listOfFiles.add(thisNodeFile); } } return listOfFiles; } + private List getAllNodeFiles( + RemoteIterator nodeFiles, ApplicationId appId) + throws IOException { + List listOfFiles = new ArrayList<>(); + while (nodeFiles != null && nodeFiles.hasNext()) { + FileStatus thisNodeFile = nodeFiles.next(); + String nodeName = thisNodeFile.getPath().getName(); + if (nodeName.equals(appId + ".har")) { + Path p = new Path("har:///" + + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } + listOfFiles.add(thisNodeFile); + } + return listOfFiles; + } + @Private public FileStatus getAllChecksumFiles(Map fileMap, String fileName) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java index 9c02c1b1488..79226797578 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller.ifile; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; @@ -27,6 +28,7 @@ import java.io.IOException; import java.io.PrintStream; import java.io.Writer; +import java.net.URL; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; @@ -364,6 +366,58 @@ public boolean isRollover(final FileContext fc, sysOutStream.reset(); } + @Test(timeout = 15000) + public void testFetchApplictionLogsHar() throws Exception { + List newLogTypes = new ArrayList<>(); + newLogTypes.add("syslog"); + newLogTypes.add("stdout"); + newLogTypes.add("stderr"); + newLogTypes.add("test1"); + newLogTypes.add("test2"); + URL harUrl = ClassLoader.getSystemClassLoader() + .getResource("application_123456_0001.har"); + assertNotNull(harUrl); + + Path path = new Path(remoteLogDir + "/" + USER_UGI.getShortUserName() + + "/logs/application_123456_0001"); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + Path harPath = new Path(path, "application_123456_0001.har"); + fs.copyFromLocalFile(false, new Path(harUrl.toURI()), harPath); + assertTrue(fs.exists(harPath)); + LogAggregationIndexedFileController fileFormat + = new LogAggregationIndexedFileController(); + fileFormat.initialize(conf, "Indexed"); + ContainerLogsRequest logRequest = new ContainerLogsRequest(); + logRequest.setAppId(appId); + logRequest.setNodeId(nodeId.toString()); + logRequest.setAppOwner(USER_UGI.getShortUserName()); + logRequest.setContainerId(containerId.toString()); + logRequest.setBytes(Long.MAX_VALUE); + List meta = fileFormat.readAggregatedLogsMeta( + logRequest); + Assert.assertEquals(meta.size(), 3); + List fileNames = new ArrayList<>(); + for (ContainerLogMeta log : meta) { + Assert.assertTrue(log.getContainerId().equals(containerId.toString())); + Assert.assertTrue(log.getNodeId().equals(nodeId.toString())); + for (ContainerLogFileInfo file : log.getContainerLogMeta()) { + fileNames.add(file.getFileName()); + } + } + fileNames.removeAll(newLogTypes); + Assert.assertTrue(fileNames.isEmpty()); + boolean foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out); + Assert.assertTrue(foundLogs); + for (String logType : newLogTypes) { + Assert.assertTrue(sysOutStream.toString().contains(logMessage( + containerId, logType))); + } + sysOutStream.reset(); + } + private File createAndWriteLocalLogFile(ContainerId containerId, Path localLogDir, String logType) throws IOException { File file = new File(localLogDir.toString(), logType); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_SUCCESS b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_SUCCESS new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_index b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_index new file mode 100644 index 00000000000..b042846e1b0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_index @@ -0,0 +1,3 @@ +%2F dir 1517728311922+493+xuan+supergroup 0 0 localhost_9999_1517727665265 localhost_9999_1517727668513 +%2Flocalhost_9999_1517727665265 file part-0 0 2895 1517728301581+420+xuan+supergroup +%2Flocalhost_9999_1517727668513 file part-0 2895 1228 1517728311919+420+xuan+supergroup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_masterindex b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_masterindex new file mode 100644 index 00000000000..cda8cbdcab0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/_masterindex @@ -0,0 +1,2 @@ +3 +0 1897968749 0 280 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/part-0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/part-0 new file mode 100644 index 00000000000..aea95fa8da3 Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_123456_0001.har/part-0 differ