Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5199. Close LogReader in in AHSWebServices#getStreamingOutput and 

FileInputStream in NMWebServices#getLogs. Contributed by Xuan Gong

(cherry picked from commit 58be55b6e0)
This commit is contained in:
Xuan 2016-06-07 16:07:02 -07:00
parent 6a9f38ebaf
commit 10f0c0475e
2 changed files with 118 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
View File

@ -40,7 +40,6 @@
import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.Response.Status;
import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -363,8 +362,9 @@ public void write(OutputStream os) throws IOException,
if ((nodeId == null || nodeName.contains(LogAggregationUtils if ((nodeId == null || nodeName.contains(LogAggregationUtils
.getNodeString(nodeId))) && !nodeName.endsWith( .getNodeString(nodeId))) && !nodeName.endsWith(
LogAggregationUtils.TMP_FILE_SUFFIX)) { LogAggregationUtils.TMP_FILE_SUFFIX)) {
AggregatedLogFormat.LogReader reader = AggregatedLogFormat.LogReader reader = null;
new AggregatedLogFormat.LogReader(conf, try {
reader = new AggregatedLogFormat.LogReader(conf,
thisNodeFile.getPath()); thisNodeFile.getPath());
DataInputStream valueStream; DataInputStream valueStream;
LogKey key = new LogKey(); LogKey key = new LogKey();
@ -392,7 +392,8 @@ public void write(OutputStream os) throws IOException,
sb.append("LogLength:"); sb.append("LogLength:");
sb.append(fileLengthStr + "\n"); sb.append(fileLengthStr + "\n");
sb.append("Log Contents:\n"); sb.append("Log Contents:\n");
byte[] b = sb.toString().getBytes(Charset.forName("UTF-8")); byte[] b = sb.toString().getBytes(
Charset.forName("UTF-8"));
os.write(b, 0, b.length); os.write(b, 0, b.length);
long toSkip = 0; long toSkip = 0;
@ -437,7 +438,8 @@ public void write(OutputStream os) throws IOException,
long totalSkipped = 0; long totalSkipped = 0;
long currSkipped = 0; long currSkipped = 0;
while (currSkipped != -1 && totalSkipped < fileLength) { while (currSkipped != -1 && totalSkipped < fileLength) {
currSkipped = valueStream.skip(fileLength - totalSkipped); currSkipped = valueStream.skip(
fileLength - totalSkipped);
totalSkipped += currSkipped; totalSkipped += currSkipped;
} }
} }
@ -445,6 +447,11 @@ public void write(OutputStream os) throws IOException,
break; break;
} }
} }
} finally {
if (reader != null) {
reader.close();
}
}
} }
} }
os.flush(); os.flush();

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
View File

@ -37,7 +37,7 @@
import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo; import javax.ws.rs.core.UriInfo;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -236,7 +236,6 @@ public Response getLogs(@PathParam("containerid") String containerIdStr,
} }
boolean downloadFile = parseBooleanParam(download); boolean downloadFile = parseBooleanParam(download);
final long bytes = parseLongParam(size); final long bytes = parseLongParam(size);
try { try {
final FileInputStream fis = ContainerLogsUtils.openLogFileForRead( final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(
containerIdStr, logFile, nmContext); containerIdStr, logFile, nmContext);
@ -246,6 +245,7 @@ public Response getLogs(@PathParam("containerid") String containerIdStr,
@Override @Override
public void write(OutputStream os) throws IOException, public void write(OutputStream os) throws IOException,
WebApplicationException { WebApplicationException {
try {
int bufferSize = 65536; int bufferSize = 65536;
byte[] buf = new byte[bufferSize]; byte[] buf = new byte[bufferSize];
long toSkip = 0; long toSkip = 0;
@ -282,6 +282,9 @@ public void write(OutputStream os) throws IOException,
len = fis.read(buf, 0, toRead); len = fis.read(buf, 0, toRead);
} }
os.flush(); os.flush();
} finally {
IOUtils.closeQuietly(fis);
}
} }
}; };
ResponseBuilder resp = Response.ok(stream); ResponseBuilder resp = Response.ok(stream);