Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1587954 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Daryn Sharp 2014-04-16 16:05:22 +00:00
parent 14fee881fd
commit 5e47eeca9d
3 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -301,6 +301,8 @@ Release 2.5.0 - UNRELEASED
OPTIMIZATIONS OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
BUG FIXES BUG FIXES
HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration. HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration.

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
View File

@ -430,10 +430,18 @@ public class DatanodeWebHdfsMethods {
Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) : Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
in.getVisibleLength() - offset.getValue(); in.getVisibleLength() - offset.getValue();
// jetty 6 reserves 12 bytes in the out buffer for chunked responses
// (file length > 2GB) which causes extremely poor performance when
// 12 bytes of the output spill into another buffer which results
// in a big and little write
int outBufferSize = response.getBufferSize();
if (n > Integer.MAX_VALUE) {
outBufferSize -= 12;
}
/** /**
* Allow the Web UI to perform an AJAX request to get the data. * Allow the Web UI to perform an AJAX request to get the data.
*/ */
return Response.ok(new OpenEntity(in, n, dfsclient)) return Response.ok(new OpenEntity(in, n, outBufferSize, dfsclient))
.type(MediaType.APPLICATION_OCTET_STREAM) .type(MediaType.APPLICATION_OCTET_STREAM)
.header("Access-Control-Allow-Methods", "GET") .header("Access-Control-Allow-Methods", "GET")
.header("Access-Control-Allow-Origin", "*") .header("Access-Control-Allow-Origin", "*")

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
View File

@ -37,12 +37,14 @@ import org.apache.hadoop.io.IOUtils;
public class OpenEntity { public class OpenEntity {
private final HdfsDataInputStream in; private final HdfsDataInputStream in;
private final long length; private final long length;
private final int outBufferSize;
private final DFSClient dfsclient; private final DFSClient dfsclient;
OpenEntity(final HdfsDataInputStream in, final long length, OpenEntity(final HdfsDataInputStream in, final long length,
final DFSClient dfsclient) { final int outBufferSize, final DFSClient dfsclient) {
this.in = in; this.in = in;
this.length = length; this.length = length;
this.outBufferSize = outBufferSize;
this.dfsclient = dfsclient; this.dfsclient = dfsclient;
} }
@ -71,7 +73,17 @@ public class OpenEntity {
MultivaluedMap<String, Object> httpHeaders, OutputStream out MultivaluedMap<String, Object> httpHeaders, OutputStream out
) throws IOException { ) throws IOException {
try { try {
IOUtils.copyBytes(e.in, out, e.length, false); byte[] buf = new byte[e.outBufferSize];
long remaining = e.length;
while (remaining > 0) {
int read = e.in.read(buf, 0, (int)Math.min(buf.length, remaining));
if (read == -1) { // EOF
break;
}
out.write(buf, 0, read);
out.flush();
remaining -= read;
}
} finally { } finally {
IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.in); IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.in);
IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.dfsclient); IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.dfsclient);