From 20b49d292c4b3928acc060ca16d1c2f36d57c3de Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Tue, 20 Aug 2013 22:45:12 +0000 Subject: [PATCH] HDFS-4594. Merging change r1515989 from trunk to branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1516008 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../web/resources/DatanodeWebHdfsMethods.java | 5 +- .../web/TestWebHdfsFileSystemContract.java | 102 ++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index be8ca3a2095..d3ebbf2585d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -146,6 +146,9 @@ Release 2.1.1-beta - UNRELEASED HDFS-5106. TestDatanodeBlockScanner fails on Windows due to incorrect path format. (Chuan Liu via cnauroth) + HDFS-4594. WebHDFS open sets Content-Length header to what is specified by + length parameter rather than how much data is actually returned. (cnauroth) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java index 262b66f9bcf..973d0916b90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java @@ -410,8 +410,9 @@ private Response get( throw ioe; } - final long n = length.getValue() != null? length.getValue() - : in.getVisibleLength() - offset.getValue(); + final long n = length.getValue() != null ? + Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) : + in.getVisibleLength() - offset.getValue(); return Response.ok(new OpenEntity(in, n, dfsclient)).type( MediaType.APPLICATION_OCTET_STREAM).build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java index 343aa775d0b..4181ce60376 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java @@ -21,6 +21,7 @@ import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; @@ -45,8 +46,11 @@ import org.apache.hadoop.hdfs.web.resources.DoAsParam; import org.apache.hadoop.hdfs.web.resources.GetOpParam; import org.apache.hadoop.hdfs.web.resources.HttpOpParam; +import org.apache.hadoop.hdfs.web.resources.LengthParam; import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam; +import org.apache.hadoop.hdfs.web.resources.OffsetParam; import org.apache.hadoop.hdfs.web.resources.PutOpParam; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.Assert; @@ -288,6 +292,104 @@ public void testRootDir() throws IOException { } } + /** + * Test get with length parameter greater than actual file length. + */ + public void testLengthParamLongerThanFile() throws IOException { + WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs; + Path dir = new Path("/test"); + assertTrue(webhdfs.mkdirs(dir)); + + // Create a file with some content. + Path testFile = new Path("/test/testLengthParamLongerThanFile"); + String content = "testLengthParamLongerThanFile"; + FSDataOutputStream testFileOut = webhdfs.create(testFile); + try { + testFileOut.write(content.getBytes("US-ASCII")); + } finally { + IOUtils.closeStream(testFileOut); + } + + // Open the file, but request length longer than actual file length by 1. + HttpOpParam.Op op = GetOpParam.Op.OPEN; + URL url = webhdfs.toUrl(op, testFile, new LengthParam(Long.valueOf( + content.length() + 1))); + HttpURLConnection conn = null; + InputStream is = null; + try { + conn = (HttpURLConnection)url.openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(op.getDoOutput()); + conn.setInstanceFollowRedirects(true); + + // Expect OK response and Content-Length header equal to actual length. + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertEquals(String.valueOf(content.length()), conn.getHeaderField( + "Content-Length")); + + // Check content matches. + byte[] respBody = new byte[content.length()]; + is = conn.getInputStream(); + IOUtils.readFully(is, respBody, 0, content.length()); + assertEquals(content, new String(respBody, "US-ASCII")); + } finally { + IOUtils.closeStream(is); + if (conn != null) { + conn.disconnect(); + } + } + } + + /** + * Test get with offset and length parameters that combine to request a length + * greater than actual file length. + */ + public void testOffsetPlusLengthParamsLongerThanFile() throws IOException { + WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs; + Path dir = new Path("/test"); + assertTrue(webhdfs.mkdirs(dir)); + + // Create a file with some content. + Path testFile = new Path("/test/testOffsetPlusLengthParamsLongerThanFile"); + String content = "testOffsetPlusLengthParamsLongerThanFile"; + FSDataOutputStream testFileOut = webhdfs.create(testFile); + try { + testFileOut.write(content.getBytes("US-ASCII")); + } finally { + IOUtils.closeStream(testFileOut); + } + + // Open the file, but request offset starting at 1 and length equal to file + // length. Considering the offset, this is longer than the actual content. + HttpOpParam.Op op = GetOpParam.Op.OPEN; + URL url = webhdfs.toUrl(op, testFile, new LengthParam(Long.valueOf( + content.length())), new OffsetParam(1L)); + HttpURLConnection conn = null; + InputStream is = null; + try { + conn = (HttpURLConnection)url.openConnection(); + conn.setRequestMethod(op.getType().toString()); + conn.setDoOutput(op.getDoOutput()); + conn.setInstanceFollowRedirects(true); + + // Expect OK response and Content-Length header equal to actual length. + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertEquals(String.valueOf(content.length() - 1), conn.getHeaderField( + "Content-Length")); + + // Check content matches. + byte[] respBody = new byte[content.length() - 1]; + is = conn.getInputStream(); + IOUtils.readFully(is, respBody, 0, content.length() - 1); + assertEquals(content.substring(1), new String(respBody, "US-ASCII")); + } finally { + IOUtils.closeStream(is); + if (conn != null) { + conn.disconnect(); + } + } + } + public void testResponseCode() throws IOException { final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs; final Path root = new Path("/");