From ca3e09bab5df35a1360deea1746fdd914c043814 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 26 Mar 2013 09:54:48 +0000 Subject: [PATCH] HDFS-4598. Fix the default value of ConcatSourcesParam and the WebHDFS doc. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461040 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/DistributedFileSystem.java | 7 ++-- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 39 +++++++++++++------ .../web/resources/ConcatSourcesParam.java | 19 ++++++++- .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 15 ++----- .../hadoop/hdfs/web/resources/TestParam.java | 26 +++++++++++-- 6 files changed, 80 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8253761f9c7..aff2f516f22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -448,6 +448,9 @@ Release 2.0.5-beta - UNRELEASED HDFS-4584. Skip TestNNWithQJM.testNewNamenodeTakesOverWriter() on Windows. (Arpit Agarwal via szetszwo) + HDFS-4598. Fix the default value of ConcatSourcesParam and the WebHDFS doc. + (szetszwo) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index d5d5d626ea8..3476f678708 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -312,13 +312,14 @@ public class DistributedFileSystem extends FileSystem { } /** - * Move blocks from srcs to trg - * and delete srcs afterwards - * RESTRICTION: all blocks should be the same size + * Move blocks from srcs to trg and delete srcs afterwards. + * The file block sizes must be the same. + * * @param trg existing file to append to * @param psrcs list of files (same block size, same replication) * @throws IOException */ + @Override public void concat(Path trg, Path [] psrcs) throws IOException { String [] srcs = new String [psrcs.length]; for(int i=0; i strPaths = new ArrayList(psrcs.length); - for(Path psrc : psrcs) { - strPaths.add(psrc.toUri().getPath()); - } - - String srcs = StringUtils.join(",", strPaths); - ConcatSourcesParam param = new ConcatSourcesParam(srcs); run(op, trg, param); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java index e6afbe3e4ec..b68c5f5b58e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java @@ -18,15 +18,28 @@ package org.apache.hadoop.hdfs.web.resources; +import org.apache.hadoop.fs.Path; + /** The concat source paths parameter. */ public class ConcatSourcesParam extends StringParam { /** Parameter name. */ public static final String NAME = "sources"; - public static final String DEFAULT = NULL; + public static final String DEFAULT = ""; private static final Domain DOMAIN = new Domain(NAME, null); + private static String paths2String(Path[] paths) { + if (paths == null || paths.length == 0) { + return ""; + } + final StringBuilder b = new StringBuilder(paths[0].toUri().getPath()); + for(int i = 1; i < paths.length; i++) { + b.append(',').append(paths[i].toUri().getPath()); + } + return b.toString(); + } + /** * Constructor. * @param str a string representation of the parameter value. @@ -35,6 +48,10 @@ public class ConcatSourcesParam extends StringParam { super(DOMAIN, str); } + public ConcatSourcesParam(Path[] paths) { + this(paths2String(paths)); + } + @Override public String getName() { return NAME; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm index 90f8dabce71..7735f8dafed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm @@ -109,7 +109,7 @@ WebHDFS REST API * {{{Append to a File}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append) - * {{{Concat File(s)}<<>>}} + * {{{Concatenate Files}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat) * HTTP DELETE @@ -307,7 +307,7 @@ Content-Length: 0 * Submit a HTTP POST request. +--------------------------------- -curl -i -X POST "http://:/webhdfs/v1/?op=CONCAT&sources=" +curl -i -X POST "http://:/webhdfs/v1/?op=CONCAT&sources=" +--------------------------------- The client receives a response with zero content length: @@ -319,10 +319,6 @@ Content-Length: 0 [] - This REST API call is available as of Hadoop version 2.0.3. - Please note that is a comma seperated list of absolute paths. - (Example: sources=/test/file1,/test/file2,/test/file3) - See also: {{{Sources}<<>>}}, {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat @@ -1761,7 +1757,7 @@ var tokenProperties = *----------------+-------------------------------------------------------------------+ || Name | <<>> | *----------------+-------------------------------------------------------------------+ -|| Description | The comma seperated absolute paths used for concatenation. | +|| Description | A list of source paths. | *----------------+-------------------------------------------------------------------+ || Type | String | *----------------+-------------------------------------------------------------------+ @@ -1769,12 +1765,9 @@ var tokenProperties = *----------------+-------------------------------------------------------------------+ || Valid Values | A list of comma seperated absolute FileSystem paths without scheme and authority. | *----------------+-------------------------------------------------------------------+ -|| Syntax | See the note in {{Delegation}}. | +|| Syntax | Any string. | *----------------+-------------------------------------------------------------------+ - <> that sources are absolute FileSystem paths. - - See also: {{{Concat File(s)}<<>>}} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index c228c1f2989..97223ab2fb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -17,18 +17,22 @@ */ package org.apache.hadoop.hdfs.web.resources; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.util.Arrays; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - public class TestParam { public static final Log LOG = LogFactory.getLog(TestParam.class); @@ -265,4 +269,20 @@ public class TestParam { UserParam userParam = new UserParam("a$"); assertNotNull(userParam.getValue()); } + + @Test + public void testConcatSourcesParam() { + final String[] strings = {"/", "/foo", "/bar"}; + for(int n = 0; n < strings.length; n++) { + final String[] sub = new String[n]; + final Path[] paths = new Path[n]; + for(int i = 0; i < paths.length; i++) { + paths[i] = new Path(sub[i] = strings[i]); + } + + final String expected = StringUtils.join(",", Arrays.asList(sub)); + final ConcatSourcesParam computed = new ConcatSourcesParam(paths); + Assert.assertEquals(expected, computed.getValue()); + } + } }