From f5e83a0b3e33376d3378b23f6889d954f4e975f3 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 4 Dec 2013 22:24:45 +0000 Subject: [PATCH] HDFS-4983. Numeric usernames do not work with WebHDFS FS. Contributed by Yongjun Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1547935 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/namenode/NameNodeHttpServer.java | 6 ++- .../hadoop/hdfs/web/WebHdfsConfigKeys.java | 38 +++++++++++++++++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 2 + .../hadoop/hdfs/web/resources/UserParam.java | 35 ++++++++++++++--- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 29 ++++++++++++++ .../hadoop/hdfs/web/resources/TestParam.java | 15 ++++++++ 7 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 72c44d0bb2f..0d5ae680a5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -775,6 +775,9 @@ Release 2.3.0 - UNRELEASED HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe) + HDFS-4983. Numeric usernames do not work with WebHDFS FS. + (Yongjun Zhang via wang) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index b72d546d14e..5e7198d64d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMetho import org.apache.hadoop.hdfs.web.AuthFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; +import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.net.NetUtils; @@ -73,7 +74,10 @@ public class NameNodeHttpServer { private void initWebHdfs(Configuration conf) throws IOException { if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) { - //add SPNEGO authentication filter for webhdfs + // set user pattern based on configuration file + UserParam.setUserPattern(conf); + + // add SPNEGO authentication filter for webhdfs final String name = "SPNEGO"; final String classname = AuthFilter.class.getName(); final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java new file mode 100644 index 00000000000..a4165a08466 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConfigKeys.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.web; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.DFSConfigKeys; + +/** + * This class contains constants for configuration keys used + * in WebHdfs. + * + */ + +@InterfaceAudience.Private +public class WebHdfsConfigKeys extends DFSConfigKeys { + /** User name pattern key */ + public static final String USER_PATTERN_KEY = + "webhdfs.user.provider.user.pattern"; + /** Default user name pattern value */ + public static final String USER_PATTERN_DEFAULT = + "^[A-Za-z_][A-Za-z0-9._-]*[$]?$"; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index ce4531c55ef..ad948d319f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -157,6 +157,8 @@ public class WebHdfsFileSystem extends FileSystem ) throws IOException { super.initialize(uri, conf); setConf(conf); + /** set user pattern based on configuration file */ + UserParam.setUserPattern(conf); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); initializeTokenAspect(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java index 36e128feedb..bdc1cfe55d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java @@ -17,8 +17,12 @@ */ package org.apache.hadoop.hdfs.web.resources; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; - +import static org.apache.hadoop.hdfs.web.WebHdfsConfigKeys.USER_PATTERN_KEY; +import static org.apache.hadoop.hdfs.web.WebHdfsConfigKeys.USER_PATTERN_DEFAULT; +import com.google.common.annotations.VisibleForTesting; + import java.text.MessageFormat; import java.util.regex.Pattern; @@ -29,8 +33,29 @@ public class UserParam extends StringParam { /** Default parameter value. */ public static final String DEFAULT = ""; - private static final Domain DOMAIN = new Domain(NAME, - Pattern.compile("^[A-Za-z_][A-Za-z0-9._-]*[$]?$")); + private static String userPattern = null; + private static Domain domain = null; + + static { + setUserPattern(USER_PATTERN_DEFAULT); + } + + @VisibleForTesting + public static String getUserPattern() { + return userPattern; + } + + @VisibleForTesting + public static void setUserPattern(String pattern) { + userPattern = pattern; + Pattern pt = Pattern.compile(userPattern); + domain = new Domain(NAME, pt); + } + + public static void setUserPattern(Configuration conf) { + String pattern = conf.get(USER_PATTERN_KEY, USER_PATTERN_DEFAULT); + setUserPattern(pattern); + } private static String validateLength(String str) { if (str == null) { @@ -50,7 +75,7 @@ public class UserParam extends StringParam { * @param str a string representation of the parameter value. */ public UserParam(final String str) { - super(DOMAIN, str == null || str.equals(DEFAULT)? null : validateLength(str)); + super(domain, str == null || str.equals(DEFAULT)? null : validateLength(str)); } /** @@ -64,4 +89,4 @@ public class UserParam extends StringParam { public String getName() { return NAME; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index e49f2268a6a..5dc86a11162 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.TestDFSClientRetries; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; +import org.apache.hadoop.hdfs.web.WebHdfsConfigKeys; import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Level; import org.junit.Assert; @@ -261,6 +262,34 @@ public class TestWebHDFS { } } + @Test(timeout=300000) + public void testNumericalUserName() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + conf.set(WebHdfsConfigKeys.USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"); + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + try { + cluster.waitActive(); + WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME) + .setPermission(new Path("/"), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + + UserGroupInformation.createUserForTesting("123", new String[]{"my-group"}) + .doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, URISyntaxException { + FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsFileSystem.SCHEME); + Path d = new Path("/my-dir"); + Assert.assertTrue(fs.mkdirs(d)); + return null; + } + }); + } finally { + cluster.shutdown(); + } + } + /** * WebHdfs should be enabled by default after HDFS-5532 * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index 97223ab2fb4..b43fc507ca0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -285,4 +285,19 @@ public class TestParam { Assert.assertEquals(expected, computed.getValue()); } } + + @Test + public void testUserNameOkAfterResettingPattern() { + String oldPattern = UserParam.getUserPattern(); + String newPattern = "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"; + + UserParam.setUserPattern(newPattern); + + UserParam userParam = new UserParam("1x"); + assertNotNull(userParam.getValue()); + userParam = new UserParam("123"); + assertNotNull(userParam.getValue()); + + UserParam.setUserPattern(oldPattern); + } }