Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-7786. svn merge -c 1196129 from trunk 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1196136 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2011-11-01 17:08:02 +00:00
parent 8e799e112a
commit 256b7b1ca9
5 changed files with 10 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -979,6 +979,8 @@ Release 0.22.0 - Unreleased
HADOOP-7772. javadoc the topology classes (stevel)
HADOOP-7786. Remove HDFS-specific config keys defined in FsConfig. (eli)
OPTIMIZATIONS
HADOOP-6884. Add LOG.isDebugEnabled() guard for each LOG.debug(..).

2
hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/cluster_setup.xml
View File

@ -627,7 +627,7 @@
</tr>
<tr>
<td>conf/hdfs-site.xml</td>
<td>dfs.block.size</td>
<td>dfs.blocksize</td>
<td>134217728</td>
<td>HDFS blocksize of 128MB for large file-systems.</td>
</tr>

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
View File

@ -44,6 +44,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RpcClientException;
import org.apache.hadoop.ipc.RpcServerException;
@ -443,7 +445,9 @@ public final class FileContext {
*/
public static FileContext getFileContext(final Configuration aConf)
throws UnsupportedFileSystemException {
return getFileContext(URI.create(FsConfig.getDefaultFsURI(aConf)), aConf);
return getFileContext(
URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)),
aConf);
}
/**

114
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConfig.java
View File

@ -1,114 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;
import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_HOME_DIR_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_HOME_DIR_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
/**
* This class is thin layer to manage the FS related keys in
* a configuration object.
* It provides convenience static method to set and get the keys from a
* a configuration.
*
*/
final class FsConfig {
private FsConfig() {}
// Configuration keys and default values in the config file
// TBD note we should deprecate the keys constants elsewhere
// The Keys
static final String FS_REPLICATION_FACTOR_KEY = "dfs.replication";
static final String FS_BLOCK_SIZE_KEY = "dfs.block.size";
// The default values
// Default values of SERVER_DEFAULT(-1) implies use the ones from
// the target file system where files are created.
static final short FS_DEFAULT_REPLICATION_FACTOR = 3;
static final long FS_DEFAULT_BLOCK_SIZE = 32 * 1024 * 1024;
public static String getDefaultFsURI(final Configuration conf) {
return conf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT);
}
public static String getHomeDir(final Configuration conf) {
return conf.get(FS_HOME_DIR_KEY, FS_HOME_DIR_DEFAULT);
}
public static short getDefaultReplicationFactor(final Configuration conf) {
return (short)
conf.getInt(FS_REPLICATION_FACTOR_KEY, FS_DEFAULT_REPLICATION_FACTOR);
}
public static long getDefaultBlockSize(final Configuration conf) {
return conf.getLong(FS_BLOCK_SIZE_KEY, FS_DEFAULT_BLOCK_SIZE);
}
public static int getDefaultIOBuffersize(final Configuration conf) {
return conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT);
}
public static Class<?> getImplClass(URI uri, Configuration conf) {
String scheme = uri.getScheme();
if (scheme == null) {
throw new IllegalArgumentException("No scheme");
}
return conf.getClass("fs." + uri.getScheme() + ".impl", null);
}
/**
* The Setters: see the note on the javdoc for the class above.
*/
public static void setDefaultFS(final Configuration conf, String uri) {
conf.set(FS_DEFAULT_NAME_KEY, uri);
}
public static void setHomeDir(final Configuration conf, String path) {
conf.set(FS_HOME_DIR_KEY, path);
}
public static void setDefaultReplicationFactor(final Configuration conf,
short rf) {
conf.setInt(FS_REPLICATION_FACTOR_KEY, rf);
}
public static void setDefaultBlockSize(final Configuration conf, long bs) {
conf.setLong(FS_BLOCK_SIZE_KEY, bs);
}
public static void setDefaultIOBuffersize(final Configuration conf, int bs) {
conf.setInt(IO_FILE_BUFFER_SIZE_KEY, bs);
}
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
View File

@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.ftpserver.command.impl.STAT;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import org.junit.BeforeClass;
import org.junit.Test;
@ -301,7 +302,7 @@ public class TestFsShellReturnCode {
// arguments is valid - fsshell should work
FsShell shell = new FsShell();
Configuration conf = new Configuration();
FsConfig.setDefaultFS(conf, "hhhh://doesnotexist/");
conf.set(FS_DEFAULT_NAME_KEY, "hhhh://doesnotexist/");
shell.setConf(conf);
String [] args = new String[2];
args[0] = "-ls";