Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

svn merge -c -1354790 for reverting HDFS-3576 since it requires more changes. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1354792 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2012-06-28 01:25:32 +00:00
parent 44389399d4
commit f105784d6a
12 changed files with 29 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -250,9 +250,6 @@ Branch-2 ( Unreleased changes )
HDFS-3572. Cleanup code which inits SPNEGO in HttpServer (todd)
HDFS-3576. Move the constant NameNode.DEFAULT_PORT to
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT. (Brandon Li via szetszwo)
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
View File

@ -33,7 +33,6 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.CorruptFileBlockIterator;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
@ -43,6 +42,7 @@
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@ -71,8 +71,7 @@ public class Hdfs extends AbstractFileSystem {
* @throws IOException
*/
Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException {
super(theUri, HdfsConstants.HDFS_URI_SCHEME, true,
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
super(theUri, HdfsConstants.HDFS_URI_SCHEME, true, NameNode.DEFAULT_PORT);
if (!theUri.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) {
throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs");
@ -87,7 +86,7 @@ public class Hdfs extends AbstractFileSystem {
@Override
public int getUriDefaultPort() {
return DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
return NameNode.DEFAULT_PORT;
}
@Override

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -87,7 +87,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address";
public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT;
public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address";
public static final int DFS_NAMENODE_RPC_PORT_DEFAULT = 8020;
public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address";
public static final String DFS_NAMENODE_MAX_OBJECTS_KEY = "dfs.namenode.max.objects";
public static final long DFS_NAMENODE_MAX_OBJECTS_DEFAULT = 0;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -734,7 +734,7 @@ public void setTimes(Path p, long mtime, long atime
@Override
protected int getDefaultPort() {
return DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
return NameNode.DEFAULT_PORT;
}
@Override

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
View File

@ -402,7 +402,7 @@ private static <T> Class<FailoverProxyProvider<T>> getFailoverProxyProviderClass
// If we found a proxy provider, then this URI should be a logical NN.
// Given that, it shouldn't have a non-default port number.
int port = nameNodeUri.getPort();
if (port > 0 && port != DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) {
if (port > 0 && port != NameNode.DEFAULT_PORT) {
throw new IOException("Port " + port + " specified in URI "
+ nameNodeUri + " but host '" + host
+ "' is a logical (HA) namenode"

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java
View File

@ -22,7 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@ -57,7 +57,7 @@ public Token<DelegationTokenIdentifier> selectToken(
Text serviceName = SecurityUtil.buildTokenService(nnUri);
final String nnServiceName = conf.get(SERVICE_NAME_KEY + serviceName);
int nnRpcPort = DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
int nnRpcPort = NameNode.DEFAULT_PORT;
if (nnServiceName != null) {
nnRpcPort = NetUtils.createSocketAddr(nnServiceName, nnRpcPort).getPort();
}

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
View File

@ -214,6 +214,7 @@ public long getProtocolVersion(String protocol,
}
}
public static final int DEFAULT_PORT = 8020;
public static final Log LOG = LogFactory.getLog(NameNode.class.getName());
public static final Log stateChangeLog = LogFactory.getLog("org.apache.hadoop.hdfs.StateChange");
public static final HAState ACTIVE_STATE = new ActiveState();
@ -269,7 +270,7 @@ public static NameNodeMetrics getNameNodeMetrics() {
}
public static InetSocketAddress getAddress(String address) {
return NetUtils.createSocketAddr(address, DFS_NAMENODE_RPC_PORT_DEFAULT);
return NetUtils.createSocketAddr(address, DEFAULT_PORT);
}
/**
@ -328,8 +329,7 @@ public static InetSocketAddress getAddress(URI filesystemURI) {
public static URI getUri(InetSocketAddress namenode) {
int port = namenode.getPort();
String portString = (port == DFS_NAMENODE_RPC_PORT_DEFAULT) ?
"" : (":"+port);
String portString = port == DEFAULT_PORT ? "" : (":"+port);
return URI.create(HdfsConstants.HDFS_URI_SCHEME + "://"
+ namenode.getHostName()+portString);
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
View File

@ -77,7 +77,7 @@ public NNHAServiceTarget(Configuration conf,
"Unable to determine service address for namenode '" + nnId + "'");
}
this.addr = NetUtils.createSocketAddr(serviceAddr,
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
NameNode.DEFAULT_PORT);
this.autoFailoverEnabled = targetConf.getBoolean(
DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY,

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.test.GenericTestUtils;
@ -81,9 +82,9 @@ public void testDfsClientFailover() throws IOException, URISyntaxException {
// Check that it functions even if the URL becomes canonicalized
// to include a port number.
Path withPort = new Path("hdfs://" + HATestUtil.getLogicalHostname(cluster)
+ ":" + DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + "/"
+ TEST_FILE.toUri().getPath());
Path withPort = new Path("hdfs://" +
HATestUtil.getLogicalHostname(cluster) + ":" +
NameNode.DEFAULT_PORT + "/" + TEST_FILE.toUri().getPath());
FileSystem fs2 = withPort.getFileSystem(fs.getConf());
assertTrue(fs2.exists(withPort));

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java
View File

@ -31,9 +31,9 @@ public class TestDefaultNameNodePort extends TestCase {
public void testGetAddressFromString() throws Exception {
assertEquals(NameNode.getAddress("foo").getPort(),
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
NameNode.DEFAULT_PORT);
assertEquals(NameNode.getAddress("hdfs://foo/").getPort(),
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
NameNode.DEFAULT_PORT);
assertEquals(NameNode.getAddress("hdfs://foo:555").getPort(),
555);
assertEquals(NameNode.getAddress("foo:555").getPort(),
@ -43,20 +43,18 @@ public void testGetAddressFromString() throws Exception {
public void testGetAddressFromConf() throws Exception {
Configuration conf = new HdfsConfiguration();
FileSystem.setDefaultUri(conf, "hdfs://foo/");
assertEquals(NameNode.getAddress(conf).getPort(),
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT);
FileSystem.setDefaultUri(conf, "hdfs://foo:555/");
assertEquals(NameNode.getAddress(conf).getPort(), 555);
FileSystem.setDefaultUri(conf, "foo");
assertEquals(NameNode.getAddress(conf).getPort(),
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
assertEquals(NameNode.getAddress(conf).getPort(), NameNode.DEFAULT_PORT);
}
public void testGetUri() {
assertEquals(NameNode.getUri(new InetSocketAddress("foo", 555)),
URI.create("hdfs://foo:555"));
assertEquals(NameNode.getUri(new InetSocketAddress("foo",
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT)),
NameNode.DEFAULT_PORT)),
URI.create("hdfs://foo"));
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
View File

@ -25,7 +25,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@ -33,6 +32,7 @@
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.junit.Test;
@ -67,7 +67,7 @@ public void testBalancerWithHANameNodes() throws Exception {
assertEquals(capacities.length, racks.length);
int numOfDatanodes = capacities.length;
NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
nn1Conf.setIpcPort(DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
nn1Conf.setIpcPort(NameNode.DEFAULT_PORT);
Configuration copiedConf = new Configuration(conf);
cluster = new MiniDFSCluster.Builder(copiedConf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())

15
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java
View File

@ -39,7 +39,6 @@
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.fs.shell.CommandFormat;
@ -510,10 +509,10 @@ public FileSystem run() throws IOException {
{
try {
runTestCache(DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
runTestCache(NameNode.DEFAULT_PORT);
} catch(java.net.BindException be) {
LOG.warn("Cannot test NameNode's default RPC port (="
+ DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + ")", be);
LOG.warn("Cannot test NameNode.DEFAULT_PORT (="
+ NameNode.DEFAULT_PORT + ")", be);
}
runTestCache(0);
@ -536,11 +535,11 @@ static void runTestCache(int port) throws Exception {
}
}
if (port == DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) {
if (port == NameNode.DEFAULT_PORT) {
//test explicit default port
URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(),
DFSConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT, uri.getPath(),
uri.getQuery(), uri.getFragment());
URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(),
uri.getHost(), NameNode.DEFAULT_PORT, uri.getPath(),
uri.getQuery(), uri.getFragment());
LOG.info("uri2=" + uri2);
FileSystem fs = FileSystem.get(uri2, conf);
checkPath(cluster, fs);