From 89bd14913aa5f00506a610ca325db495372c4c87 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Mon, 21 Jan 2013 19:29:06 +0000 Subject: [PATCH] HDFS-4416. Rename dfs.datanode.domain.socket.path to dfs.domain.socket.path. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-347@1436568 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/conf/Configuration.java | 17 ++++++++++++++++- .../apache/hadoop/net/unix/DomainSocket.java | 4 ++-- .../hadoop/net/unix/TestDomainSocket.java | 2 +- .../hadoop-hdfs/CHANGES.HDFS-347.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 3 ++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++- .../apache/hadoop/hdfs/DomainSocketFactory.java | 4 ++-- .../hadoop/hdfs/server/datanode/DataNode.java | 7 ++++--- .../src/main/resources/hdfs-default.xml | 11 +++++++++++ .../hdfs/TestParallelShortCircuitRead.java | 2 +- .../TestParallelShortCircuitReadNoChecksum.java | 2 +- .../hadoop/hdfs/TestParallelUnixDomainRead.java | 2 +- .../hadoop/hdfs/TestShortCircuitLocalRead.java | 4 ++-- 13 files changed, 48 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 73faf968628..c44a18f9737 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -743,6 +743,21 @@ public class Configuration implements Iterable>, return value.trim(); } } + + /** + * Get the value of the name property as a trimmed String, + * defaultVal if no such property exists. + * See @{Configuration#getTrimmed} for more details. + * + * @param name the property name. + * @param defaultVal the property default value. + * @return the value of the name or defaultVal + * if it is not set. + */ + public String getTrimmed(String name, String defaultValue) { + String ret = getTrimmed(name); + return ret == null ? defaultValue : ret; + } /** * Get the value of the name property, without doing @@ -877,7 +892,7 @@ public class Configuration implements Iterable>, } return result; } - + /** * Get the value of the name property as an int. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java index 034ac3b62da..aa4b952e8d1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java @@ -119,7 +119,7 @@ public class DomainSocket implements Closeable { /** * Given a path and a port, compute the effective path by replacing - * occurrences of __PORT__ with the port. This is mainly to make it + * occurrences of _PORT with the port. This is mainly to make it * possible to run multiple DataNodes locally for testing purposes. * * @param path The source path @@ -128,7 +128,7 @@ public class DomainSocket implements Closeable { * @return The effective path */ public static String getEffectivePath(String path, int port) { - return path.replace("__PORT__", String.valueOf(port)); + return path.replace("_PORT", String.valueOf(port)); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java index a997f52056a..a490fefcac7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java @@ -91,7 +91,7 @@ public class TestDomainSocket { @Test(timeout=180000) public void testSocketPathSetGet() throws IOException { Assert.assertEquals("/var/run/hdfs/sock.100", - DomainSocket.getEffectivePath("/var/run/hdfs/sock.__PORT__", 100)); + DomainSocket.getEffectivePath("/var/run/hdfs/sock._PORT", 100)); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-347.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-347.txt index 5ae5750fe2b..d1096376bcc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-347.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-347.txt @@ -27,3 +27,6 @@ HDFS-4402. Some small DomainSocket fixes: avoid findbugs warning, change log lev (Colin Patrick McCabe via todd) HDFS-4418. increase default FileInputStreamCache size (todd) + +HDFS-4416. Rename dfs.datanode.domain.socket.path to dfs.domain.socket.path +(Colin Patrick McCabe via todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index f68dae7b909..d8e08620072 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -291,7 +291,8 @@ public class DFSClient implements java.io.Closeable { getFileBlockStorageLocationsTimeout = conf.getInt( DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT, DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_DEFAULT); - domainSocketPath = conf.get(DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY); + domainSocketPath = conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT); skipShortCircuitChecksums = conf.getBoolean( DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 034a0152f19..c8317c01d7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -399,7 +399,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY = "dfs.web.authentication.kerberos.keytab"; public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user"; - public static final String DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY = "dfs.datanode.domain.socket.path"; + public static final String DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path"; + public static final String DFS_DOMAIN_SOCKET_PATH_DEFAULT = ""; // HA related configuration public static final String DFS_HA_NAMENODES_KEY_PREFIX = "dfs.ha.namenodes"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DomainSocketFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DomainSocketFactory.java index db9afc1c40c..103c946e5e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DomainSocketFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DomainSocketFactory.java @@ -58,9 +58,9 @@ class DomainSocketFactory { feature = "UNIX domain socket data traffic"; } if (feature != null) { - if (conf.domainSocketPath == null) { + if (conf.domainSocketPath.isEmpty()) { LOG.warn(feature + " is disabled because you have not set " + - DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY); + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY); } else if (DomainSocket.getLoadingFailureReason() != null) { LOG.warn(feature + " is disabled because " + DomainSocket.getLoadingFailureReason()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index d3e3b5b2d63..99e508b53d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -572,13 +572,14 @@ public class DataNode extends Configured static DomainPeerServer getDomainPeerServer(Configuration conf, int port) throws IOException { String domainSocketPath = - conf.get(DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY); - if (domainSocketPath == null) { + conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT); + if (domainSocketPath.isEmpty()) { if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) { LOG.warn("Although short-circuit local reads are configured, " + "they are disabled because you didn't configure " + - DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY); + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY); } return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 34cd8465fd7..5ec12106c47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1197,4 +1197,15 @@ + + dfs.domain.socket.path + + + Optional. This is a path to a UNIX domain socket that will be used for + communication between the DataNode and local HDFS clients. + If the string "_PORT" is present in this path, it will be replaced by the + TCP port of the DataNode. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitRead.java index f5b0ef6672a..0e21cdcd8f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitRead.java @@ -35,7 +35,7 @@ public class TestParallelShortCircuitRead extends TestParallelReadUtil { if (DomainSocket.getLoadingFailureReason() != null) return; sockDir = new TemporarySocketDirectory(); HdfsConfiguration conf = new HdfsConfiguration(); - conf.set(DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY, + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestParallelLocalRead.%d.sock").getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); conf.setBoolean(DFSConfigKeys. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitReadNoChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitReadNoChecksum.java index a27016ab702..a704dd30bb2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitReadNoChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelShortCircuitReadNoChecksum.java @@ -35,7 +35,7 @@ public class TestParallelShortCircuitReadNoChecksum extends TestParallelReadUtil if (DomainSocket.getLoadingFailureReason() != null) return; sockDir = new TemporarySocketDirectory(); HdfsConfiguration conf = new HdfsConfiguration(); - conf.set(DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY, + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestParallelLocalRead.%d.sock").getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); conf.setBoolean(DFSConfigKeys. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelUnixDomainRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelUnixDomainRead.java index 5113dbf8eb2..84688cb4081 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelUnixDomainRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelUnixDomainRead.java @@ -35,7 +35,7 @@ public class TestParallelUnixDomainRead extends TestParallelReadUtil { if (DomainSocket.getLoadingFailureReason() != null) return; sockDir = new TemporarySocketDirectory(); HdfsConfiguration conf = new HdfsConfiguration(); - conf.set(DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY, + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestParallelLocalRead.%d.sock").getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false); DomainSocket.disableBindPathValidation(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java index b0a1c4a91ea..e71fc8ce183 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java @@ -193,9 +193,9 @@ public class TestShortCircuitLocalRead { conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, ignoreChecksum); - conf.set(DFSConfigKeys.DFS_DATANODE_DOMAIN_SOCKET_PATH_KEY, + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), - "TestShortCircuitLocalRead.__PORT__.sock").getAbsolutePath()); + "TestShortCircuitLocalRead._PORT.sock").getAbsolutePath()); if (simulatedStorage) { SimulatedFSDataset.setFactory(conf); }