From 46d62e701625e01dc438c30afaf3bf54e860b0b6 Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Fri, 15 May 2015 19:12:40 -0700 Subject: [PATCH] HDFS-8403. Eliminate retries in TestFileCreation#testOverwriteOpenForWrite. Contributed by Arpit Agarwal. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSClient.java | 5 +++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 ++++++ .../apache/hadoop/hdfs/NameNodeProxies.java | 28 ++++++++++++++++++- .../apache/hadoop/hdfs/TestFileCreation.java | 10 +++++-- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 81dc05773c4..3e3bab34b79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -445,6 +445,9 @@ Release 2.8.0 - UNRELEASED HDFS-8371. Fix test failure in TestHdfsConfigFields for spanreceiver properties. (Ray Chiang via aajisaka) + HDFS-8403. Eliminate retries in TestFileCreation + #testOverwriteOpenForWrite. (Arpit Agarwal via wheat9) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 60c53b02d22..924d02e7cdf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -343,10 +343,13 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, this.namenode = rpcNamenode; dtService = null; } else { + boolean noRetries = conf.getBoolean( + DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES, + DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT); Preconditions.checkArgument(nameNodeUri != null, "null URI"); proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri, - ClientProtocol.class, nnFallbackToSimpleAuth); + ClientProtocol.class, nnFallbackToSimpleAuth, !noRetries); this.dtService = proxyInfo.getDelegationTokenService(); this.namenode = proxyInfo.getProxy(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 3669685fc9c..b3e511d9501 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs; import java.util.concurrent.TimeUnit; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; @@ -994,6 +995,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys { // For testing NN retry cache, we can set this property with positive value. public static final String DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY = "dfs.client.test.drop.namenode.response.number"; public static final int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0; + + // Create a NN proxy without retries for testing. + @VisibleForTesting + public static final String DFS_CLIENT_TEST_NO_PROXY_RETRIES = + "dfs.client.test.no.proxy.retries"; + @VisibleForTesting + public static final boolean DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT = false; + public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY = "dfs.client.slow.io.warning.threshold.ms"; public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index 09ea11e53bc..a7dda7bf330 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -161,6 +161,31 @@ public class NameNodeProxies { public static ProxyAndInfo createProxy(Configuration conf, URI nameNodeUri, Class xface, AtomicBoolean fallbackToSimpleAuth) throws IOException { + return createProxy(conf, nameNodeUri, xface, fallbackToSimpleAuth, true); + } + + /** + * Creates the namenode proxy with the passed protocol. This will handle + * creation of either HA- or non-HA-enabled proxy objects, depending upon + * if the provided URI is a configured logical URI. + * + * @param conf the configuration containing the required IPC + * properties, client failover configurations, etc. + * @param nameNodeUri the URI pointing either to a specific NameNode + * or to a logical nameservice. + * @param xface the IPC interface which should be created + * @param fallbackToSimpleAuth set to true or false during calls to + * indicate if a secure client falls back to simple auth + * @param withRetries certain interfaces have a non-standard retry policy + * @return an object containing both the proxy and the associated + * delegation token service it corresponds to + * @throws IOException if there is an error creating the proxy + **/ + @SuppressWarnings("unchecked") + public static ProxyAndInfo createProxy(Configuration conf, + URI nameNodeUri, Class xface, AtomicBoolean fallbackToSimpleAuth, + boolean withRetries) + throws IOException { AbstractNNFailoverProxyProvider failoverProxyProvider = createFailoverProxyProvider(conf, nameNodeUri, xface, true, fallbackToSimpleAuth); @@ -168,7 +193,8 @@ public class NameNodeProxies { if (failoverProxyProvider == null) { // Non-HA case return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface, - UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth); + UserGroupInformation.getCurrentUser(), withRetries, + fallbackToSimpleAuth); } else { // HA case DfsClientConf config = new DfsClientConf(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index f732ace684e..49770152f67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -65,6 +65,7 @@ import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -73,6 +74,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; @@ -377,6 +379,11 @@ public class TestFileCreation { Configuration conf = new HdfsConfiguration(); SimulatedFSDataset.setFactory(conf); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false); + + // Force NameNodeProxies' createNNProxyWithClientProtocol to give + // up file creation after one failure. + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES, true); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); @@ -405,9 +412,8 @@ public class TestFileCreation { } catch (IOException abce) { GenericTestUtils.assertExceptionContains("Failed to CREATE_FILE", abce); } - // NameNodeProxies' createNNProxyWithClientProtocol has 5 retries. assertCounter("AlreadyBeingCreatedExceptionNumOps", - 6L, getMetrics(metricsName)); + 1L, getMetrics(metricsName)); FSDataOutputStream stm2 = fs2.create(p, true); stm2.write(2); stm2.close();