From f1840669cfc92c033521bf95989479523e2a649d Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Tue, 11 Feb 2020 14:50:10 -0600 Subject: [PATCH] HDFS-14758. Make lease hard limit configurable and reduce the default. Contributed by hemanthboyina. (cherry picked from commit 9b8a78d97bfd825ce840c6033371c7f10e49a5b8) Conflicts: hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java --- .../main/java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++-- .../hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 +++ .../hadoop/hdfs/client/impl/DfsClientConf.java | 12 ++++++++++++ .../apache/hadoop/hdfs/protocol/HdfsConstants.java | 11 ----------- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../hadoop/hdfs/server/namenode/LeaseManager.java | 7 ++++++- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++ .../java/org/apache/hadoop/hdfs/TestFileAppend4.java | 5 +++-- .../test/java/org/apache/hadoop/hdfs/TestLease.java | 5 +++-- .../org/apache/hadoop/hdfs/TestLeaseRecovery2.java | 6 +++--- .../hdfs/server/namenode/TestFileTruncate.java | 3 ++- .../TestINodeFileUnderConstructionWithSnapshot.java | 3 ++- 12 files changed, 48 insertions(+), 23 deletions(-) mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java old mode 100644 new mode 100755 index 614fc6812f0..187143e7bcf --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -572,10 +572,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } catch (IOException e) { // Abort if the lease has already expired. final long elapsed = Time.monotonicNow() - getLastLeaseRenewal(); - if (elapsed > HdfsConstants.LEASE_HARDLIMIT_PERIOD) { + if (elapsed > dfsClientConf.getleaseHardLimitPeriod()) { LOG.warn("Failed to renew lease for " + clientName + " for " + (elapsed/1000) + " seconds (>= hard-limit =" - + (HdfsConstants.LEASE_HARDLIMIT_PERIOD / 1000) + " seconds.) " + + (dfsClientConf.getleaseHardLimitPeriod() / 1000) + " seconds.) " + "Closing all files being written ...", e); closeAllFilesBeingWritten(true); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java old mode 100644 new mode 100755 index b596c4096a4..6a7e0a8c4c9 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -200,6 +200,9 @@ public interface HdfsClientConfigKeys { String DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS = "dfs.provided.aliasmap.inmemory.dnrpc-address"; + String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec"; + long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60; + /** * These are deprecated config keys to client code. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 49709f587e3..cb49295cdd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -146,6 +146,7 @@ public class DfsClientConf { private final int stripedReadThreadpoolSize; private final boolean dataTransferTcpNoDelay; + private final long leaseHardLimitPeriod; public DfsClientConf(Configuration conf) { // The hdfsTimeout is currently the same as the ipc timeout @@ -273,6 +274,10 @@ public class DfsClientConf { HdfsClientConfigKeys.StripedRead.THREADPOOL_SIZE_KEY + " must be greater than 0."); replicaAccessorBuilderClasses = loadReplicaAccessorBuilderClasses(conf); + + leaseHardLimitPeriod = + conf.getLong(HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000; } @SuppressWarnings("unchecked") @@ -595,6 +600,13 @@ public class DfsClientConf { return stripedReadThreadpoolSize; } + /** + * @return the leaseHardLimitPeriod + */ + public long getleaseHardLimitPeriod() { + return leaseHardLimitPeriod; + } + /** * @return the replicaAccessorBuilderClasses */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java old mode 100644 new mode 100755 index 6de186a1bc7..cb09a9b4d43 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -111,17 +111,6 @@ public final class HdfsConstants { * lease, another client can preempt the lease. */ public static final long LEASE_SOFTLIMIT_PERIOD = 60 * 1000; - /** - * For a HDFS client to write to a file, a lease is granted; During the lease - * period, no other client can write to the file. The writing client can - * periodically renew the lease. When the file is closed, the lease is - * revoked. The lease duration is bound by a - * {@link HdfsConstants#LEASE_SOFTLIMIT_PERIOD soft limit} and this hard - * limit. If after the hard limit expires and the client has failed to renew - * the lease, HDFS assumes that the client has quit and will automatically - * close the file on behalf of the writer, and recover the lease. - */ - public static final long LEASE_HARDLIMIT_PERIOD = 60 * LEASE_SOFTLIMIT_PERIOD; // SafeMode actions public enum SafeModeAction { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java old mode 100644 new mode 100755 index dcfe401a6c9..e3dac0a81b8 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1676,4 +1676,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT = HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT; + public static final String DFS_LEASE_HARDLIMIT_KEY = + HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_KEY; + public static final long DFS_LEASE_HARDLIMIT_DEFAULT = + HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index 31fb2bb2843..63bff063d3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -40,7 +40,9 @@ import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; @@ -82,7 +84,7 @@ public class LeaseManager { .getName()); private final FSNamesystem fsnamesystem; private long softLimit = HdfsConstants.LEASE_SOFTLIMIT_PERIOD; - private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD; + private long hardLimit; static final int INODE_FILTER_WORKER_COUNT_MAX = 4; static final int INODE_FILTER_WORKER_TASK_MIN = 512; private long lastHolderUpdateTime; @@ -110,7 +112,10 @@ public class LeaseManager { private volatile boolean shouldRunMonitor; LeaseManager(FSNamesystem fsnamesystem) { + Configuration conf = new Configuration(); this.fsnamesystem = fsnamesystem; + this.hardLimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000; updateInternalLeaseHolder(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 2fe095d38e6..53e3afbb73b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5387,4 +5387,12 @@ octal or symbolic. + + + dfs.namenode.lease-hard-limit-sec + 1200 + + Determines the namenode automatic lease recovery interval in seconds. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java old mode 100644 new mode 100755 index ff0fd1964b7..9bf8acb1b37 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java @@ -41,7 +41,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; @@ -108,7 +107,9 @@ public class TestFileAppend4 { // set the soft limit to be 1 second so that the // namenode triggers lease recovery upon append request - cluster.setLeasePeriod(1000, HdfsConstants.LEASE_HARDLIMIT_PERIOD); + cluster.setLeasePeriod(1, + conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT)); // Trying recovery int tries = 60; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java old mode 100644 new mode 100755 index 381fe7148b6..e23308d2f15 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java @@ -114,9 +114,10 @@ public class TestLease { Assert.fail("Write failed."); } + long hardlimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000; // make it look like the hard limit has been exceeded. - dfs.lastLeaseRenewal = Time.monotonicNow() - - HdfsConstants.LEASE_HARDLIMIT_PERIOD - 1000; + dfs.lastLeaseRenewal = Time.monotonicNow() - hardlimit - 1000; dfs.renewLease(); // this should not work. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java old mode 100644 new mode 100755 index d65fed2b934..b283c16d58e --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java @@ -407,10 +407,10 @@ public class TestLeaseRecovery2 { Map u2g_map = new HashMap(1); u2g_map.put(fakeUsername, new String[] {fakeGroup}); DFSTestUtil.updateConfWithFakeGroupMapping(conf, u2g_map); - + long hardlimit = conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000; // Reset default lease periods - cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD, - HdfsConstants.LEASE_HARDLIMIT_PERIOD); + cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD, hardlimit); //create a file // create a random file name String filestr = "/foo" + AppendTestUtil.nextInt(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java old mode 100644 new mode 100755 index f1083875e13..a1197fb9f9d --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java @@ -659,7 +659,8 @@ public class TestFileTruncate { NameNodeAdapter.getLeaseManager(cluster.getNamesystem()) .setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD, - HdfsConstants.LEASE_HARDLIMIT_PERIOD); + conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000); checkFullFile(p, newLength, contents); fs.delete(p, false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java old mode 100644 new mode 100755 index 77c9ecc3a0c..4f32af9b9a4 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestINodeFileUnderConstructionWithSnapshot.java @@ -304,7 +304,8 @@ public class TestINodeFileUnderConstructionWithSnapshot { } } finally { NameNodeAdapter.setLeasePeriod(fsn, HdfsConstants.LEASE_SOFTLIMIT_PERIOD, - HdfsConstants.LEASE_HARDLIMIT_PERIOD); + conf.getLong(DFSConfigKeys.DFS_LEASE_HARDLIMIT_KEY, + DFSConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT) * 1000); } } }