From ae20516ebc0709bbe674c4846461ef6d81cc98b4 Mon Sep 17 00:00:00 2001 From: zhihaixu2012 Date: Tue, 27 Jul 2021 19:18:44 -0700 Subject: [PATCH] HDFS-16111. Add a configuration to RoundRobinVolumeChoosingPolicy to avoid failed volumes at datanodes. (#3175) Change-Id: Iead25812d4073e3980893e3e76f7d2b03b57442a Co-authored-by: Zhihai Xu --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +++ .../RoundRobinVolumeChoosingPolicy.java | 29 ++++++++++- .../src/main/resources/hdfs-default.xml | 11 +++++ .../apache/hadoop/hdfs/MiniDFSCluster.java | 3 ++ .../TestRoundRobinVolumeChoosingPolicy.java | 48 ++++++++++++++++++- 5 files changed, 94 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 66e3c8b71ed..808ecfbe0c4 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1112,6 +1112,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT = 1024L * 1024L * 1024L * 10L; // 10 GB public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction"; public static final float DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.75f; + public static final String + DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY = + "dfs.datanode.round-robin-volume-choosing-policy.additional-available-space"; + public static final long + DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_DEFAULT = + 1024L * 1024L * 1024L; // 1 GB public static final String DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY = HdfsClientConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; public static final String DFS_DATANODE_STARTUP_KEY = "dfs.datanode.startup"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java index 0f23e470969..fe010b35a4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java @@ -17,11 +17,16 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY; + import java.io.IOException; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; @@ -30,7 +35,7 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; * Use fine-grained locks to synchronize volume choosing. */ public class RoundRobinVolumeChoosingPolicy - implements VolumeChoosingPolicy { + implements VolumeChoosingPolicy, Configurable { public static final Logger LOG = LoggerFactory.getLogger(RoundRobinVolumeChoosingPolicy.class); @@ -41,6 +46,9 @@ public class RoundRobinVolumeChoosingPolicy // syncLocks stores the locks for each storage type. private Object[] syncLocks; + // The required additional available space when choosing a volume. + private long additionalAvailableSpace; + public RoundRobinVolumeChoosingPolicy() { int numStorageTypes = StorageType.values().length; curVolumes = new int[numStorageTypes]; @@ -50,6 +58,23 @@ public class RoundRobinVolumeChoosingPolicy } } + @Override + public void setConf(Configuration conf) { + additionalAvailableSpace = conf.getLong( + DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY, + DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_DEFAULT); + + LOG.info("Round robin volume choosing policy initialized: " + + DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY + + " = " + additionalAvailableSpace); + } + + @Override + public Configuration getConf() { + // Nothing to do. Only added to fulfill the Configurable contract. + return null; + } + @Override public V chooseVolume(final List volumes, long blockSize, String storageId) throws IOException { @@ -83,7 +108,7 @@ public class RoundRobinVolumeChoosingPolicy final V volume = volumes.get(curVolume); curVolume = (curVolume + 1) % volumes.size(); long availableVolumeSize = volume.getAvailable(); - if (availableVolumeSize > blockSize) { + if (availableVolumeSize > blockSize + additionalAvailableSpace) { curVolumes[curVolumeIndex] = curVolume; return volume; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 58e709f77b4..5818ae18128 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2657,6 +2657,17 @@ + + dfs.datanode.round-robin-volume-choosing-policy.additional-available-space + 1073741824 + + Only used when the dfs.datanode.fsdataset.volume.choosing.policy is set to + org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy. + This setting controls how much additional available space (unit is byte) is needed + when choosing a volume. + + + dfs.namenode.edits.noeditlogchannelflush false diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index f1b63acd964..4de55d64063 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -533,6 +533,9 @@ public class MiniDFSCluster implements AutoCloseable { DEFAULT_DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD); this.storagesPerDatanode = FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs(); + conf.setLong(DFSConfigKeys + .DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY, + 0); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java index 44e2a30e552..fc99d3c7e54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java @@ -17,10 +17,13 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY; + import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; import org.apache.hadoop.util.ReflectionUtils; @@ -70,7 +73,50 @@ public class TestRoundRobinVolumeChoosingPolicy { // Passed. } } - + + // Test the Round-Robin block-volume choosing algorithm with + // additional available space configured. + @Test + @SuppressWarnings("unchecked") + public void testRRWithAdditionalAvailableSpace() throws Exception { + Configuration conf = new Configuration(); + // Set the additional available space needed + conf.setLong( + DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY, + 100); + final RoundRobinVolumeChoosingPolicy policy = + ReflectionUtils.newInstance(RoundRobinVolumeChoosingPolicy.class, conf); + testRRWithAdditionalAvailableSpace(policy); + } + + public static void testRRWithAdditionalAvailableSpace( + VolumeChoosingPolicy policy) throws Exception { + final List volumes = new ArrayList(); + + // First volume, with 100 bytes of space. + volumes.add(Mockito.mock(FsVolumeSpi.class)); + Mockito.when(volumes.get(0).getAvailable()).thenReturn(100L); + + // Second volume, with 200 bytes of space. + volumes.add(Mockito.mock(FsVolumeSpi.class)); + Mockito.when(volumes.get(1).getAvailable()).thenReturn(200L); + + // The first volume has only 100L space, so the policy should choose + // the second one with additional available space configured as 100L. + Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0, + null)); + Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0, + null)); + + // Fail if no volume can be chosen? + try { + policy.chooseVolume(volumes, 100, null); + Assert.fail(); + } catch (IOException e) { + // Passed. + } + } + // ChooseVolume should throw DiskOutOfSpaceException // with volume and block sizes in exception message. @Test