From 26705bbc60543719739da40ebf89e01cc54e24cb Mon Sep 17 00:00:00 2001 From: singer-bin Date: Mon, 11 Apr 2022 13:59:34 +0800 Subject: [PATCH] HDFS-16457. Make fs.getspaceused.classname reconfigurable (apache#4069) (#4156) --- .../hadoop/hdfs/server/datanode/DataNode.java | 35 ++++++++++++++++--- .../fsdataset/impl/BlockPoolSlice.java | 7 +++- .../datanode/TestDataNodeReconfiguration.java | 31 ++++++++++++++++ .../hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 27a6e19f733..ccdef5302a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY; @@ -84,6 +85,9 @@ import static org.apache.hadoop.util.Preconditions.checkNotNull; import static org.apache.hadoop.util.Time.now; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.DU; +import org.apache.hadoop.fs.GetSpaceUsed; +import org.apache.hadoop.fs.WindowsGetSpaceUsed; import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ReconfigurationProtocolService; import java.io.BufferedOutputStream; @@ -243,6 +247,7 @@ import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ServicePlugin; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; @@ -352,7 +357,8 @@ public class DataNode extends ReconfigurableBase DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY, DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY, FS_DU_INTERVAL_KEY, - FS_GETSPACEUSED_JITTER_KEY)); + FS_GETSPACEUSED_JITTER_KEY, + FS_GETSPACEUSED_CLASSNAME)); public static final Log METRICS_LOG = LogFactory.getLog("DataNodeMetricsLog"); @@ -678,6 +684,7 @@ public class DataNode extends ReconfigurableBase return reconfSlowDiskParameters(property, newVal); case FS_DU_INTERVAL_KEY: case FS_GETSPACEUSED_JITTER_KEY: + case FS_GETSPACEUSED_CLASSNAME: return reconfDfsUsageParameters(property, newVal); default: break; @@ -874,7 +881,7 @@ public class DataNode extends ReconfigurableBase for (FsVolumeImpl fsVolume : volumeList) { Map blockPoolSlices = fsVolume.getBlockPoolSlices(); for (BlockPoolSlice value : blockPoolSlices.values()) { - value.updateDfsUsageConfig(interval, null); + value.updateDfsUsageConfig(interval, null, null); } } } else if (property.equals(FS_GETSPACEUSED_JITTER_KEY)) { @@ -886,13 +893,33 @@ public class DataNode extends ReconfigurableBase for (FsVolumeImpl fsVolume : volumeList) { Map blockPoolSlices = fsVolume.getBlockPoolSlices(); for (BlockPoolSlice value : blockPoolSlices.values()) { - value.updateDfsUsageConfig(null, jitter); + value.updateDfsUsageConfig(null, jitter, null); + } + } + } else if (property.equals(FS_GETSPACEUSED_CLASSNAME)) { + Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); + Class klass; + if (newVal == null) { + if (Shell.WINDOWS) { + klass = DU.class; + } else { + klass = WindowsGetSpaceUsed.class; + } + } else { + klass = Class.forName(newVal).asSubclass(GetSpaceUsed.class); + } + result = klass.getName(); + List volumeList = data.getVolumeList(); + for (FsVolumeImpl fsVolume : volumeList) { + Map blockPoolSlices = fsVolume.getBlockPoolSlices(); + for (BlockPoolSlice value : blockPoolSlices.values()) { + value.updateDfsUsageConfig(null, null, klass); } } } LOG.info("RECONFIGURE* changed {} to {}", property, newVal); return result; - } catch (IllegalArgumentException | IOException e) { + } catch (IllegalArgumentException | IOException | ClassNotFoundException e) { throw new ReconfigurationException(property, newVal, getConf().get(property), e); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java index 2a0823735ad..5bfba59f94a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java @@ -80,6 +80,7 @@ import org.apache.hadoop.util.Timer; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY; /** @@ -236,7 +237,8 @@ public class BlockPoolSlice { SHUTDOWN_HOOK_PRIORITY); } - public void updateDfsUsageConfig(Long interval, Long jitter) throws IOException { + public void updateDfsUsageConfig(Long interval, Long jitter, Class klass) + throws IOException { // Close the old dfsUsage if it is CachingGetSpaceUsed. if (dfsUsage instanceof CachingGetSpaceUsed) { ((CachingGetSpaceUsed) dfsUsage).close(); @@ -251,6 +253,9 @@ public class BlockPoolSlice { FS_GETSPACEUSED_JITTER_KEY + " should be larger than or equal to 0"); config.setLong(FS_GETSPACEUSED_JITTER_KEY, jitter); } + if (klass != null) { + config.setClass(FS_GETSPACEUSED_CLASSNAME, klass, CachingGetSpaceUsed.class); + } // Start new dfsUsage. this.dfsUsage = new FSCachingGetSpaceUsed.Builder().setBpid(bpid) .setVolume(volume) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java index 172a44557c1..e15c8ca80b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT; @@ -86,6 +87,7 @@ public class TestDataNodeReconfiguration { private final int NUM_NAME_NODE = 1; private final int NUM_DATA_NODE = 10; private MiniDFSCluster cluster; + private static long counter = 0; @Before public void Setup() throws IOException { @@ -756,4 +758,33 @@ public class TestDataNodeReconfiguration { } } } + + public static class DummyCachingGetSpaceUsed extends CachingGetSpaceUsed { + public DummyCachingGetSpaceUsed(Builder builder) throws IOException { + super(builder.setInterval(1000).setJitter(0L)); + } + + @Override + protected void refresh() { + counter++; + } + } + + @Test + public void testDfsUsageKlass() throws ReconfigurationException, InterruptedException { + + long lastCounter = counter; + Thread.sleep(5000); + assertEquals(lastCounter, counter); + + for (int i = 0; i < NUM_DATA_NODE; i++) { + DataNode dn = cluster.getDataNodes().get(i); + dn.reconfigurePropertyImpl(FS_GETSPACEUSED_CLASSNAME, + DummyCachingGetSpaceUsed.class.getName()); + } + + lastCounter = counter; + Thread.sleep(5000); + assertTrue(counter > lastCounter); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 21e0e1a8c17..8d9d48a163e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -331,7 +331,7 @@ public class TestDFSAdmin { final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("datanode", address, outs, errs); - assertEquals(18, outs.size()); + assertEquals(19, outs.size()); assertEquals(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, outs.get(1)); }