From 9d891f08c28d1acdd0c1527efb784972f739ad1b Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 18 Jun 2020 12:26:22 +0100 Subject: [PATCH] HDFS-15406. Improve the speed of Datanode Block Scan. Contributed by hemanthboyina (cherry picked from commit 123777823edc98553fcef61f1913ab6e4cd5aa9a) --- .../hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java | 4 +++- .../apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 5a8229aee14..8810bfc2ac0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -133,6 +133,7 @@ public class FsVolumeImpl implements FsVolumeSpi { protected volatile long configuredCapacity; private final FileIoProvider fileIoProvider; private final DataNodeVolumeMetrics metrics; + private URI baseURI; /** * Per-volume worker pool that processes new blocks to cache. @@ -175,6 +176,7 @@ public class FsVolumeImpl implements FsVolumeSpi { File parent = currentDir.getParentFile(); cacheExecutor = initializeCacheExecutor(parent); this.metrics = DataNodeVolumeMetrics.create(conf, parent.getPath()); + this.baseURI = new File(currentDir.getParent()).toURI(); } else { cacheExecutor = null; this.metrics = null; @@ -500,7 +502,7 @@ public class FsVolumeImpl implements FsVolumeSpi { @Override public URI getBaseURI() { - return new File(currentDir.getParent()).toURI(); + return baseURI; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java index bb78cbce304..6201624b798 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java @@ -239,7 +239,7 @@ public class DataNodeTestUtils { try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset() .getFsVolumeReferences()) { for (FsVolumeSpi vol : volumes) { - if (vol.getBaseURI().equals(basePath.toURI())) { + if (new File(vol.getBaseURI()).equals(basePath)) { return (FsVolumeImpl) vol; } }