From ee450ae94370fb047a3d60883c0ae6016eac13e4 Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Tue, 1 Sep 2020 15:51:33 +0530 Subject: [PATCH] HBASE-24569 Get hostAndWeights in addition using localhost if it is null in local mode Closes #1909 Signed-off-by: Duo Zhang Signed-off-by: Viraj Jasani --- .../hadoop/hbase/HDFSBlocksDistribution.java | 14 +++++++++++++- .../hbase/TestHDFSBlocksDistribution.java | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java index 9d677f8053b..0f0e9380623 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java @@ -26,9 +26,9 @@ import java.util.NavigableSet; import java.util.TreeMap; import java.util.TreeSet; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hbase.util.DNS; import org.apache.yetus.audience.InterfaceAudience; - /** * Data structure to describe the distribution of HDFS blocks among hosts. * @@ -280,6 +280,18 @@ public class HDFSBlocksDistribution { private long getBlocksLocalityWeightInternal(String host, Visitor visitor) { long localityIndex = 0; HostAndWeight hostAndWeight = this.hostAndWeights.get(host); + // Compatible with local mode, see HBASE-24569 + if (hostAndWeight == null) { + String currentHost = ""; + try { + currentHost = DNS.getDefaultHost("default", "default"); + } catch (Exception e) { + // Just ignore, it's ok, avoid too many log info + } + if (host.equals(currentHost)) { + hostAndWeight = this.hostAndWeights.get(HConstants.LOCALHOST); + } + } if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) { localityIndex = visitor.visit(hostAndWeight); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java index bebc7e357c9..17f503855c8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import java.util.HashMap; import java.util.Map; @@ -25,6 +26,7 @@ import java.util.Map; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.DNS; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -81,4 +83,19 @@ public class TestHDFSBlocksDistribution { assertEquals("Should be one host", 1, distribution.getHostAndWeights().size()); assertEquals("Total weight should be 10", 10, distribution.getUniqueBlocksTotalWeight()); } + + @Test + public void testLocalHostCompatibility() throws Exception { + String currentHost = DNS.getDefaultHost("default", "default"); + HDFSBlocksDistribution distribution = new HDFSBlocksDistribution(); + assertEquals("Locality should be 0.0", 0.0, + distribution.getBlockLocalityIndex(currentHost), 0.01); + distribution.addHostsAndBlockWeight(new String[] { "localhost" }, 10); + assertEquals("Should be one host", 1, distribution.getHostAndWeights().size()); + assertEquals("Locality should be 0.0", 0.0, + distribution.getBlockLocalityIndex("test"), 0.01); + assertNotEquals("Locality should be 0.0", 0.0, + distribution.getBlockLocalityIndex(currentHost), 0.01); + } + }