From 3dc03ccfb7e1964cdd06a57a11f292a95c13248f Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 26 Jan 2017 10:46:03 -0800 Subject: [PATCH] HBASE-17515 Reduce memory footprint of RegionLoads kept by StochasticLoadBalancer - revert due to pending discussion --- .../master/balancer/BalancerRegionLoad.java | 57 ------------------- .../master/balancer/BaseLoadBalancer.java | 10 ++-- .../balancer/StochasticLoadBalancer.java | 37 ++++++------ .../balancer/TestStochasticLoadBalancer.java | 12 ++-- 4 files changed, 30 insertions(+), 86 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java deleted file mode 100644 index 190981516d2..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.master.balancer; - -import org.apache.hadoop.hbase.RegionLoad; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * Wrapper class for the few fields required by the {@link StochasticLoadBalancer} - * from the full {@link RegionLoad}. - */ -@InterfaceStability.Evolving -class BalancerRegionLoad { - private final long readRequestsCount; - private final long writeRequestsCount; - private final int memStoreSizeMB; - private final int storefileSizeMB; - - BalancerRegionLoad(RegionLoad regionLoad) { - readRequestsCount = regionLoad.getReadRequestsCount(); - writeRequestsCount = regionLoad.getWriteRequestsCount(); - memStoreSizeMB = regionLoad.getMemStoreSizeMB(); - storefileSizeMB = regionLoad.getStorefileSizeMB(); - } - - public long getReadRequestsCount() { - return readRequestsCount; - } - - public long getWriteRequestsCount() { - return writeRequestsCount; - } - - public int getMemStoreSizeMB() { - return memStoreSizeMB; - } - - public int getStorefileSizeMB() { - return storefileSizeMB; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index e4a80b80b27..498d03dc139 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -113,7 +113,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { ArrayList tables; HRegionInfo[] regions; - Deque[] regionLoads; + Deque[] regionLoads; private RegionLocationFinder regionFinder; int[][] regionLocations; //regionIndex -> list of serverIndex sorted by locality @@ -161,7 +161,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { protected Cluster( Map> clusterState, - Map> loads, + Map> loads, RegionLocationFinder regionFinder, RackManager rackManager) { this(null, clusterState, loads, regionFinder, rackManager); @@ -171,7 +171,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { protected Cluster( Collection unassignedRegions, Map> clusterState, - Map> loads, + Map> loads, RegionLocationFinder regionFinder, RackManager rackManager) { @@ -426,7 +426,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { /** Helper for Cluster constructor to handle a region */ private void registerRegion(HRegionInfo region, int regionIndex, - int serverIndex, Map> loads, + int serverIndex, Map> loads, RegionLocationFinder regionFinder) { String tableName = region.getTable().getNameAsString(); if (!tablesToIndex.containsKey(tableName)) { @@ -443,7 +443,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { // region load if (loads != null) { - Deque rl = loads.get(region.getRegionNameAsString()); + Deque rl = loads.get(region.getRegionNameAsString()); // That could have failed if the RegionLoad is using the other regionName if (rl == null) { // Try getting the region load using encoded name. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index d8cb659786b..36d37099262 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -113,7 +113,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { private static final Random RANDOM = new Random(System.currentTimeMillis()); private static final Log LOG = LogFactory.getLog(StochasticLoadBalancer.class); - Map> loads = new HashMap>(); + Map> loads = new HashMap>(); // values are defaults private int maxSteps = 1000000; @@ -498,8 +498,8 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { private synchronized void updateRegionLoad() { // We create a new hashmap so that regions that are no longer there are removed. // However we temporarily need the old loads so we can use them to keep the rolling average. - Map> oldLoads = loads; - loads = new HashMap>(); + Map> oldLoads = loads; + loads = new HashMap>(); for (ServerName sn : clusterStatus.getServers()) { ServerLoad sl = clusterStatus.getLoad(sn); @@ -507,15 +507,16 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { continue; } for (Entry entry : sl.getRegionsLoad().entrySet()) { - Deque rLoads = oldLoads.get(Bytes.toString(entry.getKey())); + Deque rLoads = oldLoads.get(Bytes.toString(entry.getKey())); if (rLoads == null) { // There was nothing there - rLoads = new ArrayDeque(); + rLoads = new ArrayDeque(); } else if (rLoads.size() >= numRegionLoadsToRemember) { rLoads.remove(); } - rLoads.add(new BalancerRegionLoad(entry.getValue())); + rLoads.add(entry.getValue()); loads.put(Bytes.toString(entry.getKey()), rLoads); + } } @@ -1250,7 +1251,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { abstract static class CostFromRegionLoadFunction extends CostFunction { private ClusterStatus clusterStatus = null; - private Map> loads = null; + private Map> loads = null; private double[] stats = null; CostFromRegionLoadFunction(Configuration conf) { super(conf); @@ -1260,7 +1261,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { this.clusterStatus = status; } - void setLoads(Map> l) { + void setLoads(Map> l) { this.loads = l; } @@ -1280,7 +1281,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { // for every region on this server get the rl for(int regionIndex:cluster.regionsPerServer[i]) { - Collection regionLoadList = cluster.regionLoads[regionIndex]; + Collection regionLoadList = cluster.regionLoads[regionIndex]; // Now if we found a region load get the type of cost that was requested. if (regionLoadList != null) { @@ -1296,17 +1297,17 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { return costFromArray(stats); } - protected double getRegionLoadCost(Collection regionLoadList) { + protected double getRegionLoadCost(Collection regionLoadList) { double cost = 0; - for (BalancerRegionLoad rl : regionLoadList) { + for (RegionLoad rl : regionLoadList) { cost += getCostFromRl(rl); } return cost / regionLoadList.size(); } - protected abstract double getCostFromRl(BalancerRegionLoad rl); + protected abstract double getCostFromRl(RegionLoad rl); } /** @@ -1321,11 +1322,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { } @Override - protected double getRegionLoadCost(Collection regionLoadList) { + protected double getRegionLoadCost(Collection regionLoadList) { double cost = 0; double previous = 0; boolean isFirst = true; - for (BalancerRegionLoad rl : regionLoadList) { + for (RegionLoad rl : regionLoadList) { double current = getCostFromRl(rl); if (isFirst) { isFirst = false; @@ -1356,7 +1357,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { @Override - protected double getCostFromRl(BalancerRegionLoad rl) { + protected double getCostFromRl(RegionLoad rl) { return rl.getReadRequestsCount(); } } @@ -1377,7 +1378,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { } @Override - protected double getCostFromRl(BalancerRegionLoad rl) { + protected double getCostFromRl(RegionLoad rl) { return rl.getWriteRequestsCount(); } } @@ -1555,7 +1556,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { } @Override - protected double getCostFromRl(BalancerRegionLoad rl) { + protected double getCostFromRl(RegionLoad rl) { return rl.getMemStoreSizeMB(); } } @@ -1575,7 +1576,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { } @Override - protected double getCostFromRl(BalancerRegionLoad rl) { + protected double getCostFromRl(RegionLoad rl) { return rl.getStorefileSizeMB(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index ba030c878ba..3d975b8a179 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -69,7 +69,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { ServerLoad sl = mock(ServerLoad.class); RegionLoad rl = mock(RegionLoad.class); - when(rl.getStorefileSizeMB()).thenReturn(i); + when(rl.getStores()).thenReturn(i); Map regionLoadMap = new TreeMap(Bytes.BYTES_COMPARATOR); @@ -85,11 +85,11 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { assertTrue(loadBalancer.loads.get(REGION_KEY) != null); assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15); - Queue loads = loadBalancer.loads.get(REGION_KEY); + Queue loads = loadBalancer.loads.get(REGION_KEY); int i = 0; while(loads.size() > 0) { - BalancerRegionLoad rl = loads.remove(); - assertEquals(i + (numClusterStatusToAdd - 15), rl.getStorefileSizeMB()); + RegionLoad rl = loads.remove(); + assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores()); i ++; } } @@ -232,9 +232,9 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { @Test public void testRegionLoadCost() { - List regionLoads = new ArrayList<>(); + List regionLoads = new ArrayList<>(); for (int i = 1; i < 5; i++) { - BalancerRegionLoad regionLoad = mock(BalancerRegionLoad.class); + RegionLoad regionLoad = mock(RegionLoad.class); when(regionLoad.getReadRequestsCount()).thenReturn(new Long(i)); when(regionLoad.getStorefileSizeMB()).thenReturn(i); regionLoads.add(regionLoad);