Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20196 Maintain all regions with same size in memstore flusher

This commit is contained in:
tedyu 2018-03-19 08:10:39 -07:00
parent 67f013430c
commit df5de33a02
2 changed files with 41 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -2759,14 +2759,20 @@ public class HRegionServer extends HasThread implements
configurationManager.registerObserver(region); configurationManager.registerObserver(region);
} }
private void addRegion(SortedMap<Long, Collection<HRegion>> sortedRegions, HRegion region,
long size) {
if (!sortedRegions.containsKey(size)) {
sortedRegions.put(size, new ArrayList<>());
}
sortedRegions.get(size).add(region);
}
/** /**
* @return A new Map of online regions sorted by region off-heap size with the first entry being * @return A new Map of online regions sorted by region off-heap size with the first entry being
* the biggest. If two regions are the same size, then the last one found wins; i.e. this * the biggest.
* method may NOT return all regions.
*/ */
SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedByOffHeapSize() { SortedMap<Long, Collection<HRegion>> getCopyOfOnlineRegionsSortedByOffHeapSize() {
// we'll sort the regions in reverse // we'll sort the regions in reverse
SortedMap<Long, HRegion> sortedRegions = new TreeMap<>( SortedMap<Long, Collection<HRegion>> sortedRegions = new TreeMap<>(
new Comparator<Long>() { new Comparator<Long>() {
@Override @Override
public int compare(Long a, Long b) { public int compare(Long a, Long b) {
@ -2775,19 +2781,18 @@ public class HRegionServer extends HasThread implements
}); });
// Copy over all regions. Regions are sorted by size with biggest first. // Copy over all regions. Regions are sorted by size with biggest first.
for (HRegion region : this.onlineRegions.values()) { for (HRegion region : this.onlineRegions.values()) {
sortedRegions.put(region.getMemStoreOffHeapSize(), region); addRegion(sortedRegions, region, region.getMemStoreOffHeapSize());
} }
return sortedRegions; return sortedRegions;
} }
/** /**
* @return A new Map of online regions sorted by region heap size with the first entry being the * @return A new Map of online regions sorted by region heap size with the first entry being the
* biggest. If two regions are the same size, then the last one found wins; i.e. this method * biggest.
* may NOT return all regions.
*/ */
SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedByOnHeapSize() { SortedMap<Long, Collection<HRegion>> getCopyOfOnlineRegionsSortedByOnHeapSize() {
// we'll sort the regions in reverse // we'll sort the regions in reverse
SortedMap<Long, HRegion> sortedRegions = new TreeMap<>( SortedMap<Long, Collection<HRegion>> sortedRegions = new TreeMap<>(
new Comparator<Long>() { new Comparator<Long>() {
@Override @Override
public int compare(Long a, Long b) { public int compare(Long a, Long b) {
@ -2796,7 +2801,7 @@ public class HRegionServer extends HasThread implements
}); });
// Copy over all regions. Regions are sorted by size with biggest first. // Copy over all regions. Regions are sorted by size with biggest first.
for (HRegion region : this.onlineRegions.values()) { for (HRegion region : this.onlineRegions.values()) {
sortedRegions.put(region.getMemStoreHeapSize(), region); addRegion(sortedRegions, region, region.getMemStoreHeapSize());
} }
return sortedRegions; return sortedRegions;
} }

47
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
import java.io.IOException; import java.io.IOException;
import java.lang.Thread.UncaughtExceptionHandler; import java.lang.Thread.UncaughtExceptionHandler;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.ConcurrentModificationException; import java.util.ConcurrentModificationException;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -155,7 +156,7 @@ class MemStoreFlusher implements FlushRequester {
* @return true if successful * @return true if successful
*/ */
private boolean flushOneForGlobalPressure() { private boolean flushOneForGlobalPressure() {
SortedMap<Long, HRegion> regionsBySize = null; SortedMap<Long, Collection<HRegion>> regionsBySize = null;
switch(flushType) { switch(flushType) {
case ABOVE_OFFHEAP_HIGHER_MARK: case ABOVE_OFFHEAP_HIGHER_MARK:
case ABOVE_OFFHEAP_LOWER_MARK: case ABOVE_OFFHEAP_LOWER_MARK:
@ -387,41 +388,45 @@ class MemStoreFlusher implements FlushRequester {
} }
private HRegion getBiggestMemStoreRegion( private HRegion getBiggestMemStoreRegion(
SortedMap<Long, HRegion> regionsBySize, SortedMap<Long, Collection<HRegion>> regionsBySize,
Set<HRegion> excludedRegions, Set<HRegion> excludedRegions,
boolean checkStoreFileCount) { boolean checkStoreFileCount) {
synchronized (regionsInQueue) { synchronized (regionsInQueue) {
for (HRegion region : regionsBySize.values()) { for (Map.Entry<Long, Collection<HRegion>> entry : regionsBySize.entrySet()) {
if (excludedRegions.contains(region)) { for (HRegion region : entry.getValue()) {
continue; if (excludedRegions.contains(region)) {
} continue;
}
if (region.writestate.flushing || !region.writestate.writesEnabled) { if (region.writestate.flushing || !region.writestate.writesEnabled) {
continue; continue;
} }
if (checkStoreFileCount && isTooManyStoreFiles(region)) { if (checkStoreFileCount && isTooManyStoreFiles(region)) {
continue; continue;
}
return region;
} }
return region;
} }
} }
return null; return null;
} }
private HRegion getBiggestMemStoreOfRegionReplica(SortedMap<Long, HRegion> regionsBySize, private HRegion getBiggestMemStoreOfRegionReplica(
SortedMap<Long, Collection<HRegion>> regionsBySize,
Set<HRegion> excludedRegions) { Set<HRegion> excludedRegions) {
synchronized (regionsInQueue) { synchronized (regionsInQueue) {
for (HRegion region : regionsBySize.values()) { for (Map.Entry<Long, Collection<HRegion>> entry : regionsBySize.entrySet()) {
if (excludedRegions.contains(region)) { for (HRegion region : entry.getValue()) {
continue; if (excludedRegions.contains(region)) {
} continue;
}
if (RegionReplicaUtil.isDefaultReplica(region.getRegionInfo())) { if (RegionReplicaUtil.isDefaultReplica(region.getRegionInfo())) {
continue; continue;
}
return region;
} }
return region;
} }
} }
return null; return null;