Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13863 Multi-wal feature breaks reported number and size of HLogs (Abhilash)

This commit is contained in:
tedyu 2015-06-26 08:52:24 -07:00
parent f046c4b043
commit 183fa5d232
3 changed files with 63 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View File

@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CacheStats;
import org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@ -516,14 +517,16 @@ class MetricsRegionServerWrapperImpl
//If we've time traveled keep the last requests per second.
if ((currentTime - lastRan) > 0) {
long currentRequestCount = getTotalRequestCount();
requestsPerSecond = (currentRequestCount - lastRequestCount) / ((currentTime - lastRan) / 1000.0);
requestsPerSecond = (currentRequestCount - lastRequestCount) /
((currentTime - lastRan) / 1000.0);
lastRequestCount = currentRequestCount;
}
lastRan = currentTime;
numWALFiles = DefaultWALProvider.getNumLogFiles(regionServer.walFactory);
walFileSize = DefaultWALProvider.getLogFileSize(regionServer.walFactory);
numWALFiles = DefaultWALProvider.getNumLogFiles(regionServer.walFactory) +
BoundedRegionGroupingProvider.getNumLogFiles(regionServer.walFactory);
walFileSize = DefaultWALProvider.getLogFileSize(regionServer.walFactory) +
BoundedRegionGroupingProvider.getLogFileSize(regionServer.walFactory);
//Copy over computed values so that no thread sees half computed values.
numStores = tempNumStores;
numStoreFiles = tempNumStoreFiles;

57
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRegionGroupingProvider.java
View File

@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
// imports for classes still in regionserver.wal
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
* {@link RegionGroupingProvider}.
*/
@InterfaceAudience.Private
class BoundedRegionGroupingProvider extends RegionGroupingProvider {
public class BoundedRegionGroupingProvider extends RegionGroupingProvider {
private static final Log LOG = LogFactory.getLog(BoundedRegionGroupingProvider.class);
static final String NUM_REGION_GROUPS = "hbase.wal.regiongrouping.numgroups";
@ -103,4 +103,57 @@ class BoundedRegionGroupingProvider extends RegionGroupingProvider {
throw failure;
}
}
/**
* iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or non-meta,
* count the number of files (rolled and active). if either of them isn't, count 0
* for that provider.
* @param walFactory may not be null.
*/
public static long getNumLogFiles(WALFactory walFactory) {
long result = 0;
if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
BoundedRegionGroupingProvider groupProviders =
(BoundedRegionGroupingProvider)walFactory.provider;
for (int i = 0; i < groupProviders.delegates.length; i++) {
result +=
((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getNumLogFiles();
}
}
WALProvider meta = walFactory.metaProvider.get();
if (meta instanceof BoundedRegionGroupingProvider) {
for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++) {
result += ((FSHLog)
((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
.getNumLogFiles(); }
}
return result;
}
/**
* iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or non-meta,
* count the size of files (rolled and active). if either of them isn't, count 0
* for that provider.
* @param walFactory may not be null.
*/
public static long getLogFileSize(WALFactory walFactory) {
long result = 0;
if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
BoundedRegionGroupingProvider groupProviders =
(BoundedRegionGroupingProvider)walFactory.provider;
for (int i = 0; i < groupProviders.delegates.length; i++) {
result +=
((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getLogFileSize();
}
}
WALProvider meta = walFactory.metaProvider.get();
if (meta instanceof BoundedRegionGroupingProvider) {
for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++) {
result += ((FSHLog)
((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
.getLogFileSize();
}
}
return result;
}
}

2
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
View File

@ -72,7 +72,7 @@ public class DefaultWALProvider implements WALProvider {
void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
}
private FSHLog log = null;
protected FSHLog log = null;
/**
* @param factory factory that made us, identity used for FS layout. may not be null