Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11683 Metrics for MOB (Jingcheng Du)

This commit is contained in:
Jonathan M Hsieh 2014-10-30 06:50:13 -07:00
parent 7df56b2039
commit e5a1b86dba
14 changed files with 601 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View File

@ -233,5 +233,37 @@ public interface MetricsRegionServerSource extends BaseSource {
String MAJOR_COMPACTED_CELLS_SIZE = "majorCompactedCellsSize";
String MAJOR_COMPACTED_CELLS_SIZE_DESC =
"The total amount of data processed during major compactions, in bytes";
String MOB_COMPACTED_INTO_MOB_CELLS_COUNT = "mobCompactedIntoMobCellsCount";
String MOB_COMPACTED_INTO_MOB_CELLS_COUNT_DESC =
"The number of cells moved to mob during compaction";
String MOB_COMPACTED_FROM_MOB_CELLS_COUNT = "mobCompactedFromMobCellsCount";
String MOB_COMPACTED_FROM_MOB_CELLS_COUNT_DESC =
"The number of cells moved from mob during compaction";
String MOB_COMPACTED_INTO_MOB_CELLS_SIZE = "mobCompactedIntoMobCellsSize";
String MOB_COMPACTED_INTO_MOB_CELLS_SIZE_DESC =
"The total amount of cells move to mob during compaction, in bytes";
String MOB_COMPACTED_FROM_MOB_CELLS_SIZE = "mobCompactedFromMobCellsSize";
String MOB_COMPACTED_FROM_MOB_CELLS_SIZE_DESC =
"The total amount of cells move from mob during compaction, in bytes";
String MOB_FLUSH_COUNT = "mobFlushCount";
String MOB_FLUSH_COUNT_DESC = "The number of the flushes in mob-enabled stores";
String MOB_FLUSHED_CELLS_COUNT = "mobFlushedCellsCount";
String MOB_FLUSHED_CELLS_COUNT_DESC = "The number of mob cells flushed to disk";
String MOB_FLUSHED_CELLS_SIZE = "mobFlushedCellsSize";
String MOB_FLUSHED_CELLS_SIZE_DESC = "The total amount of mob cells flushed to disk, in bytes";
String MOB_SCAN_CELLS_COUNT = "mobScanCellsCount";
String MOB_SCAN_CELLS_COUNT_DESC = "The number of scanned mob cells";
String MOB_SCAN_CELLS_SIZE = "mobScanCellsSize";
String MOB_SCAN_CELLS_SIZE_DESC = "The total amount of scanned mob cells, in bytes";
String MOB_FILE_CACHE_ACCESS_COUNT = "mobFileCacheAccessCount";
String MOB_FILE_CACHE_ACCESS_COUNT_DESC = "The count of accesses to the mob file cache";
String MOB_FILE_CACHE_MISS_COUNT = "mobFileCacheMissCount";
String MOB_FILE_CACHE_MISS_COUNT_DESC = "The count of misses to the mob file cache";
String MOB_FILE_CACHE_HIT_PERCENT = "mobFileCacheHitPercent";
String MOB_FILE_CACHE_HIT_PERCENT_DESC = "The hit percent to the mob file cache";
String MOB_FILE_CACHE_EVICTED_COUNT = "mobFileCacheEvictedCount";
String MOB_FILE_CACHE_EVICTED_COUNT_DESC = "The number of items evicted from the mob file cache";
String MOB_FILE_CACHE_COUNT = "mobFileCacheCount";
String MOB_FILE_CACHE_COUNT_DESC = "The count of cached mob files";
}

70
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
View File

@ -246,4 +246,74 @@ public interface MetricsRegionServerWrapper {
* Get the total amount of data processed during major compactions, in bytes.
*/
long getMajorCompactedCellsSize();
/**
* Gets the number of cells move to mob during compaction.
*/
long getMobCompactedIntoMobCellsCount();
/**
* Gets the number of cells move from mob during compaction.
*/
long getMobCompactedFromMobCellsCount();
/**
* Gets the total amount of cells move to mob during compaction, in bytes.
*/
long getMobCompactedIntoMobCellsSize();
/**
* Gets the total amount of cells move from mob during compaction, in bytes.
*/
long getMobCompactedFromMobCellsSize();
/**
* Gets the number of the flushes in mob-enabled stores.
*/
long getMobFlushCount();
/**
* Gets the number of mob cells flushed to disk.
*/
long getMobFlushedCellsCount();
/**
* Gets the total amount of mob cells flushed to disk, in bytes.
*/
long getMobFlushedCellsSize();
/**
* Gets the number of scanned mob cells.
*/
long getMobScanCellsCount();
/**
* Gets the total amount of scanned mob cells, in bytes.
*/
long getMobScanCellsSize();
/**
* Gets the count of accesses to the mob file cache.
*/
long getMobFileCacheAccessCount();
/**
* Gets the count of misses to the mob file cache.
*/
long getMobFileCacheMissCount();
/**
* Gets the number of items evicted from the mob file cache.
*/
long getMobFileCacheEvictedCount();
/**
* Gets the count of cached mob files.
*/
long getMobFileCacheCount();
/**
* Gets the hit percent to the mob file cache.
*/
int getMobFileCacheHitPercent();
}

29
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
View File

@ -219,6 +219,35 @@ public class MetricsRegionServerSourceImpl
rsWrap.getCompactedCellsSize())
.addCounter(Interns.info(MAJOR_COMPACTED_CELLS_SIZE, MAJOR_COMPACTED_CELLS_SIZE_DESC),
rsWrap.getMajorCompactedCellsSize())
.addCounter(Interns.info(MOB_COMPACTED_FROM_MOB_CELLS_COUNT, MOB_COMPACTED_FROM_MOB_CELLS_COUNT_DESC),
rsWrap.getMobCompactedFromMobCellsCount())
.addCounter(Interns.info(MOB_COMPACTED_INTO_MOB_CELLS_COUNT, MOB_COMPACTED_INTO_MOB_CELLS_COUNT_DESC),
rsWrap.getMobCompactedIntoMobCellsCount())
.addCounter(Interns.info(MOB_COMPACTED_FROM_MOB_CELLS_SIZE, MOB_COMPACTED_FROM_MOB_CELLS_SIZE_DESC),
rsWrap.getMobCompactedFromMobCellsSize())
.addCounter(Interns.info(MOB_COMPACTED_INTO_MOB_CELLS_SIZE, MOB_COMPACTED_INTO_MOB_CELLS_SIZE_DESC),
rsWrap.getMobCompactedIntoMobCellsSize())
.addCounter(Interns.info(MOB_FLUSH_COUNT, MOB_FLUSH_COUNT_DESC),
rsWrap.getMobFlushCount())
.addCounter(Interns.info(MOB_FLUSHED_CELLS_COUNT, MOB_FLUSHED_CELLS_COUNT_DESC),
rsWrap.getMobFlushedCellsCount())
.addCounter(Interns.info(MOB_FLUSHED_CELLS_SIZE, MOB_FLUSHED_CELLS_SIZE_DESC),
rsWrap.getMobFlushedCellsSize())
.addCounter(Interns.info(MOB_SCAN_CELLS_COUNT, MOB_SCAN_CELLS_COUNT_DESC),
rsWrap.getMobScanCellsCount())
.addCounter(Interns.info(MOB_SCAN_CELLS_SIZE, MOB_SCAN_CELLS_SIZE_DESC),
rsWrap.getMobScanCellsSize())
.addGauge(Interns.info(MOB_FILE_CACHE_COUNT, MOB_FILE_CACHE_COUNT_DESC),
rsWrap.getMobFileCacheCount())
.addCounter(Interns.info(MOB_FILE_CACHE_ACCESS_COUNT, MOB_FILE_CACHE_ACCESS_COUNT_DESC),
rsWrap.getMobFileCacheAccessCount())
.addCounter(Interns.info(MOB_FILE_CACHE_MISS_COUNT, MOB_FILE_CACHE_MISS_COUNT_DESC),
rsWrap.getMobFileCacheAccessCount())
.addCounter(
Interns.info(MOB_FILE_CACHE_EVICTED_COUNT, MOB_FILE_CACHE_EVICTED_COUNT_DESC),
rsWrap.getMobFileCacheEvictedCount())
.addGauge(Interns.info(MOB_FILE_CACHE_HIT_PERCENT, MOB_FILE_CACHE_HIT_PERCENT_DESC),
rsWrap.getMobFileCacheHitPercent())
.tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC),
rsWrap.getZookeeperQuorum())
.tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), rsWrap.getServerName())

12
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
View File

@ -128,6 +128,10 @@ public class DefaultMobCompactor extends DefaultCompactor {
long mobCells = 0;
Tag tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName()
.getName());
long mobCompactedIntoMobCellsCount = 0;
long mobCompactedFromMobCellsCount = 0;
long mobCompactedIntoMobCellsSize = 0;
long mobCompactedFromMobCellsSize = 0;
try {
try {
// If the mob file writer could not be created, directly write the cell to the store file.
@ -173,6 +177,8 @@ public class DefaultMobCompactor extends DefaultCompactor {
// next compaction.
writer.append(kv);
}
mobCompactedFromMobCellsCount++;
mobCompactedFromMobCellsSize += cell.getValueLength();
}
} else {
LOG.warn("The value format of the KeyValue " + kv
@ -192,6 +198,8 @@ public class DefaultMobCompactor extends DefaultCompactor {
KeyValue reference = MobUtils.createMobRefKeyValue(kv, fileName, tableNameTag);
// write the cell whose value is the path of a mob file to the store file.
writer.append(reference);
mobCompactedIntoMobCellsCount++;
mobCompactedIntoMobCellsSize += kv.getValueLength();
}
++progress.currentCompactedKVs;
@ -227,6 +235,10 @@ public class DefaultMobCompactor extends DefaultCompactor {
}
}
}
mobStore.updateMobCompactedFromMobCellsCount(mobCompactedFromMobCellsCount);
mobStore.updateMobCompactedIntoMobCellsCount(mobCompactedIntoMobCellsCount);
mobStore.updateMobCompactedFromMobCellsSize(mobCompactedFromMobCellsSize);
mobStore.updateMobCompactedIntoMobCellsSize(mobCompactedIntoMobCellsSize);
progress.complete();
return true;
}

11
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
View File

@ -155,7 +155,8 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
StoreFile.Writer mobFileWriter = null;
int compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX,
HConstants.COMPACTION_KV_MAX_DEFAULT);
long mobKVCount = 0;
long mobCount = 0;
long mobSize = 0;
long time = snapshot.getTimeRangeTracker().getMaximumTimestamp();
mobFileWriter = mobStore.createWriterInTmp(new Date(time), snapshot.getCellsCount(),
store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
@ -181,7 +182,8 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
} else {
// append the original keyValue in the mob file.
mobFileWriter.append(kv);
mobKVCount++;
mobSize += kv.getValueLength();
mobCount++;
// append the tags to the KeyValue.
// The key is same, the value is the filename of the mob file
@ -199,12 +201,15 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
mobFileWriter.close();
}
if (mobKVCount > 0) {
if (mobCount > 0) {
// commit the mob file from temp folder to target folder.
// If the mob file is committed successfully but the store file is not,
// the committed mob file will be handled by the sweep tool as an unused
// file.
mobStore.commitFile(mobFileWriter.getPath(), targetPath);
mobStore.updateMobFlushCount();
mobStore.updateMobFlushedCellsCount(mobCount);
mobStore.updateMobFlushedCellsSize(mobSize);
} else {
try {
// If the mob file is empty, delete it instead of committing.

9
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobCacheConfig.java
View File

@ -36,14 +36,21 @@ public class MobCacheConfig extends CacheConfig {
instantiateMobFileCache(conf);
}
public MobCacheConfig(Configuration conf) {
super(conf);
instantiateMobFileCache(conf);
}
/**
* Instantiates the MobFileCache.
* @param conf The current configuration.
* @return The current instance of MobFileCache.
*/
public static synchronized void instantiateMobFileCache(Configuration conf) {
public static synchronized MobFileCache instantiateMobFileCache(Configuration conf) {
if (mobFileCache == null) {
mobFileCache = new MobFileCache(conf);
}
return mobFileCache;
}
/**

75
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java
View File

@ -74,10 +74,12 @@ public class MobFileCache {
// a ConcurrentHashMap, accesses to this map are synchronized.
private Map<String, CachedMobFile> map = null;
// caches access count
private final AtomicLong count;
private long lastAccess;
private final AtomicLong miss;
private long lastMiss;
private final AtomicLong count = new AtomicLong(0);
private long lastAccess = 0;
private final AtomicLong miss = new AtomicLong(0);
private long lastMiss = 0;
private final AtomicLong evictedFileCount = new AtomicLong(0);
private long lastEvictedFileCount = 0;
// a lock to sync the evict to guarantee the eviction occurs in sequence.
// the method evictFile is not sync by this lock, the ConcurrentHashMap does the sync there.
@ -101,10 +103,6 @@ public class MobFileCache {
MobConstants.DEFAULT_MOB_FILE_CACHE_SIZE);
isCacheEnabled = (mobFileMaxCacheSize > 0);
map = new ConcurrentHashMap<String, CachedMobFile>(mobFileMaxCacheSize);
this.count = new AtomicLong(0);
this.miss = new AtomicLong(0);
this.lastAccess = 0;
this.lastMiss = 0;
if (isCacheEnabled) {
long period = conf.getLong(MobConstants.MOB_CACHE_EVICT_PERIOD,
MobConstants.DEFAULT_MOB_CACHE_EVICT_PERIOD); // in seconds
@ -159,6 +157,7 @@ public class MobFileCache {
for (CachedMobFile evictedFile : evictedFiles) {
closeFile(evictedFile);
}
evictedFileCount.addAndGet(evictedFiles.size());
}
}
@ -175,6 +174,7 @@ public class MobFileCache {
CachedMobFile evictedFile = map.remove(fileName);
if (evictedFile != null) {
evictedFile.close();
evictedFileCount.incrementAndGet();
}
} catch (IOException e) {
LOG.error("Fail to evict the file " + fileName, e);
@ -245,6 +245,26 @@ public class MobFileCache {
}
}
public void shutdown() {
this.scheduleThreadPool.shutdown();
for (int i = 0; i < 10; i++) {
if (!this.scheduleThreadPool.isShutdown()) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
LOG.warn("Interrupted while sleeping");
Thread.currentThread().interrupt();
break;
}
}
}
if (!this.scheduleThreadPool.isShutdown()) {
List<Runnable> runnables = this.scheduleThreadPool.shutdownNow();
LOG.debug("Still running " + runnables);
}
}
/**
* Gets the count of cached mob files.
* @return The count of the cached mob files.
@ -253,18 +273,51 @@ public class MobFileCache {
return map == null ? 0 : map.size();
}
/**
* Gets the count of accesses to the mob file cache.
* @return The count of accesses to the mob file cache.
*/
public long getAccessCount() {
return count.get();
}
/**
* Gets the count of misses to the mob file cache.
* @return The count of misses to the mob file cache.
*/
public long getMissCount() {
return miss.get();
}
/**
* Gets the number of items evicted from the mob file cache.
* @return The number of items evicted from the mob file cache.
*/
public long getEvictedFileCount() {
return evictedFileCount.get();
}
/**
* Gets the hit ratio to the mob file cache.
* @return The hit ratio to the mob file cache.
*/
public double getHitRatio() {
return count.get() == 0 ? 0 : ((float) (count.get() - miss.get())) / (float) count.get();
}
/**
* Prints the statistics.
*/
public void printStatistics() {
long access = count.get() - lastAccess;
long missed = miss.get() - lastMiss;
long hitRate = (access - missed) * 100 / access;
long evicted = evictedFileCount.get() - lastEvictedFileCount;
int hitRatio = access == 0 ? 0 : (int) (((float) (access - missed)) / (float) access * 100);
LOG.info("MobFileCache Statistics, access: " + access + ", miss: " + missed + ", hit: "
+ (access - missed) + ", hit rate: "
+ ((access == 0) ? 0 : hitRate) + "%");
+ (access - missed) + ", hit ratio: " + hitRatio + "%, evicted files: " + evicted);
lastAccess += access;
lastMiss += missed;
lastEvictedFileCount += evicted;
}
}

81
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
View File

@ -80,6 +80,15 @@ public class HMobStore extends HStore {
private MobCacheConfig mobCacheConfig;
private Path homePath;
private Path mobFamilyPath;
private volatile long mobCompactedIntoMobCellsCount = 0;
private volatile long mobCompactedFromMobCellsCount = 0;
private volatile long mobCompactedIntoMobCellsSize = 0;
private volatile long mobCompactedFromMobCellsSize = 0;
private volatile long mobFlushCount = 0;
private volatile long mobFlushedCellsCount = 0;
private volatile long mobFlushedCellsSize = 0;
private volatile long mobScanCellsCount = 0;
private volatile long mobScanCellsSize = 0;
private List<Path> mobDirLocations;
private HColumnDescriptor family;
@ -438,4 +447,76 @@ public class HMobStore extends HStore {
return super.compact(compaction);
}
}
public void updateMobCompactedIntoMobCellsCount(long count) {
mobCompactedIntoMobCellsCount += count;
}
public long getMobCompactedIntoMobCellsCount() {
return mobCompactedIntoMobCellsCount;
}
public void updateMobCompactedFromMobCellsCount(long count) {
mobCompactedFromMobCellsCount += count;
}
public long getMobCompactedFromMobCellsCount() {
return mobCompactedFromMobCellsCount;
}
public void updateMobCompactedIntoMobCellsSize(long size) {
mobCompactedIntoMobCellsSize += size;
}
public long getMobCompactedIntoMobCellsSize() {
return mobCompactedIntoMobCellsSize;
}
public void updateMobCompactedFromMobCellsSize(long size) {
mobCompactedFromMobCellsSize += size;
}
public long getMobCompactedFromMobCellsSize() {
return mobCompactedFromMobCellsSize;
}
public void updateMobFlushCount() {
mobFlushCount++;
}
public long getMobFlushCount() {
return mobFlushCount;
}
public void updateMobFlushedCellsCount(long count) {
mobFlushedCellsCount += count;
}
public long getMobFlushedCellsCount() {
return mobFlushedCellsCount;
}
public void updateMobFlushedCellsSize(long size) {
mobFlushedCellsSize += size;
}
public long getMobFlushedCellsSize() {
return mobFlushedCellsSize;
}
public void updateMobScanCellsCount(long count) {
mobScanCellsCount += count;
}
public long getMobScanCellsCount() {
return mobScanCellsCount;
}
public void updateMobScanCellsSize(long size) {
mobScanCellsSize += size;
}
public long getMobScanCellsSize() {
return mobScanCellsSize;
}
}

5
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -91,6 +91,7 @@ import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.master.TableLockManager;
import org.apache.hadoop.hbase.mob.MobCacheConfig;
import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
@ -347,6 +348,8 @@ public class HRegionServer extends HasThread implements
// Cache configuration and block cache reference
final CacheConfig cacheConfig;
// Cache configuration for mob
final MobCacheConfig mobCacheConfig;
/** The health check chore. */
private HealthCheckChore healthCheckChore;
@ -476,6 +479,7 @@ public class HRegionServer extends HasThread implements
regionServerAccounting = new RegionServerAccounting();
cacheConfig = new CacheConfig(conf);
mobCacheConfig = new MobCacheConfig(conf);
uncaughtExceptionHandler = new UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
@ -830,6 +834,7 @@ public class HRegionServer extends HasThread implements
if (cacheConfig.isBlockCacheEnabled()) {
cacheConfig.getBlockCache().shutdown();
}
mobCacheConfig.getMobFileCache().shutdown();
if (movedRegionsCleaner != null) {
movedRegionsCleaner.stop("Region Server stopping");

134
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View File

@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CacheStats;
import org.apache.hadoop.hbase.mob.MobCacheConfig;
import org.apache.hadoop.hbase.mob.MobFileCache;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.metrics2.MetricsExecutor;
@ -48,6 +50,7 @@ class MetricsRegionServerWrapperImpl
private final HRegionServer regionServer;
private BlockCache blockCache;
private MobFileCache mobFileCache;
private volatile long numStores = 0;
private volatile long numHLogFiles = 0;
@ -72,6 +75,20 @@ class MetricsRegionServerWrapperImpl
private volatile long flushedCellsSize = 0;
private volatile long compactedCellsSize = 0;
private volatile long majorCompactedCellsSize = 0;
private volatile long mobCompactedIntoMobCellsCount = 0;
private volatile long mobCompactedFromMobCellsCount = 0;
private volatile long mobCompactedIntoMobCellsSize = 0;
private volatile long mobCompactedFromMobCellsSize = 0;
private volatile long mobFlushCount = 0;
private volatile long mobFlushedCellsCount = 0;
private volatile long mobFlushedCellsSize = 0;
private volatile long mobScanCellsCount = 0;
private volatile long mobScanCellsSize = 0;
private volatile long mobFileCacheAccessCount = 0;
private volatile long mobFileCacheMissCount = 0;
private volatile double mobFileCacheHitRatio = 0;
private volatile long mobFileCacheEvictedCount = 0;
private volatile long mobFileCacheCount = 0;
private CacheStats cacheStats;
private ScheduledExecutorService executor;
@ -81,6 +98,7 @@ class MetricsRegionServerWrapperImpl
public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) {
this.regionServer = regionServer;
initBlockCache();
initMobFileCache();
this.period =
regionServer.conf.getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
@ -112,6 +130,16 @@ class MetricsRegionServerWrapperImpl
}
}
/**
* Initializes the mob file cache.
*/
private synchronized void initMobFileCache() {
MobCacheConfig mobCacheConfig = this.regionServer.mobCacheConfig;
if (mobCacheConfig != null && this.mobFileCache == null) {
this.mobFileCache = mobCacheConfig.getMobFileCache();
}
}
@Override
public String getClusterId() {
return regionServer.getClusterId();
@ -389,6 +417,76 @@ class MetricsRegionServerWrapperImpl
return majorCompactedCellsSize;
}
@Override
public long getMobCompactedFromMobCellsCount() {
return mobCompactedFromMobCellsCount;
}
@Override
public long getMobCompactedIntoMobCellsCount() {
return mobCompactedIntoMobCellsCount;
}
@Override
public long getMobCompactedFromMobCellsSize() {
return mobCompactedFromMobCellsSize;
}
@Override
public long getMobCompactedIntoMobCellsSize() {
return mobCompactedIntoMobCellsSize;
}
@Override
public long getMobFlushCount() {
return mobFlushCount;
}
@Override
public long getMobFlushedCellsCount() {
return mobFlushedCellsCount;
}
@Override
public long getMobFlushedCellsSize() {
return mobFlushedCellsSize;
}
@Override
public long getMobScanCellsCount() {
return mobScanCellsCount;
}
@Override
public long getMobScanCellsSize() {
return mobScanCellsSize;
}
@Override
public long getMobFileCacheAccessCount() {
return mobFileCacheAccessCount;
}
@Override
public long getMobFileCacheMissCount() {
return mobFileCacheMissCount;
}
@Override
public long getMobFileCacheCount() {
return mobFileCacheCount;
}
@Override
public long getMobFileCacheEvictedCount() {
return mobFileCacheEvictedCount;
}
@Override
public int getMobFileCacheHitPercent() {
return (int) (mobFileCacheHitRatio * 100);
}
/**
* This is the runnable that will be executed on the executor every PERIOD number of seconds
* It will take metrics/numbers from all of the regions and use them to compute point in
@ -402,6 +500,7 @@ class MetricsRegionServerWrapperImpl
@Override
synchronized public void run() {
initBlockCache();
initMobFileCache();
cacheStats = blockCache.getStats();
HDFSBlocksDistribution hdfsBlocksDistribution =
@ -427,6 +526,15 @@ class MetricsRegionServerWrapperImpl
long tempFlushedCellsSize = 0;
long tempCompactedCellsSize = 0;
long tempMajorCompactedCellsSize = 0;
long tempMobCompactedIntoMobCellsCount = 0;
long tempMobCompactedFromMobCellsCount = 0;
long tempMobCompactedIntoMobCellsSize = 0;
long testMobCompactedFromMobCellsSize = 0;
long tempMobFlushCount = 0;
long tempMobFlushedCellsCount = 0;
long tempMobFlushedCellsSize = 0;
long tempMobScanCellsCount = 0;
long tempMobScanCellsSize = 0;
for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get();
@ -449,6 +557,18 @@ class MetricsRegionServerWrapperImpl
tempFlushedCellsSize += store.getFlushedCellsSize();
tempCompactedCellsSize += store.getCompactedCellsSize();
tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
if (store instanceof HMobStore) {
HMobStore mobStore = (HMobStore) store;
tempMobCompactedIntoMobCellsCount += mobStore.getMobCompactedIntoMobCellsCount();
tempMobCompactedFromMobCellsCount += mobStore.getMobCompactedFromMobCellsCount();
tempMobCompactedIntoMobCellsSize += mobStore.getMobCompactedIntoMobCellsSize();
testMobCompactedFromMobCellsSize += mobStore.getMobCompactedFromMobCellsSize();
tempMobFlushCount += mobStore.getMobFlushCount();
tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
tempMobScanCellsCount += mobStore.getMobScanCellsCount();
tempMobScanCellsSize += mobStore.getMobScanCellsSize();
}
}
hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution());
@ -511,6 +631,20 @@ class MetricsRegionServerWrapperImpl
flushedCellsSize = tempFlushedCellsSize;
compactedCellsSize = tempCompactedCellsSize;
majorCompactedCellsSize = tempMajorCompactedCellsSize;
mobCompactedIntoMobCellsCount = tempMobCompactedIntoMobCellsCount;
mobCompactedFromMobCellsCount = tempMobCompactedFromMobCellsCount;
mobCompactedIntoMobCellsSize = tempMobCompactedIntoMobCellsSize;
mobCompactedFromMobCellsSize = testMobCompactedFromMobCellsSize;
mobFlushCount = tempMobFlushCount;
mobFlushedCellsCount = tempMobFlushedCellsCount;
mobFlushedCellsSize = tempMobFlushedCellsSize;
mobScanCellsCount = tempMobScanCellsCount;
mobScanCellsSize = tempMobScanCellsSize;
mobFileCacheAccessCount = mobFileCache.getAccessCount();
mobFileCacheMissCount = mobFileCache.getMissCount();
mobFileCacheHitRatio = mobFileCache.getHitRatio();
mobFileCacheEvictedCount = mobFileCache.getEvictedFileCount();
mobFileCacheCount = mobFileCache.getCacheSize();
}
}
}

15
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java
View File

@ -36,11 +36,16 @@ import org.apache.hadoop.hbase.mob.MobUtils;
public class MobStoreScanner extends StoreScanner {
private boolean cacheMobBlocks = false;
private final HMobStore mobStore;
public MobStoreScanner(Store store, ScanInfo scanInfo, Scan scan,
final NavigableSet<byte[]> columns, long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
cacheMobBlocks = MobUtils.isCacheMobBlocks(scan);
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
mobStore = (HMobStore) store;
}
/**
@ -56,13 +61,19 @@ public class MobStoreScanner extends StoreScanner {
if (outResult.isEmpty()) {
return result;
}
HMobStore mobStore = (HMobStore) store;
long mobKVCount = 0;
long mobKVSize = 0;
for (int i = 0; i < outResult.size(); i++) {
Cell cell = outResult.get(i);
if (MobUtils.isMobReferenceCell(cell)) {
outResult.set(i, mobStore.resolve(cell, cacheMobBlocks));
Cell mobCell = mobStore.resolve(cell, cacheMobBlocks);
mobKVCount++;
mobKVSize += mobCell.getValueLength();
outResult.set(i, mobCell);
}
}
mobStore.updateMobScanCellsCount(mobKVCount);
mobStore.updateMobScanCellsSize(mobKVSize);
}
return result;
}

15
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
View File

@ -36,11 +36,16 @@ import org.apache.hadoop.hbase.mob.MobUtils;
public class ReversedMobStoreScanner extends ReversedStoreScanner {
private boolean cacheMobBlocks = false;
protected final HMobStore mobStore;
ReversedMobStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet<byte[]> columns,
long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
cacheMobBlocks = MobUtils.isCacheMobBlocks(scan);
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
mobStore = (HMobStore) store;
}
/**
@ -56,13 +61,19 @@ public class ReversedMobStoreScanner extends ReversedStoreScanner {
if (outResult.isEmpty()) {
return result;
}
HMobStore mobStore = (HMobStore) store;
long mobKVCount = 0;
long mobKVSize = 0;
for (int i = 0; i < outResult.size(); i++) {
Cell cell = outResult.get(i);
if (MobUtils.isMobReferenceCell(cell)) {
outResult.set(i, mobStore.resolve(cell, cacheMobBlocks));
Cell mobCell = mobStore.resolve(cell, cacheMobBlocks);
mobKVCount++;
mobKVSize += mobCell.getValueLength();
outResult.set(i, mobCell);
}
}
mobStore.updateMobScanCellsCount(mobKVCount);
mobStore.updateMobScanCellsSize(mobKVSize);
}
return result;
}

70
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
View File

@ -241,4 +241,74 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 10240000;
}
@Override
public long getMobCompactedIntoMobCellsCount() {
return 20;
}
@Override
public long getMobCompactedFromMobCellsCount() {
return 10;
}
@Override
public long getMobCompactedIntoMobCellsSize() {
return 200;
}
@Override
public long getMobCompactedFromMobCellsSize() {
return 100;
}
@Override
public long getMobFlushCount() {
return 1;
}
@Override
public long getMobFlushedCellsCount() {
return 10;
}
@Override
public long getMobFlushedCellsSize() {
return 1000;
}
@Override
public long getMobScanCellsCount() {
return 10;
}
@Override
public long getMobScanCellsSize() {
return 1000;
}
@Override
public long getMobFileCacheAccessCount() {
return 100;
}
@Override
public long getMobFileCacheMissCount() {
return 50;
}
@Override
public long getMobFileCacheEvictedCount() {
return 0;
}
@Override
public long getMobFileCacheCount() {
return 100;
}
@Override
public int getMobFileCacheHitPercent() {
return 50;
}
}

62
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
View File

@ -376,4 +376,66 @@ public class TestRegionServerMetrics {
metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
}
}
@Test
public void testMobMetrics() throws IOException, InterruptedException {
String tableNameString = "testMobMetrics";
TableName tableName = TableName.valueOf(tableNameString);
byte[] cf = Bytes.toBytes("d");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("mobdata");
int compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(cf);
hcd.setMobEnabled(true);
hcd.setMobThreshold(0);
htd.addFamily(hcd);
HBaseAdmin admin = new HBaseAdmin(conf);
HTable t = TEST_UTIL.createTable(htd, new byte[0][0], conf);
HRegion region = rs.getOnlineRegions(tableName).get(0);
t.setAutoFlush(true, true);
for (int insertCount = 0; insertCount < compactionThreshold; insertCount++) {
Put p = new Put(Bytes.toBytes(insertCount));
p.add(cf, qualifier, val);
t.put(p);
admin.flush(tableName);
}
metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("mobFlushCount", compactionThreshold, serverSource);
Scan scan = new Scan(Bytes.toBytes(0), Bytes.toBytes(2));
ResultScanner scanner = t.getScanner(scan);
scanner.next(100);
scanner.close();
metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("mobScanCellsCount", 2, serverSource);
region.getTableDesc().getFamily(cf).setMobThreshold(100);
region.initialize();
region.compactStores(true);
metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("mobCompactedFromMobCellsCount", compactionThreshold,
serverSource);
metricsHelper.assertCounter("mobCompactedIntoMobCellsCount", 0, serverSource);
scanner = t.getScanner(scan);
scanner.next(100);
metricsRegionServer.getRegionServerWrapper().forceRecompute();
// metrics are reset by the region initialization
metricsHelper.assertCounter("mobScanCellsCount", 0, serverSource);
for (int insertCount = compactionThreshold;
insertCount < 2 * compactionThreshold - 1; insertCount++) {
Put p = new Put(Bytes.toBytes(insertCount));
p.add(cf, qualifier, val);
t.put(p);
admin.flush(tableName);
}
region.getTableDesc().getFamily(cf).setMobThreshold(0);
region.initialize();
region.compactStores(true);
metricsRegionServer.getRegionServerWrapper().forceRecompute();
// metrics are reset by the region initialization
metricsHelper.assertCounter("mobCompactedFromMobCellsCount", 0, serverSource);
metricsHelper.assertCounter("mobCompactedIntoMobCellsCount", 2 * compactionThreshold - 1,
serverSource);
t.close();
admin.close();
}
}