Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3507 requests count per HRegion and rebalance command; part 1 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1081610 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2011-03-14 23:38:57 +00:00
parent 941f2d89ae
commit 4196e5404c
6 changed files with 60 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -555,6 +555,11 @@
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
</dependency>
<dependency>
<groupId>com.github.stephenc.high-scale-lib</groupId>
<artifactId>high-scale-lib</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>

31
src/main/java/org/apache/hadoop/hbase/HServerLoad.java
View File

@ -65,6 +65,8 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
private int memstoreSizeMB;
/** the current total size of storefile indexes for the region, in MB */
private int storefileIndexSizeMB;
/** the current total request made to region */
private long requestsCount;
/**
* Constructor, for Writable
@ -80,16 +82,19 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
* @param storefileSizeMB
* @param memstoreSizeMB
* @param storefileIndexSizeMB
* @param requestsCount
*/
public RegionLoad(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB) {
final int memstoreSizeMB, final int storefileIndexSizeMB,
final long requestsCount) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
this.requestsCount = requestsCount;
}
// Getters
@ -143,6 +148,13 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
return storefileIndexSizeMB;
}
/**
* @return the number of requests made to region
*/
public long getRequestsCount() {
return requestsCount;
}
// Setters
/**
@ -181,6 +193,13 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
this.storefileIndexSizeMB = storefileIndexSizeMB;
}
/**
* @param requestsCount the number of requests to region
*/
public void setRequestsCount(long requestsCount) {
this.requestsCount = requestsCount;
}
// Writable
public void readFields(DataInput in) throws IOException {
int namelen = in.readInt();
@ -191,6 +210,7 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
this.storefileSizeMB = in.readInt();
this.memstoreSizeMB = in.readInt();
this.storefileIndexSizeMB = in.readInt();
this.requestsCount = in.readLong();
}
public void write(DataOutput out) throws IOException {
@ -201,6 +221,7 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
out.writeInt(storefileSizeMB);
out.writeInt(memstoreSizeMB);
out.writeInt(storefileIndexSizeMB);
out.writeLong(requestsCount);
}
/**
@ -218,6 +239,8 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
Integer.valueOf(this.memstoreSizeMB));
sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
Integer.valueOf(this.storefileIndexSizeMB));
sb = Strings.appendKeyValue(sb, "requestsCount",
Long.valueOf(this.requestsCount));
return sb.toString();
}
}
@ -452,14 +475,16 @@ public class HServerLoad implements WritableComparable<HServerLoad> {
* @param storefiles
* @param memstoreSizeMB
* @param storefileIndexSizeMB
* @param requestsCount
* @deprecated Use {@link #addRegionInfo(RegionLoad)}
*/
@Deprecated
public void addRegionInfo(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB) {
final int memstoreSizeMB, final int storefileIndexSizeMB,
final long requestsCount) {
this.regionLoad.add(new HServerLoad.RegionLoad(name, stores, storefiles,
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB));
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount));
}
// Writable

14
src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -93,9 +93,10 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
import org.cliffc.high_scale_lib.Counter;
import com.google.common.collect.ClassToInstanceMap;
import com.google.common.collect.Lists;
import com.google.common.collect.MutableClassToInstanceMap;
@ -174,6 +175,8 @@ public class HRegion implements HeapSize { // , Writable{
final AtomicLong memstoreSize = new AtomicLong(0);
final Counter requestsCount = new Counter();
/**
* The directory for the table this region is part of.
* This directory contains the directory for this region.
@ -455,6 +458,11 @@ public class HRegion implements HeapSize { // , Writable{
return this.regionInfo;
}
/** @return requestsCount for this region */
public long getRequestsCount() {
return this.requestsCount.get();
}
/** @return true if region is closed */
public boolean isClosed() {
return this.closed.get();
@ -2962,6 +2970,7 @@ public class HRegion implements HeapSize { // , Writable{
listPaths(fs, newRegionDir);
}
HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf, newRegionInfo, null);
dstRegion.requestsCount.set(a.requestsCount.get() + b.requestsCount.get());
dstRegion.initialize();
dstRegion.compactStores();
if (LOG.isDebugEnabled()) {
@ -3371,7 +3380,7 @@ public class HRegion implements HeapSize { // , Writable{
public static final long FIXED_OVERHEAD = ClassSize.align(
(4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
(24 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
(25 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
(ClassSize.OBJECT * 2) + (2 * ClassSize.ATOMIC_BOOLEAN) +
@ -3634,6 +3643,7 @@ public class HRegion implements HeapSize { // , Writable{
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closed");
}
this.requestsCount.increment();
}
/**

8
src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -901,6 +901,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
int storefileIndexSizeMB = 0;
long requestsCount = r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Store store : r.stores.values()) {
@ -909,8 +910,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
}
}
return new HServerLoad.RegionLoad(name, stores, storefiles,
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB);
return new HServerLoad.RegionLoad(name,stores, storefiles,
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount);
}
/**
@ -1149,11 +1150,13 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
int stores = 0;
int storefiles = 0;
long memstoreSize = 0;
long requestsCount = 0;
long storefileIndexSize = 0;
synchronized (this.onlineRegions) {
for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
HRegion r = e.getValue();
memstoreSize += r.memstoreSize.get();
requestsCount += r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Map.Entry<byte[], Store> ee : r.stores.entrySet()) {
@ -1167,6 +1170,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
this.metrics.stores.set(stores);
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
this.metrics.requestsCount.set(requestsCount);
this.metrics.storefileIndexSizeMB
.set((int) (storefileIndexSize / (1024 * 1024)));
this.metrics.compactionQueueSize.set(compactSplitThread

1
src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
View File

@ -543,6 +543,7 @@ public class SplitTransaction {
HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
this.parent.getLog(), fs, this.parent.getConf(),
hri, rsServices);
r.requestsCount.set(this.parent.getRequestsCount() / 2);
HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
return r;
}

8
src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
View File

@ -127,6 +127,11 @@ public class RegionServerMetrics implements Updater {
*/
public final MetricsIntValue storefiles = new MetricsIntValue("storefiles", registry);
/**
* Count of requests
*/
public final MetricsLongValue requestsCount = new MetricsLongValue("requestsCount", registry);
/**
* Sum of all the storefile index sizes in this regionserver in MB
*/
@ -243,6 +248,7 @@ public class RegionServerMetrics implements Updater {
this.storefiles.pushMetric(this.metricsRecord);
this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
this.memstoreSizeMB.pushMetric(this.metricsRecord);
this.requestsCount.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
this.requests.pushMetric(this.metricsRecord);
this.compactionQueueSize.pushMetric(this.metricsRecord);
@ -345,6 +351,8 @@ public class RegionServerMetrics implements Updater {
Integer.valueOf(this.storefileIndexSizeMB.get()));
sb = Strings.appendKeyValue(sb, "memstoreSize",
Integer.valueOf(this.memstoreSizeMB.get()));
sb = Strings.appendKeyValue(sb, "requestsCount",
Long.valueOf(this.requestsCount.get()));
sb = Strings.appendKeyValue(sb, "compactionQueueSize",
Integer.valueOf(this.compactionQueueSize.get()));
sb = Strings.appendKeyValue(sb, "flushQueueSize",