Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-3647 Distinguish read and write request count in region -- reversed the patch because missing a file 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1087930 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2011-04-01 23:18:24 +00:00
parent 556fa73400
commit 371f963359
6 changed files with 51 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/apache/hadoop/hbase/HServerInfo.java
View File

@ -27,7 +27,6 @@ import java.util.Comparator;
import java.util.Set;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.io.VersionedWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@ -44,10 +43,7 @@ import org.apache.hadoop.io.WritableComparable;
* by. In subsequent communications, the regionserver will pass a HServerInfo
* with the master-supplied address.
*/
public class HServerInfo extends VersionedWritable
implements WritableComparable<HServerInfo> {
private static final byte VERSION = 0;
public class HServerInfo implements WritableComparable<HServerInfo> {
/*
* This character is used as separator between server hostname and port and
* its startcode. Servername is formatted as
@ -65,11 +61,6 @@ public class HServerInfo extends VersionedWritable
private String hostname;
private String cachedHostnamePort = null;
/** @return the object version number */
public byte getVersion() {
return VERSION;
}
public HServerInfo() {
this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT,
"default name");

73
src/main/java/org/apache/hadoop/hbase/HServerLoad.java
View File

@ -28,17 +28,13 @@ import java.util.TreeMap;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Strings;
import org.apache.hadoop.io.VersionedWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
/**
* This class encapsulates metrics for determining the load on a HRegionServer
*/
public class HServerLoad extends VersionedWritable
implements WritableComparable<HServerLoad> {
private static final byte VERSION = 0;
public class HServerLoad implements WritableComparable<HServerLoad> {
/** number of regions */
// could just use regionLoad.size() but master.RegionManager likes to play
// around with this value while passing HServerLoad objects around during
@ -53,11 +49,6 @@ public class HServerLoad extends VersionedWritable
/** per-region load metrics */
private Map<byte[], RegionLoad> regionLoad = new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
/** @return the object version number */
public byte getVersion() {
return VERSION;
}
/**
* Encapsulates per-region loading metrics.
*/
@ -74,10 +65,8 @@ public class HServerLoad extends VersionedWritable
private int memstoreSizeMB;
/** the current total size of storefile indexes for the region, in MB */
private int storefileIndexSizeMB;
/** the current total read requests made to region */
private int readRequestsCount;
/** the current total write requests made to region */
private int writeRequestsCount;
/** the current total request made to region */
private long requestsCount;
/**
* Constructor, for Writable
@ -93,21 +82,18 @@ public class HServerLoad extends VersionedWritable
* @param storefileSizeMB
* @param memstoreSizeMB
* @param storefileIndexSizeMB
* @param readRequestsCount
* @param writeRequestsCount
* @param requestsCount
*/
public RegionLoad(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB,
final int readRequestsCount, final int writeRequestsCount) {
final int memstoreSizeMB, final int storefileIndexSizeMB,final long requestsCount) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
this.readRequestsCount = readRequestsCount;
this.writeRequestsCount = writeRequestsCount;
this.requestsCount = requestsCount;
}
// Getters
@ -160,26 +146,12 @@ public class HServerLoad extends VersionedWritable
public int getStorefileIndexSizeMB() {
return storefileIndexSizeMB;
}
/**
* @return the number of requests made to region
*/
public long getRequestsCount() {
return readRequestsCount + writeRequestsCount;
}
/**
* @return the number of read requests made to region
*/
public long getReadRequestsCount() {
return readRequestsCount;
}
/**
* @return the number of read requests made to region
*/
public long getWriteRequestsCount() {
return writeRequestsCount;
return requestsCount;
}
// Setters
@ -221,17 +193,10 @@ public class HServerLoad extends VersionedWritable
}
/**
* @param requestsCount the number of read requests to region
* @param requestsCount the number of requests to region
*/
public void setReadRequestsCount(int requestsCount) {
this.readRequestsCount = requestsCount;
}
/**
* @param requestsCount the number of write requests to region
*/
public void setWriteRequestsCount(int requestsCount) {
this.writeRequestsCount = requestsCount;
public void setRequestsCount(long requestsCount) {
this.requestsCount = requestsCount;
}
// Writable
@ -244,8 +209,7 @@ public class HServerLoad extends VersionedWritable
this.storefileSizeMB = in.readInt();
this.memstoreSizeMB = in.readInt();
this.storefileIndexSizeMB = in.readInt();
this.readRequestsCount = in.readInt();
this.writeRequestsCount = in.readInt();
this.requestsCount = in.readLong();
}
public void write(DataOutput out) throws IOException {
@ -256,8 +220,7 @@ public class HServerLoad extends VersionedWritable
out.writeInt(storefileSizeMB);
out.writeInt(memstoreSizeMB);
out.writeInt(storefileIndexSizeMB);
out.writeInt(readRequestsCount);
out.writeInt(writeRequestsCount);
out.writeLong(requestsCount);
}
/**
@ -275,10 +238,8 @@ public class HServerLoad extends VersionedWritable
Integer.valueOf(this.memstoreSizeMB));
sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
Integer.valueOf(this.storefileIndexSizeMB));
sb = Strings.appendKeyValue(sb, "readRequestsCount",
Long.valueOf(this.readRequestsCount));
sb = Strings.appendKeyValue(sb, "writeRequestsCount",
Long.valueOf(this.writeRequestsCount));
sb = Strings.appendKeyValue(sb, "requestsCount",
Long.valueOf(this.requestsCount));
return sb.toString();
}
}
@ -522,9 +483,9 @@ public class HServerLoad extends VersionedWritable
public void addRegionInfo(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB,
final int readRequestsCount, final int writeRequestsCount) {
final long requestsCount) {
this.regionLoad.put(name, new HServerLoad.RegionLoad(name, stores, storefiles,
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, readRequestsCount, writeRequestsCount));
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount));
}
// Writable

48
src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -177,8 +177,7 @@ public class HRegion implements HeapSize { // , Writable{
final AtomicLong memstoreSize = new AtomicLong(0);
final Counter readRequestsCount = new Counter();
final Counter writeRequestsCount = new Counter();
final Counter requestsCount = new Counter();
/**
* The directory for the table this region is part of.
@ -463,17 +462,7 @@ public class HRegion implements HeapSize { // , Writable{
/** @return requestsCount for this region */
public long getRequestsCount() {
return this.readRequestsCount.get() + this.writeRequestsCount.get();
}
/** @return readRequestsCount for this region */
public long getReadRequestsCount() {
return this.readRequestsCount.get();
}
/** @return writeRequestsCount for this region */
public long getWriteRequestsCount() {
return this.writeRequestsCount.get();
return this.requestsCount.get();
}
/** @return true if region is closed */
@ -1142,7 +1131,6 @@ public class HRegion implements HeapSize { // , Writable{
// closest key is across all column families, since the data may be sparse
checkRow(row);
startRegionOperation();
this.readRequestsCount.increment();
try {
Store store = getStore(family);
KeyValue kv = new KeyValue(row, HConstants.LATEST_TIMESTAMP);
@ -1179,7 +1167,6 @@ public class HRegion implements HeapSize { // , Writable{
protected InternalScanner getScanner(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
startRegionOperation();
this.readRequestsCount.increment();
try {
// Verify families are all valid
if(scan.hasFamilies()) {
@ -1248,7 +1235,6 @@ public class HRegion implements HeapSize { // , Writable{
checkResources();
Integer lid = null;
startRegionOperation();
this.writeRequestsCount.increment();
try {
byte [] row = delete.getRow();
// If we did not pass an existing row lock, obtain a new one
@ -1402,7 +1388,6 @@ public class HRegion implements HeapSize { // , Writable{
// will be extremely rare; we'll deal with it when it happens.
checkResources();
startRegionOperation();
this.writeRequestsCount.increment();
try {
// We obtain a per-row lock, so other clients will block while one client
// performs an update. The read lock is released by the client calling
@ -1475,7 +1460,6 @@ public class HRegion implements HeapSize { // , Writable{
long newSize;
startRegionOperation();
this.writeRequestsCount.increment();
try {
long addedSize = doMiniBatchPut(batchOp);
newSize = memstoreSize.addAndGet(addedSize);
@ -1677,7 +1661,6 @@ public class HRegion implements HeapSize { // , Writable{
}
startRegionOperation();
this.writeRequestsCount.increment();
try {
RowLock lock = isPut ? ((Put)w).getRowLock() : ((Delete)w).getRowLock();
Get get = new Get(row, lock);
@ -2246,7 +2229,6 @@ public class HRegion implements HeapSize { // , Writable{
*/
public Integer obtainRowLock(final byte [] row) throws IOException {
startRegionOperation();
this.writeRequestsCount.increment();
try {
return internalObtainRowLock(row, true);
} finally {
@ -2254,6 +2236,21 @@ public class HRegion implements HeapSize { // , Writable{
}
}
/**
* Tries to obtain a row lock on the given row, but does not block if the
* row lock is not available. If the lock is not available, returns false.
* Otherwise behaves the same as the above method.
* @see HRegion#obtainRowLock(byte[])
*/
public Integer tryObtainRowLock(final byte[] row) throws IOException {
startRegionOperation();
try {
return internalObtainRowLock(row, false);
} finally {
closeRegionOperation();
}
}
/**
* Obtains or tries to obtain the given row lock.
* @param waitForLock if true, will block until the lock is available.
@ -2365,7 +2362,6 @@ public class HRegion implements HeapSize { // , Writable{
public void bulkLoadHFile(String hfilePath, byte[] familyName)
throws IOException {
startRegionOperation();
this.writeRequestsCount.increment();
try {
Store store = getStore(familyName);
if (store == null) {
@ -2471,7 +2467,6 @@ public class HRegion implements HeapSize { // , Writable{
"or a lengthy garbage collection");
}
startRegionOperation();
readRequestsCount.increment();
try {
// This could be a new thread from the last time we called next().
@ -2978,8 +2973,7 @@ public class HRegion implements HeapSize { // , Writable{
listPaths(fs, newRegionDir);
}
HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf, newRegionInfo, null);
dstRegion.readRequestsCount.set(a.readRequestsCount.get() + b.readRequestsCount.get());
dstRegion.writeRequestsCount.set(a.writeRequestsCount.get() + b.writeRequestsCount.get());
dstRegion.requestsCount.set(a.requestsCount.get() + b.requestsCount.get());
dstRegion.initialize();
dstRegion.compactStores();
if (LOG.isDebugEnabled()) {
@ -3225,7 +3219,6 @@ public class HRegion implements HeapSize { // , Writable{
// Lock row
startRegionOperation();
this.writeRequestsCount.increment();
try {
Integer lid = getLock(lockid, row, true);
this.updatesLock.readLock().lock();
@ -3316,7 +3309,6 @@ public class HRegion implements HeapSize { // , Writable{
// Lock row
long result = amount;
startRegionOperation();
this.writeRequestsCount.increment();
try {
Integer lid = obtainRowLock(row);
this.updatesLock.readLock().lock();
@ -3391,7 +3383,7 @@ public class HRegion implements HeapSize { // , Writable{
public static final long FIXED_OVERHEAD = ClassSize.align(
(4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
ClassSize.align(26 * ClassSize.REFERENCE) + ClassSize.OBJECT +
ClassSize.align(25 * ClassSize.REFERENCE) + ClassSize.OBJECT +
ClassSize.align(Bytes.SIZEOF_INT));
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
@ -3571,6 +3563,7 @@ public class HRegion implements HeapSize { // , Writable{
} finally {
scanner.close();
}
// System.out.println(region.getClosestRowBefore(Bytes.toBytes("GeneratedCSVContent2,E3652782193BC8D66A0BA1629D0FAAAB,9993372036854775807")));
}
} finally {
region.close();
@ -3654,6 +3647,7 @@ public class HRegion implements HeapSize { // , Writable{
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closed");
}
this.requestsCount.increment();
}
/**

15
src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HMsg;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
@ -73,7 +74,6 @@ import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.UnknownRowLockException;
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.YouAreDeadException;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.catalog.RootLocationEditor;
@ -897,6 +897,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
int storefileIndexSizeMB = 0;
long requestsCount = r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Store store : r.stores.values()) {
@ -906,8 +907,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
}
}
return new HServerLoad.RegionLoad(name,stores, storefiles,
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB,
r.readRequestsCount.get(), r.writeRequestsCount.get());
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount);
}
/**
@ -1142,14 +1142,12 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
int stores = 0;
int storefiles = 0;
long memstoreSize = 0;
int readRequestsCount = 0;
int writeRequestsCount = 0;
long requestsCount = 0;
long storefileIndexSize = 0;
for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
HRegion r = e.getValue();
memstoreSize += r.memstoreSize.get();
readRequestsCount += r.readRequestsCount.get();
writeRequestsCount += r.writeRequestsCount.get();
requestsCount += r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Map.Entry<byte[], Store> ee : r.stores.entrySet()) {
@ -1162,8 +1160,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
this.metrics.stores.set(stores);
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
this.metrics.readRequestsCount.set(readRequestsCount);
this.metrics.writeRequestsCount.set(writeRequestsCount);
this.metrics.requestsCount.set(requestsCount);
this.metrics.storefileIndexSizeMB
.set((int) (storefileIndexSize / (1024 * 1024)));
this.metrics.compactionQueueSize.set(compactSplitThread

3
src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
View File

@ -543,8 +543,7 @@ public class SplitTransaction {
HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
this.parent.getLog(), fs, this.parent.getConf(),
hri, rsServices);
r.readRequestsCount.set(this.parent.getReadRequestsCount() / 2);
r.writeRequestsCount.set(this.parent.getWriteRequestsCount() / 2);
r.requestsCount.set(this.parent.getRequestsCount() / 2);
HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
return r;
}

18
src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
View File

@ -128,14 +128,9 @@ public class RegionServerMetrics implements Updater {
public final MetricsIntValue storefiles = new MetricsIntValue("storefiles", registry);
/**
* Count of read requests
* Count of requests
*/
public final MetricsLongValue readRequestsCount = new MetricsLongValue("readRequestsCount", registry);
/**
* Count of write requests
*/
public final MetricsLongValue writeRequestsCount = new MetricsLongValue("writeRequestsCount", registry);
public final MetricsLongValue requestsCount = new MetricsLongValue("requestsCount", registry);
/**
* Sum of all the storefile index sizes in this regionserver in MB
@ -253,8 +248,7 @@ public class RegionServerMetrics implements Updater {
this.storefiles.pushMetric(this.metricsRecord);
this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
this.memstoreSizeMB.pushMetric(this.metricsRecord);
this.readRequestsCount.pushMetric(this.metricsRecord);
this.writeRequestsCount.pushMetric(this.metricsRecord);
this.requestsCount.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
this.requests.pushMetric(this.metricsRecord);
this.compactionQueueSize.pushMetric(this.metricsRecord);
@ -357,10 +351,8 @@ public class RegionServerMetrics implements Updater {
Integer.valueOf(this.storefileIndexSizeMB.get()));
sb = Strings.appendKeyValue(sb, "memstoreSize",
Integer.valueOf(this.memstoreSizeMB.get()));
sb = Strings.appendKeyValue(sb, "readRequestsCount",
Long.valueOf(this.readRequestsCount.get()));
sb = Strings.appendKeyValue(sb, "writeRequestsCount",
Long.valueOf(this.writeRequestsCount.get()));
sb = Strings.appendKeyValue(sb, "requestsCount",
Long.valueOf(this.requestsCount.get()));
sb = Strings.appendKeyValue(sb, "compactionQueueSize",
Integer.valueOf(this.compactionQueueSize.get()));
sb = Strings.appendKeyValue(sb, "flushQueueSize",