From 371f96335952c3ec957c56bd268fceccc16f3586 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Fri, 1 Apr 2011 23:18:24 +0000 Subject: [PATCH] HBASE-3647 Distinguish read and write request count in region -- reversed the patch because missing a file git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1087930 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/hbase/HServerInfo.java | 11 +-- .../org/apache/hadoop/hbase/HServerLoad.java | 73 +++++-------------- .../hadoop/hbase/regionserver/HRegion.java | 48 ++++++------ .../hbase/regionserver/HRegionServer.java | 15 ++-- .../hbase/regionserver/SplitTransaction.java | 3 +- .../metrics/RegionServerMetrics.java | 18 ++--- 6 files changed, 51 insertions(+), 117 deletions(-) diff --git a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java index 7aa8bd9ed3a..c742951be22 100644 --- a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java +++ b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java @@ -27,7 +27,6 @@ import java.util.Comparator; import java.util.Set; import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.io.VersionedWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; @@ -44,10 +43,7 @@ import org.apache.hadoop.io.WritableComparable; * by. In subsequent communications, the regionserver will pass a HServerInfo * with the master-supplied address. */ -public class HServerInfo extends VersionedWritable - implements WritableComparable { - private static final byte VERSION = 0; - +public class HServerInfo implements WritableComparable { /* * This character is used as separator between server hostname and port and * its startcode. Servername is formatted as @@ -65,11 +61,6 @@ public class HServerInfo extends VersionedWritable private String hostname; private String cachedHostnamePort = null; - /** @return the object version number */ - public byte getVersion() { - return VERSION; - } - public HServerInfo() { this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT, "default name"); diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java index 23720537389..561a000df0e 100644 --- a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java +++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java @@ -28,17 +28,13 @@ import java.util.TreeMap; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Strings; -import org.apache.hadoop.io.VersionedWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; /** * This class encapsulates metrics for determining the load on a HRegionServer */ -public class HServerLoad extends VersionedWritable - implements WritableComparable { - private static final byte VERSION = 0; - +public class HServerLoad implements WritableComparable { /** number of regions */ // could just use regionLoad.size() but master.RegionManager likes to play // around with this value while passing HServerLoad objects around during @@ -53,11 +49,6 @@ public class HServerLoad extends VersionedWritable /** per-region load metrics */ private Map regionLoad = new TreeMap(Bytes.BYTES_COMPARATOR); - /** @return the object version number */ - public byte getVersion() { - return VERSION; - } - /** * Encapsulates per-region loading metrics. */ @@ -74,10 +65,8 @@ public class HServerLoad extends VersionedWritable private int memstoreSizeMB; /** the current total size of storefile indexes for the region, in MB */ private int storefileIndexSizeMB; - /** the current total read requests made to region */ - private int readRequestsCount; - /** the current total write requests made to region */ - private int writeRequestsCount; + /** the current total request made to region */ + private long requestsCount; /** * Constructor, for Writable @@ -93,21 +82,18 @@ public class HServerLoad extends VersionedWritable * @param storefileSizeMB * @param memstoreSizeMB * @param storefileIndexSizeMB - * @param readRequestsCount - * @param writeRequestsCount + * @param requestsCount */ public RegionLoad(final byte[] name, final int stores, final int storefiles, final int storefileSizeMB, - final int memstoreSizeMB, final int storefileIndexSizeMB, - final int readRequestsCount, final int writeRequestsCount) { + final int memstoreSizeMB, final int storefileIndexSizeMB,final long requestsCount) { this.name = name; this.stores = stores; this.storefiles = storefiles; this.storefileSizeMB = storefileSizeMB; this.memstoreSizeMB = memstoreSizeMB; this.storefileIndexSizeMB = storefileIndexSizeMB; - this.readRequestsCount = readRequestsCount; - this.writeRequestsCount = writeRequestsCount; + this.requestsCount = requestsCount; } // Getters @@ -160,26 +146,12 @@ public class HServerLoad extends VersionedWritable public int getStorefileIndexSizeMB() { return storefileIndexSizeMB; } - + /** * @return the number of requests made to region */ public long getRequestsCount() { - return readRequestsCount + writeRequestsCount; - } - - /** - * @return the number of read requests made to region - */ - public long getReadRequestsCount() { - return readRequestsCount; - } - - /** - * @return the number of read requests made to region - */ - public long getWriteRequestsCount() { - return writeRequestsCount; + return requestsCount; } // Setters @@ -221,17 +193,10 @@ public class HServerLoad extends VersionedWritable } /** - * @param requestsCount the number of read requests to region + * @param requestsCount the number of requests to region */ - public void setReadRequestsCount(int requestsCount) { - this.readRequestsCount = requestsCount; - } - - /** - * @param requestsCount the number of write requests to region - */ - public void setWriteRequestsCount(int requestsCount) { - this.writeRequestsCount = requestsCount; + public void setRequestsCount(long requestsCount) { + this.requestsCount = requestsCount; } // Writable @@ -244,8 +209,7 @@ public class HServerLoad extends VersionedWritable this.storefileSizeMB = in.readInt(); this.memstoreSizeMB = in.readInt(); this.storefileIndexSizeMB = in.readInt(); - this.readRequestsCount = in.readInt(); - this.writeRequestsCount = in.readInt(); + this.requestsCount = in.readLong(); } public void write(DataOutput out) throws IOException { @@ -256,8 +220,7 @@ public class HServerLoad extends VersionedWritable out.writeInt(storefileSizeMB); out.writeInt(memstoreSizeMB); out.writeInt(storefileIndexSizeMB); - out.writeInt(readRequestsCount); - out.writeInt(writeRequestsCount); + out.writeLong(requestsCount); } /** @@ -275,10 +238,8 @@ public class HServerLoad extends VersionedWritable Integer.valueOf(this.memstoreSizeMB)); sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB", Integer.valueOf(this.storefileIndexSizeMB)); - sb = Strings.appendKeyValue(sb, "readRequestsCount", - Long.valueOf(this.readRequestsCount)); - sb = Strings.appendKeyValue(sb, "writeRequestsCount", - Long.valueOf(this.writeRequestsCount)); + sb = Strings.appendKeyValue(sb, "requestsCount", + Long.valueOf(this.requestsCount)); return sb.toString(); } } @@ -522,9 +483,9 @@ public class HServerLoad extends VersionedWritable public void addRegionInfo(final byte[] name, final int stores, final int storefiles, final int storefileSizeMB, final int memstoreSizeMB, final int storefileIndexSizeMB, - final int readRequestsCount, final int writeRequestsCount) { + final long requestsCount) { this.regionLoad.put(name, new HServerLoad.RegionLoad(name, stores, storefiles, - storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, readRequestsCount, writeRequestsCount)); + storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount)); } // Writable diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 5e4f367c1c3..b91e5fae885 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -177,8 +177,7 @@ public class HRegion implements HeapSize { // , Writable{ final AtomicLong memstoreSize = new AtomicLong(0); - final Counter readRequestsCount = new Counter(); - final Counter writeRequestsCount = new Counter(); + final Counter requestsCount = new Counter(); /** * The directory for the table this region is part of. @@ -463,17 +462,7 @@ public class HRegion implements HeapSize { // , Writable{ /** @return requestsCount for this region */ public long getRequestsCount() { - return this.readRequestsCount.get() + this.writeRequestsCount.get(); - } - - /** @return readRequestsCount for this region */ - public long getReadRequestsCount() { - return this.readRequestsCount.get(); - } - - /** @return writeRequestsCount for this region */ - public long getWriteRequestsCount() { - return this.writeRequestsCount.get(); + return this.requestsCount.get(); } /** @return true if region is closed */ @@ -1142,7 +1131,6 @@ public class HRegion implements HeapSize { // , Writable{ // closest key is across all column families, since the data may be sparse checkRow(row); startRegionOperation(); - this.readRequestsCount.increment(); try { Store store = getStore(family); KeyValue kv = new KeyValue(row, HConstants.LATEST_TIMESTAMP); @@ -1179,7 +1167,6 @@ public class HRegion implements HeapSize { // , Writable{ protected InternalScanner getScanner(Scan scan, List additionalScanners) throws IOException { startRegionOperation(); - this.readRequestsCount.increment(); try { // Verify families are all valid if(scan.hasFamilies()) { @@ -1248,7 +1235,6 @@ public class HRegion implements HeapSize { // , Writable{ checkResources(); Integer lid = null; startRegionOperation(); - this.writeRequestsCount.increment(); try { byte [] row = delete.getRow(); // If we did not pass an existing row lock, obtain a new one @@ -1402,7 +1388,6 @@ public class HRegion implements HeapSize { // , Writable{ // will be extremely rare; we'll deal with it when it happens. checkResources(); startRegionOperation(); - this.writeRequestsCount.increment(); try { // We obtain a per-row lock, so other clients will block while one client // performs an update. The read lock is released by the client calling @@ -1475,7 +1460,6 @@ public class HRegion implements HeapSize { // , Writable{ long newSize; startRegionOperation(); - this.writeRequestsCount.increment(); try { long addedSize = doMiniBatchPut(batchOp); newSize = memstoreSize.addAndGet(addedSize); @@ -1677,7 +1661,6 @@ public class HRegion implements HeapSize { // , Writable{ } startRegionOperation(); - this.writeRequestsCount.increment(); try { RowLock lock = isPut ? ((Put)w).getRowLock() : ((Delete)w).getRowLock(); Get get = new Get(row, lock); @@ -2246,7 +2229,6 @@ public class HRegion implements HeapSize { // , Writable{ */ public Integer obtainRowLock(final byte [] row) throws IOException { startRegionOperation(); - this.writeRequestsCount.increment(); try { return internalObtainRowLock(row, true); } finally { @@ -2254,6 +2236,21 @@ public class HRegion implements HeapSize { // , Writable{ } } + /** + * Tries to obtain a row lock on the given row, but does not block if the + * row lock is not available. If the lock is not available, returns false. + * Otherwise behaves the same as the above method. + * @see HRegion#obtainRowLock(byte[]) + */ + public Integer tryObtainRowLock(final byte[] row) throws IOException { + startRegionOperation(); + try { + return internalObtainRowLock(row, false); + } finally { + closeRegionOperation(); + } + } + /** * Obtains or tries to obtain the given row lock. * @param waitForLock if true, will block until the lock is available. @@ -2365,7 +2362,6 @@ public class HRegion implements HeapSize { // , Writable{ public void bulkLoadHFile(String hfilePath, byte[] familyName) throws IOException { startRegionOperation(); - this.writeRequestsCount.increment(); try { Store store = getStore(familyName); if (store == null) { @@ -2471,7 +2467,6 @@ public class HRegion implements HeapSize { // , Writable{ "or a lengthy garbage collection"); } startRegionOperation(); - readRequestsCount.increment(); try { // This could be a new thread from the last time we called next(). @@ -2978,8 +2973,7 @@ public class HRegion implements HeapSize { // , Writable{ listPaths(fs, newRegionDir); } HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf, newRegionInfo, null); - dstRegion.readRequestsCount.set(a.readRequestsCount.get() + b.readRequestsCount.get()); - dstRegion.writeRequestsCount.set(a.writeRequestsCount.get() + b.writeRequestsCount.get()); + dstRegion.requestsCount.set(a.requestsCount.get() + b.requestsCount.get()); dstRegion.initialize(); dstRegion.compactStores(); if (LOG.isDebugEnabled()) { @@ -3225,7 +3219,6 @@ public class HRegion implements HeapSize { // , Writable{ // Lock row startRegionOperation(); - this.writeRequestsCount.increment(); try { Integer lid = getLock(lockid, row, true); this.updatesLock.readLock().lock(); @@ -3316,7 +3309,6 @@ public class HRegion implements HeapSize { // , Writable{ // Lock row long result = amount; startRegionOperation(); - this.writeRequestsCount.increment(); try { Integer lid = obtainRowLock(row); this.updatesLock.readLock().lock(); @@ -3391,7 +3383,7 @@ public class HRegion implements HeapSize { // , Writable{ public static final long FIXED_OVERHEAD = ClassSize.align( (4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY + - ClassSize.align(26 * ClassSize.REFERENCE) + ClassSize.OBJECT + + ClassSize.align(25 * ClassSize.REFERENCE) + ClassSize.OBJECT + ClassSize.align(Bytes.SIZEOF_INT)); public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + @@ -3571,6 +3563,7 @@ public class HRegion implements HeapSize { // , Writable{ } finally { scanner.close(); } + // System.out.println(region.getClosestRowBefore(Bytes.toBytes("GeneratedCSVContent2,E3652782193BC8D66A0BA1629D0FAAAB,9993372036854775807"))); } } finally { region.close(); @@ -3654,6 +3647,7 @@ public class HRegion implements HeapSize { // , Writable{ throw new NotServingRegionException(regionInfo.getRegionNameAsString() + " is closed"); } + this.requestsCount.increment(); } /** diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 9de6d4b1ecf..99d0f0fbb49 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; @@ -73,7 +74,6 @@ import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.UnknownRowLockException; import org.apache.hadoop.hbase.UnknownScannerException; import org.apache.hadoop.hbase.YouAreDeadException; -import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.catalog.RootLocationEditor; @@ -897,6 +897,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024); int storefileIndexSizeMB = 0; + long requestsCount = r.requestsCount.get(); synchronized (r.stores) { stores += r.stores.size(); for (Store store : r.stores.values()) { @@ -906,8 +907,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } } return new HServerLoad.RegionLoad(name,stores, storefiles, - storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, - r.readRequestsCount.get(), r.writeRequestsCount.get()); + storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount); } /** @@ -1142,14 +1142,12 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, int stores = 0; int storefiles = 0; long memstoreSize = 0; - int readRequestsCount = 0; - int writeRequestsCount = 0; + long requestsCount = 0; long storefileIndexSize = 0; for (Map.Entry e : this.onlineRegions.entrySet()) { HRegion r = e.getValue(); memstoreSize += r.memstoreSize.get(); - readRequestsCount += r.readRequestsCount.get(); - writeRequestsCount += r.writeRequestsCount.get(); + requestsCount += r.requestsCount.get(); synchronized (r.stores) { stores += r.stores.size(); for (Map.Entry ee : r.stores.entrySet()) { @@ -1162,8 +1160,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, this.metrics.stores.set(stores); this.metrics.storefiles.set(storefiles); this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024))); - this.metrics.readRequestsCount.set(readRequestsCount); - this.metrics.writeRequestsCount.set(writeRequestsCount); + this.metrics.requestsCount.set(requestsCount); this.metrics.storefileIndexSizeMB .set((int) (storefileIndexSize / (1024 * 1024))); this.metrics.compactionQueueSize.set(compactSplitThread diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java index 1b521f66581..f8684e2554c 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java @@ -543,8 +543,7 @@ public class SplitTransaction { HRegion r = HRegion.newHRegion(this.parent.getTableDir(), this.parent.getLog(), fs, this.parent.getConf(), hri, rsServices); - r.readRequestsCount.set(this.parent.getReadRequestsCount() / 2); - r.writeRequestsCount.set(this.parent.getWriteRequestsCount() / 2); + r.requestsCount.set(this.parent.getRequestsCount() / 2); HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir()); return r; } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java index 345e39373bd..354ef4a0366 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java @@ -128,14 +128,9 @@ public class RegionServerMetrics implements Updater { public final MetricsIntValue storefiles = new MetricsIntValue("storefiles", registry); /** - * Count of read requests + * Count of requests */ - public final MetricsLongValue readRequestsCount = new MetricsLongValue("readRequestsCount", registry); - - /** - * Count of write requests - */ - public final MetricsLongValue writeRequestsCount = new MetricsLongValue("writeRequestsCount", registry); + public final MetricsLongValue requestsCount = new MetricsLongValue("requestsCount", registry); /** * Sum of all the storefile index sizes in this regionserver in MB @@ -253,8 +248,7 @@ public class RegionServerMetrics implements Updater { this.storefiles.pushMetric(this.metricsRecord); this.storefileIndexSizeMB.pushMetric(this.metricsRecord); this.memstoreSizeMB.pushMetric(this.metricsRecord); - this.readRequestsCount.pushMetric(this.metricsRecord); - this.writeRequestsCount.pushMetric(this.metricsRecord); + this.requestsCount.pushMetric(this.metricsRecord); this.regions.pushMetric(this.metricsRecord); this.requests.pushMetric(this.metricsRecord); this.compactionQueueSize.pushMetric(this.metricsRecord); @@ -357,10 +351,8 @@ public class RegionServerMetrics implements Updater { Integer.valueOf(this.storefileIndexSizeMB.get())); sb = Strings.appendKeyValue(sb, "memstoreSize", Integer.valueOf(this.memstoreSizeMB.get())); - sb = Strings.appendKeyValue(sb, "readRequestsCount", - Long.valueOf(this.readRequestsCount.get())); - sb = Strings.appendKeyValue(sb, "writeRequestsCount", - Long.valueOf(this.writeRequestsCount.get())); + sb = Strings.appendKeyValue(sb, "requestsCount", + Long.valueOf(this.requestsCount.get())); sb = Strings.appendKeyValue(sb, "compactionQueueSize", Integer.valueOf(this.compactionQueueSize.get())); sb = Strings.appendKeyValue(sb, "flushQueueSize",