From 8fb6662aea193972c1608cc40005dbe3932874c5 Mon Sep 17 00:00:00 2001 From: Andrew Kyle Purtell Date: Fri, 19 Jun 2009 22:42:11 +0000 Subject: [PATCH] HBASE-1553 ClassSize missing in trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@786695 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../apache/hadoop/hbase/util/ClassSize.java | 195 ++++++++++++++++++ .../apache/hadoop/hbase/io/TestHeapSize.java | 72 +++++++ 3 files changed, 268 insertions(+) create mode 100755 src/java/org/apache/hadoop/hbase/util/ClassSize.java create mode 100644 src/test/org/apache/hadoop/hbase/io/TestHeapSize.java diff --git a/CHANGES.txt b/CHANGES.txt index 5cef312f55e..15bcb5f3948 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -210,6 +210,7 @@ Release 0.20.0 - Unreleased JVM trick (Erik Holstad via Stack) HBASE-1545 atomicIncrements creating new values with Long.MAX_VALUE HBASE-1547 atomicIncrement doesnt increase hregion.memcacheSize + HBASE-1553 ClassSize missing in trunk IMPROVEMENTS HBASE-1089 Add count of regions on filesystem to master UI; add percentage diff --git a/src/java/org/apache/hadoop/hbase/util/ClassSize.java b/src/java/org/apache/hadoop/hbase/util/ClassSize.java new file mode 100755 index 00000000000..4f964185e3f --- /dev/null +++ b/src/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -0,0 +1,195 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.io.HeapSize; + +/** + * Class for determining the "size" of a class, an attempt to calculate the + * actual bytes that an object of this class will occupy in memory + * + * The core of this class is taken from the Derby project + */ +public class ClassSize { + static final Log LOG = LogFactory.getLog(ClassSize.class); + + private int refSize; + private int minObjectSize; + + /** + * Constructor + * @throws Exception + */ + public ClassSize() throws Exception{ + // Figure out whether this is a 32 or 64 bit machine. + Runtime runtime = Runtime.getRuntime(); + int loops = 10; + int sz = 0; + for(int i = 0; i < loops; i++) { + cleaner(runtime, i); + long memBase = runtime.totalMemory() - runtime.freeMemory(); + Object[] junk = new Object[10000]; + cleaner(runtime, i); + long memUsed = runtime.totalMemory() - runtime.freeMemory() - memBase; + sz = (int)((memUsed + junk.length/2)/junk.length); + if(sz > 0 ) { + break; + } + } + + refSize = ( 4 > sz) ? 4 : sz; + minObjectSize = 4*refSize; + } + + /** + * The estimate of the size of a class instance depends on whether the JVM + * uses 32 or 64 bit addresses, that is it depends on the size of an object + * reference. It is a linear function of the size of a reference, e.g. + * 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes). + * + * This method returns the coefficients of the linear function, e.g. {24, 5} + * in the above example. + * + * @param cl A class whose instance size is to be estimated + * @return an array of 3 integers. The first integer is the size of the + * primitives, the second the number of arrays and the third the number of + * references. + */ + private int [] getSizeCoefficients(Class cl, boolean debug) { + int primitives = 0; + int arrays = 0; + int references = HeapSize.OBJECT / HeapSize.REFERENCE; + + for( ; null != cl; cl = cl.getSuperclass()) { + Field[] field = cl.getDeclaredFields(); + if( null != field) { + for( int i = 0; i < field.length; i++) { + if( ! Modifier.isStatic( field[i].getModifiers())) { + Class fieldClass = field[i].getType(); + if( fieldClass.isArray()){ + arrays++; + } + else if(! fieldClass.isPrimitive()){ + references++; + } + else {// Is simple primitive + String name = fieldClass.getName(); + + if(name.equals("int") || name.equals( "I")) + primitives += Bytes.SIZEOF_INT; + else if(name.equals("long") || name.equals( "J")) + primitives += Bytes.SIZEOF_LONG; + else if(name.equals("boolean") || name.equals( "Z")) + primitives += Bytes.SIZEOF_BOOLEAN; + else if(name.equals("short") || name.equals( "S")) + primitives += Bytes.SIZEOF_SHORT; + else if(name.equals("byte") || name.equals( "B")) + primitives += Bytes.SIZEOF_BYTE; + else if(name.equals("char") || name.equals( "C")) + primitives += Bytes.SIZEOF_CHAR; + else if(name.equals("float") || name.equals( "F")) + primitives += Bytes.SIZEOF_FLOAT; + else if(name.equals("double") || name.equals( "D")) + primitives += Bytes.SIZEOF_DOUBLE; + } + if(debug) { + if (LOG.isDebugEnabled()) { + // Write out region name as string and its encoded name. + LOG.debug(field[i].getName()+ "\n\t" +field[i].getType()); + } + } + } + } + } + } + return new int [] {primitives, arrays, references}; + } + + /** + * Estimate the static space taken up by a class instance given the + * coefficients returned by getSizeCoefficients. + * + * @param coeff the coefficients + * + * @return the size estimate, in bytes + */ + private long estimateBaseFromCoefficients(int [] coeff, boolean debug) { + int size = coeff[0] + (coeff[1]*4 + coeff[2])*refSize; + + // Round up to a multiple of 8 + size = (int)alignSize(size); + if(debug) { + if (LOG.isDebugEnabled()) { + // Write out region name as string and its encoded name. + LOG.debug("Primitives " + coeff[0] + ", arrays " + coeff[1] + + ", references(inlcuding " + HeapSize.OBJECT + + ", for object overhead) " + coeff[2] + ", refSize " + refSize + + ", size " + size); + } + } + return (size < minObjectSize) ? minObjectSize : size; + } + + /** + * Estimate the static space taken up by the fields of a class. This includes + * the space taken up by by references (the pointer) but not by the referenced + * object. So the estimated size of an array field does not depend on the size + * of the array. Similarly the size of an object (reference) field does not + * depend on the object. + * + * @return the size estimate in bytes. + */ + public long estimateBase(Class cl, boolean debug) { + return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug); + } + + /** + * Tries to clear all the memory used to estimate the reference size for the + * current JVM + * @param runtime + * @param i + * @throws Exception + */ + private void cleaner(Runtime runtime, int i) throws Exception{ + Thread.sleep(i*1000); + runtime.gc();runtime.gc(); runtime.gc();runtime.gc();runtime.gc(); + runtime.runFinalization(); + } + + + /** + * Aligns a number to 8. + * @param num number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public static long alignSize(int num) { + int aligned = (num + 7)/8; + aligned *= 8; + return aligned; + } + +} + diff --git a/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java b/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java new file mode 100644 index 00000000000..d74fcbbb53a --- /dev/null +++ b/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -0,0 +1,72 @@ +package org.apache.hadoop.hbase.io; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; + +import junit.framework.TestCase; + +/** + * Testing the sizing that HeapSize offers and compares to the size given by + * ClassSize. + */ +public class TestHeapSize extends TestCase { + static final Log LOG = LogFactory.getLog(TestHeapSize.class); + // List of classes implementing HeapSize + // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry, HStoreKey + // KeyValue, LruBlockCache, LruHashMap, Put, HLogKey + + /** + * Testing the classes that implements HeapSize and are a part of 0.20. + * Some are not tested here for example BlockIndex which is tested in + * TestHFile since it is a non public class + */ + public void testSizes() { + ClassSize cs = null; + Class cl = null; + long expected = 0L; + long actual = 0L; + try { + cs = new ClassSize(); + } catch(Exception e) {} + + //KeyValue + cl = KeyValue.class; + expected = cs.estimateBase(cl, false); + KeyValue kv = new KeyValue(); + actual = kv.heapSize(); + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + + //LruBlockCache + cl = LruBlockCache.class; + expected = cs.estimateBase(cl, false); + LruBlockCache c = new LruBlockCache(1,1,200); + //Since minimum size for the for a LruBlockCache is 1 + //we need to remove one reference from the heapsize + actual = c.heapSize() - HeapSize.REFERENCE; + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + + //Put + cl = Put.class; + expected = cs.estimateBase(cl, false); + //The actual TreeMap is not included in the above calculation + expected += HeapSize.TREEMAP_SIZE; + Put put = new Put(Bytes.toBytes("")); + actual = put.heapSize(); + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + } + +}