HBASE-953 Enable BLOCKCACHE by default [WAS -> Reevaluate HBASE-288 block caching work....?]
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@708253 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2ee2b85eac
commit
7e1d49358d
|
@ -7,6 +7,8 @@ Release 0.19.0 - Unreleased
|
|||
Jim Kellerman)
|
||||
HBASE-852 Cannot scan all families in a row with a LIMIT, STARTROW, etc.
|
||||
(Izaak Rubin via Stack)
|
||||
HBASE-953 Enable BLOCKCACHE by default [WAS -> Reevaluate HBASE-288 block
|
||||
caching work....?] -- Update your hbad-default.xml file!
|
||||
|
||||
BUG FIXES
|
||||
HBASE-891 HRS.validateValuesLength throws IOE, gets caught in the retries
|
||||
|
|
|
@ -305,8 +305,12 @@
|
|||
</property>
|
||||
<property>
|
||||
<name>hbase.hstore.blockCache.blockSize</name>
|
||||
<value>65536</value>
|
||||
<description>The size of each block in any block caches.
|
||||
<value>16384</value>
|
||||
<description>The size of each block in the block cache.
|
||||
Enable blockcaching on a per column family basis; see the BLOCKCACHE setting
|
||||
in HColumnDescriptor. Blocks are kept in a java Soft Reference cache so are
|
||||
let go when high pressure on memory. Block caching is enabled by default
|
||||
as of hbase 0.19.0.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
|
|
Binary file not shown.
|
@ -95,7 +95,7 @@ public class HColumnDescriptor implements WritableComparable {
|
|||
/**
|
||||
* Default setting for whether to use a block cache or not.
|
||||
*/
|
||||
public static final boolean DEFAULT_BLOCKCACHE = false;
|
||||
public static final boolean DEFAULT_BLOCKCACHE = true;
|
||||
|
||||
/**
|
||||
* Default setting for whether or not to use bloomfilters.
|
||||
|
@ -567,4 +567,4 @@ public class HColumnDescriptor implements WritableComparable {
|
|||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -50,7 +50,7 @@ import org.apache.hadoop.hbase.ipc.HMasterInterface;
|
|||
import org.apache.hadoop.hbase.ipc.HRegionInterface;
|
||||
import org.apache.hadoop.hbase.ipc.HbaseRPC;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.SoftSortedMap;
|
||||
import org.apache.hadoop.hbase.util.SoftValueSortedMap;
|
||||
import org.apache.hadoop.hbase.util.Writables;
|
||||
import org.apache.hadoop.ipc.RemoteException;
|
||||
|
||||
|
@ -64,7 +64,9 @@ public class HConnectionManager implements HConstants {
|
|||
/*
|
||||
* Not instantiable.
|
||||
*/
|
||||
protected HConnectionManager() {}
|
||||
protected HConnectionManager() {
|
||||
super();
|
||||
}
|
||||
|
||||
// A Map of master HServerAddress -> connection information for that instance
|
||||
// Note that although the Map is synchronized, the objects it contains
|
||||
|
@ -92,7 +94,7 @@ public class HConnectionManager implements HConstants {
|
|||
}
|
||||
|
||||
/**
|
||||
* Delete connection information for the instance specified by the configuration
|
||||
* Delete connection information for the instance specified by configuration
|
||||
* @param conf
|
||||
* @param stopProxy
|
||||
*/
|
||||
|
@ -131,9 +133,9 @@ public class HConnectionManager implements HConstants {
|
|||
|
||||
private HRegionLocation rootRegionLocation;
|
||||
|
||||
private final Map<Integer, SoftSortedMap<byte [], HRegionLocation>>
|
||||
private final Map<Integer, SoftValueSortedMap<byte [], HRegionLocation>>
|
||||
cachedRegionLocations = Collections.synchronizedMap(
|
||||
new HashMap<Integer, SoftSortedMap<byte [], HRegionLocation>>());
|
||||
new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>());
|
||||
|
||||
/**
|
||||
* constructor
|
||||
|
@ -207,8 +209,7 @@ public class HConnectionManager implements HConstants {
|
|||
getPauseTime(tries));
|
||||
}
|
||||
|
||||
// We either cannot connect to master or it is not running. Sleep & retry
|
||||
|
||||
// Cannot connect to master or it is not running. Sleep & retry
|
||||
try {
|
||||
Thread.sleep(getPauseTime(tries));
|
||||
} catch (InterruptedException e) {
|
||||
|
@ -320,7 +321,8 @@ public class HConnectionManager implements HConstants {
|
|||
if (currentRegion != null) {
|
||||
byte[] endKey = currentRegion.getEndKey();
|
||||
if (endKey == null ||
|
||||
HStoreKey.equalsTwoRowKeys(currentRegion, endKey, HConstants.EMPTY_BYTE_ARRAY)) {
|
||||
HStoreKey.equalsTwoRowKeys(currentRegion, endKey,
|
||||
HConstants.EMPTY_BYTE_ARRAY)) {
|
||||
// We have reached the end of the table and we're done
|
||||
break;
|
||||
}
|
||||
|
@ -580,12 +582,12 @@ public class HConnectionManager implements HConstants {
|
|||
final byte [] row) {
|
||||
// find the map of cached locations for this table
|
||||
Integer key = Bytes.mapKey(tableName);
|
||||
SoftSortedMap<byte [], HRegionLocation> tableLocations =
|
||||
SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
|
||||
cachedRegionLocations.get(key);
|
||||
|
||||
// if tableLocations for this table isn't built yet, make one
|
||||
if (tableLocations == null) {
|
||||
tableLocations = new SoftSortedMap<byte [],
|
||||
tableLocations = new SoftValueSortedMap<byte [],
|
||||
HRegionLocation>(Bytes.BYTES_COMPARATOR);
|
||||
cachedRegionLocations.put(key, tableLocations);
|
||||
}
|
||||
|
@ -611,7 +613,7 @@ public class HConnectionManager implements HConstants {
|
|||
|
||||
// Cut the cache so that we only get the part that could contain
|
||||
// regions that match our key
|
||||
SoftSortedMap<byte[], HRegionLocation> matchingRegions =
|
||||
SoftValueSortedMap<byte[], HRegionLocation> matchingRegions =
|
||||
tableLocations.headMap(row);
|
||||
|
||||
// if that portion of the map is empty, then we're done. otherwise,
|
||||
|
@ -652,13 +654,13 @@ public class HConnectionManager implements HConstants {
|
|||
final byte [] row) {
|
||||
// find the map of cached locations for this table
|
||||
Integer key = Bytes.mapKey(tableName);
|
||||
SoftSortedMap<byte [], HRegionLocation> tableLocations =
|
||||
SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
|
||||
cachedRegionLocations.get(key);
|
||||
|
||||
// if tableLocations for this table isn't built yet, make one
|
||||
if (tableLocations == null) {
|
||||
tableLocations =
|
||||
new SoftSortedMap<byte [], HRegionLocation>(Bytes.BYTES_COMPARATOR);
|
||||
tableLocations = new SoftValueSortedMap<byte [],
|
||||
HRegionLocation>(Bytes.BYTES_COMPARATOR);
|
||||
cachedRegionLocations.put(key, tableLocations);
|
||||
}
|
||||
|
||||
|
@ -667,7 +669,7 @@ public class HConnectionManager implements HConstants {
|
|||
if (!tableLocations.isEmpty()) {
|
||||
// cut the cache so that we only get the part that could contain
|
||||
// regions that match our key
|
||||
SoftSortedMap<byte [], HRegionLocation> matchingRegions =
|
||||
SoftValueSortedMap<byte [], HRegionLocation> matchingRegions =
|
||||
tableLocations.headMap(row);
|
||||
|
||||
// if that portion of the map is empty, then we're done. otherwise,
|
||||
|
@ -704,13 +706,13 @@ public class HConnectionManager implements HConstants {
|
|||
|
||||
// find the map of cached locations for this table
|
||||
Integer key = Bytes.mapKey(tableName);
|
||||
SoftSortedMap<byte [], HRegionLocation> tableLocations =
|
||||
SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
|
||||
cachedRegionLocations.get(key);
|
||||
|
||||
// if tableLocations for this table isn't built yet, make one
|
||||
if (tableLocations == null) {
|
||||
tableLocations =
|
||||
new SoftSortedMap<byte [], HRegionLocation>(Bytes.BYTES_COMPARATOR);
|
||||
tableLocations = new SoftValueSortedMap<byte [],
|
||||
HRegionLocation>(Bytes.BYTES_COMPARATOR);
|
||||
cachedRegionLocations.put(key, tableLocations);
|
||||
}
|
||||
|
||||
|
@ -758,8 +760,8 @@ public class HConnectionManager implements HConstants {
|
|||
/*
|
||||
* Repeatedly try to find the root region by asking the master for where it is
|
||||
* @return HRegionLocation for root region if found
|
||||
* @throws NoServerForRegionException - if the root region can not be located
|
||||
* after retrying
|
||||
* @throws NoServerForRegionException - if the root region can not be
|
||||
* located after retrying
|
||||
* @throws IOException
|
||||
*/
|
||||
private HRegionLocation locateRootRegion()
|
||||
|
@ -856,7 +858,7 @@ public class HConnectionManager implements HConstants {
|
|||
t = t.getCause();
|
||||
}
|
||||
if (t instanceof RemoteException) {
|
||||
t = RemoteExceptionHandler.decodeRemoteException((RemoteException) t);
|
||||
t = RemoteExceptionHandler.decodeRemoteException((RemoteException)t);
|
||||
}
|
||||
if (t instanceof DoNotRetryIOException) {
|
||||
throw (DoNotRetryIOException)t;
|
||||
|
@ -918,6 +920,5 @@ public class HConnectionManager implements HConstants {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
|
@ -23,12 +23,12 @@ import java.io.IOException;
|
|||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.collections.map.ReferenceMap;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.fs.FSInputStream;
|
||||
import org.apache.hadoop.fs.PositionedReadable;
|
||||
import org.apache.hadoop.fs.Seekable;
|
||||
import org.apache.hadoop.hbase.util.SoftValueMap;
|
||||
import org.apache.hadoop.io.DataInputBuffer;
|
||||
|
||||
/**
|
||||
|
@ -69,11 +69,10 @@ public class BlockFSInputStream extends FSInputStream {
|
|||
this.fileLength = fileLength;
|
||||
this.blockSize = blockSize;
|
||||
// a memory-sensitive map that has soft references to values
|
||||
this.blocks = new ReferenceMap() {
|
||||
this.blocks = new SoftValueMap<Long, byte []>() {
|
||||
private long hits, misses;
|
||||
@Override
|
||||
public Object get(Object key) {
|
||||
Object value = super.get(key);
|
||||
public byte [] get(Object key) {
|
||||
byte [] value = super.get(key);
|
||||
if (value == null) {
|
||||
misses++;
|
||||
} else {
|
||||
|
@ -81,7 +80,7 @@ public class BlockFSInputStream extends FSInputStream {
|
|||
}
|
||||
if (LOG.isDebugEnabled() && ((hits + misses) % 10000) == 0) {
|
||||
long hitRate = (100 * hits) / (hits + misses);
|
||||
LOG.info("Hit rate for cache " + hashCode() + ": " + hitRate + "%");
|
||||
LOG.debug("Hit rate for cache " + hashCode() + ": " + hitRate + "%");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.lang.ref.ReferenceQueue;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* java.lang.ref.ReferenceQueue utility class.
|
||||
* @param <K>
|
||||
* @param <V>
|
||||
*/
|
||||
class ReferenceQueueUtil<K,V> {
|
||||
private final Log LOG = LogFactory.getLog(this.getClass());
|
||||
private final ReferenceQueue rq = new ReferenceQueue();
|
||||
private final Map<K,V> map;
|
||||
|
||||
private ReferenceQueueUtil() {
|
||||
this(null);
|
||||
}
|
||||
|
||||
ReferenceQueueUtil(final Map<K,V> m) {
|
||||
super();
|
||||
this.map = m;
|
||||
}
|
||||
|
||||
public ReferenceQueue getReferenceQueue() {
|
||||
return rq;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the reference queue and delete anything that has since gone away
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
void checkReferences() {
|
||||
int i = 0;
|
||||
for (Object obj = null; (obj = this.rq.poll()) != null;) {
|
||||
i++;
|
||||
this.map.remove(((SoftValue<K,V>)obj).getKey());
|
||||
}
|
||||
if (i > 0 && LOG.isDebugEnabled()) {
|
||||
LOG.debug("" + i + " reference(s) cleared.");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,212 +0,0 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.lang.ref.SoftReference;
|
||||
import java.lang.ref.ReferenceQueue;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.Comparator;
|
||||
import java.util.Collection;
|
||||
import java.util.ArrayList;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* A SortedMap implementation that uses SoftReferences internally to make it
|
||||
* play well with the GC when in a low-memory situation.
|
||||
*
|
||||
* @param <K> key class
|
||||
* @param <V> value class
|
||||
*/
|
||||
public class SoftSortedMap<K,V> implements SortedMap<K,V> {
|
||||
private static final Log LOG = LogFactory.getLog(SoftSortedMap.class);
|
||||
private final SortedMap<K, SoftValue<K,V>> internalMap;
|
||||
private ReferenceQueue<K> referenceQueue = new ReferenceQueue<K>();
|
||||
|
||||
/** Constructor */
|
||||
public SoftSortedMap() {
|
||||
this(new TreeMap<K, SoftValue<K,V>>());
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
* @param c
|
||||
*/
|
||||
public SoftSortedMap(final Comparator<K> c) {
|
||||
this(new TreeMap<K, SoftValue<K,V>>(c));
|
||||
}
|
||||
|
||||
/** For headMap and tailMap support */
|
||||
private SoftSortedMap(SortedMap<K,SoftValue<K,V>> original) {
|
||||
internalMap = original;
|
||||
}
|
||||
|
||||
/* Client methods */
|
||||
public V put(K key, V value) {
|
||||
checkReferences();
|
||||
SoftValue<K,V> oldValue =
|
||||
internalMap.put(key, new SoftValue<K,V>(key, value, referenceQueue));
|
||||
return oldValue == null ? null : oldValue.get();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void putAll(@SuppressWarnings("unused") Map map) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
|
||||
public V get(Object key) {
|
||||
checkReferences();
|
||||
SoftValue<K,V> value = internalMap.get(key);
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
if (value.get() == null) {
|
||||
internalMap.remove(key);
|
||||
return null;
|
||||
}
|
||||
return value.get();
|
||||
}
|
||||
|
||||
public V remove(Object key) {
|
||||
checkReferences();
|
||||
SoftValue<K,V> value = internalMap.remove(key);
|
||||
return value == null ? null : value.get();
|
||||
}
|
||||
|
||||
public boolean containsKey(Object key) {
|
||||
checkReferences();
|
||||
return internalMap.containsKey(key);
|
||||
}
|
||||
|
||||
public boolean containsValue(@SuppressWarnings("unused") Object value) {
|
||||
/* checkReferences();
|
||||
return internalMap.containsValue(value);*/
|
||||
throw new UnsupportedOperationException("Don't support containsValue!");
|
||||
}
|
||||
|
||||
public K firstKey() {
|
||||
checkReferences();
|
||||
return internalMap.firstKey();
|
||||
}
|
||||
|
||||
public K lastKey() {
|
||||
checkReferences();
|
||||
return internalMap.lastKey();
|
||||
}
|
||||
|
||||
public SoftSortedMap<K,V> headMap(K key) {
|
||||
checkReferences();
|
||||
return new SoftSortedMap<K,V>(internalMap.headMap(key));
|
||||
}
|
||||
|
||||
public SoftSortedMap<K,V> tailMap(K key) {
|
||||
checkReferences();
|
||||
return new SoftSortedMap<K,V>(internalMap.tailMap(key));
|
||||
}
|
||||
|
||||
public SoftSortedMap<K,V> subMap(K fromKey, K toKey) {
|
||||
checkReferences();
|
||||
return new SoftSortedMap<K,V>(internalMap.subMap(fromKey, toKey));
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
checkReferences();
|
||||
return internalMap.isEmpty();
|
||||
}
|
||||
|
||||
public int size() {
|
||||
checkReferences();
|
||||
return internalMap.size();
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
internalMap.clear();
|
||||
}
|
||||
|
||||
public Set<K> keySet() {
|
||||
return internalMap.keySet();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public Comparator comparator() {
|
||||
return internalMap.comparator();
|
||||
}
|
||||
|
||||
public Set<Map.Entry<K,V>> entrySet() {
|
||||
Set<Map.Entry<K, SoftValue<K,V>>> entries = internalMap.entrySet();
|
||||
Set<Map.Entry<K, V>> real_entries = new TreeSet<Map.Entry<K,V>>();
|
||||
for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
|
||||
real_entries.add(entry.getValue());
|
||||
}
|
||||
return real_entries;
|
||||
}
|
||||
|
||||
public Collection<V> values() {
|
||||
checkReferences();
|
||||
Collection<SoftValue<K,V>> softValues = internalMap.values();
|
||||
ArrayList<V> hardValues = new ArrayList<V>();
|
||||
for(SoftValue<K,V> softValue : softValues) {
|
||||
hardValues.add(softValue.get());
|
||||
}
|
||||
return hardValues;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the reference queue and delete anything that has since gone away
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private void checkReferences() {
|
||||
SoftValue<K,V> sv;
|
||||
Object obj;
|
||||
while((obj = referenceQueue.poll()) != null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
Object k = ((SoftValue<K,V>)obj).key;
|
||||
String name = (k instanceof byte [])? Bytes.toString((byte [])k): k.toString();
|
||||
LOG.debug("Reference for key " + name + " has been cleared.");
|
||||
}
|
||||
internalMap.remove(((SoftValue<K,V>)obj).key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A SoftReference derivative so that we can track down what keys to remove.
|
||||
*/
|
||||
private class SoftValue<K2, V2> extends SoftReference<V2> implements Map.Entry<K2,V2> {
|
||||
K2 key;
|
||||
|
||||
SoftValue(K2 key, V2 value, ReferenceQueue queue) {
|
||||
super(value, queue);
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
public K2 getKey() {return key;}
|
||||
public V2 getValue() {return get();}
|
||||
|
||||
public V2 setValue(@SuppressWarnings("unused") V2 value) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,49 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.lang.ref.ReferenceQueue;
|
||||
import java.lang.ref.SoftReference;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A SoftReference derivative so that we can track down what keys to remove.
|
||||
*/
|
||||
class SoftValue<K, V> extends SoftReference<V> implements Map.Entry<K, V> {
|
||||
private final K key;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
SoftValue(K key, V value, ReferenceQueue queue) {
|
||||
super(value, queue);
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
public K getKey() {
|
||||
return this.key;
|
||||
}
|
||||
|
||||
public V getValue() {
|
||||
return get();
|
||||
}
|
||||
|
||||
public V setValue(@SuppressWarnings("unused") V value) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
}
|
|
@ -0,0 +1,122 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* A Map that uses Soft Reference values internally. Use as a simple cache.
|
||||
*
|
||||
* @param <K> key class
|
||||
* @param <V> value class
|
||||
*/
|
||||
public class SoftValueMap<K,V> implements Map<K,V> {
|
||||
private final Map<K, SoftValue<K,V>> internalMap =
|
||||
new HashMap<K, SoftValue<K,V>>();
|
||||
private final ReferenceQueueUtil<K,SoftValue<K,V>> rq =
|
||||
new ReferenceQueueUtil<K,SoftValue<K,V>>(this.internalMap);
|
||||
|
||||
public V put(K key, V value) {
|
||||
this.rq.checkReferences();
|
||||
SoftValue<K,V> oldValue = this.internalMap.put(key,
|
||||
new SoftValue<K,V>(key, value, this.rq.getReferenceQueue()));
|
||||
return oldValue == null ? null : oldValue.get();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void putAll(@SuppressWarnings("unused") Map map) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
|
||||
public V get(Object key) {
|
||||
this.rq.checkReferences();
|
||||
SoftValue<K,V> value = this.internalMap.get(key);
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
if (value.get() == null) {
|
||||
this.internalMap.remove(key);
|
||||
return null;
|
||||
}
|
||||
return value.get();
|
||||
}
|
||||
|
||||
public V remove(Object key) {
|
||||
this.rq.checkReferences();
|
||||
SoftValue<K,V> value = this.internalMap.remove(key);
|
||||
return value == null ? null : value.get();
|
||||
}
|
||||
|
||||
public boolean containsKey(Object key) {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.containsKey(key);
|
||||
}
|
||||
|
||||
public boolean containsValue(@SuppressWarnings("unused") Object value) {
|
||||
/* checkReferences();
|
||||
return internalMap.containsValue(value);*/
|
||||
throw new UnsupportedOperationException("Don't support containsValue!");
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.isEmpty();
|
||||
}
|
||||
|
||||
public int size() {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.size();
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
this.rq.checkReferences();
|
||||
this.internalMap.clear();
|
||||
}
|
||||
|
||||
public Set<K> keySet() {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.keySet();
|
||||
}
|
||||
|
||||
public Set<Map.Entry<K,V>> entrySet() {
|
||||
this.rq.checkReferences();
|
||||
Set<Map.Entry<K, SoftValue<K,V>>> entries = this.internalMap.entrySet();
|
||||
Set<Map.Entry<K, V>> real_entries = new HashSet<Map.Entry<K,V>>();
|
||||
for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
|
||||
real_entries.add(entry.getValue());
|
||||
}
|
||||
return real_entries;
|
||||
}
|
||||
|
||||
public Collection<V> values() {
|
||||
this.rq.checkReferences();
|
||||
Collection<SoftValue<K,V>> softValues = this.internalMap.values();
|
||||
ArrayList<V> hardValues = new ArrayList<V>();
|
||||
for(SoftValue<K,V> softValue : softValues) {
|
||||
hardValues.add(softValue.get());
|
||||
}
|
||||
return hardValues;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,173 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* A SortedMap implementation that uses Soft Reference values
|
||||
* internally to make it play well with the GC when in a low-memory
|
||||
* situation. Use as a cache where you also need SortedMap functionality.
|
||||
*
|
||||
* @param <K> key class
|
||||
* @param <V> value class
|
||||
*/
|
||||
public class SoftValueSortedMap<K,V> implements SortedMap<K,V> {
|
||||
private final SortedMap<K, SoftValue<K,V>> internalMap;
|
||||
private final ReferenceQueueUtil<K,SoftValue<K,V>> rq;
|
||||
|
||||
/** Constructor */
|
||||
public SoftValueSortedMap() {
|
||||
this(new TreeMap<K, SoftValue<K,V>>());
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
* @param c
|
||||
*/
|
||||
public SoftValueSortedMap(final Comparator<K> c) {
|
||||
this(new TreeMap<K, SoftValue<K,V>>(c));
|
||||
}
|
||||
|
||||
/** For headMap and tailMap support */
|
||||
private SoftValueSortedMap(SortedMap<K,SoftValue<K,V>> original) {
|
||||
this.internalMap = original;
|
||||
this.rq = new ReferenceQueueUtil<K,SoftValue<K,V>>(this.internalMap);
|
||||
}
|
||||
|
||||
public V put(K key, V value) {
|
||||
this.rq.checkReferences();
|
||||
SoftValue<K,V> oldValue = this.internalMap.put(key,
|
||||
new SoftValue<K,V>(key, value, this.rq.getReferenceQueue()));
|
||||
return oldValue == null ? null : oldValue.get();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void putAll(@SuppressWarnings("unused") Map map) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
|
||||
public V get(Object key) {
|
||||
this.rq.checkReferences();
|
||||
SoftValue<K,V> value = this.internalMap.get(key);
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
if (value.get() == null) {
|
||||
this.internalMap.remove(key);
|
||||
return null;
|
||||
}
|
||||
return value.get();
|
||||
}
|
||||
|
||||
public V remove(Object key) {
|
||||
this.rq.checkReferences();
|
||||
SoftValue<K,V> value = this.internalMap.remove(key);
|
||||
return value == null ? null : value.get();
|
||||
}
|
||||
|
||||
public boolean containsKey(Object key) {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.containsKey(key);
|
||||
}
|
||||
|
||||
public boolean containsValue(@SuppressWarnings("unused") Object value) {
|
||||
/* checkReferences();
|
||||
return internalMap.containsValue(value);*/
|
||||
throw new UnsupportedOperationException("Don't support containsValue!");
|
||||
}
|
||||
|
||||
public K firstKey() {
|
||||
this.rq.checkReferences();
|
||||
return internalMap.firstKey();
|
||||
}
|
||||
|
||||
public K lastKey() {
|
||||
this.rq.checkReferences();
|
||||
return internalMap.lastKey();
|
||||
}
|
||||
|
||||
public SoftValueSortedMap<K,V> headMap(K key) {
|
||||
this.rq.checkReferences();
|
||||
return new SoftValueSortedMap<K,V>(this.internalMap.headMap(key));
|
||||
}
|
||||
|
||||
public SoftValueSortedMap<K,V> tailMap(K key) {
|
||||
this.rq.checkReferences();
|
||||
return new SoftValueSortedMap<K,V>(this.internalMap.tailMap(key));
|
||||
}
|
||||
|
||||
public SoftValueSortedMap<K,V> subMap(K fromKey, K toKey) {
|
||||
this.rq.checkReferences();
|
||||
return new SoftValueSortedMap<K,V>(this.internalMap.subMap(fromKey, toKey));
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.isEmpty();
|
||||
}
|
||||
|
||||
public int size() {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.size();
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
this.rq.checkReferences();
|
||||
this.internalMap.clear();
|
||||
}
|
||||
|
||||
public Set<K> keySet() {
|
||||
this.rq.checkReferences();
|
||||
return this.internalMap.keySet();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public Comparator comparator() {
|
||||
return this.internalMap.comparator();
|
||||
}
|
||||
|
||||
public Set<Map.Entry<K,V>> entrySet() {
|
||||
this.rq.checkReferences();
|
||||
Set<Map.Entry<K, SoftValue<K,V>>> entries = this.internalMap.entrySet();
|
||||
Set<Map.Entry<K, V>> real_entries = new TreeSet<Map.Entry<K,V>>();
|
||||
for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
|
||||
real_entries.add(entry.getValue());
|
||||
}
|
||||
return real_entries;
|
||||
}
|
||||
|
||||
public Collection<V> values() {
|
||||
this.rq.checkReferences();
|
||||
Collection<SoftValue<K,V>> softValues = this.internalMap.values();
|
||||
ArrayList<V> hardValues = new ArrayList<V>();
|
||||
for(SoftValue<K,V> softValue : softValues) {
|
||||
hardValues.add(softValue.get());
|
||||
}
|
||||
return hardValues;
|
||||
}
|
||||
}
|
|
@ -1,41 +0,0 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
public class SoftSortedMapTest {
|
||||
private static void testMap(SortedMap<Integer, Integer> map) {
|
||||
System.out.println("Testing " + map.getClass());
|
||||
for(int i = 0; i < 1000000; i++) {
|
||||
map.put(new Integer(i), new Integer(i));
|
||||
}
|
||||
System.out.println(map.size());
|
||||
byte[] block = new byte[849*1024*1024]; // 10 MB
|
||||
System.out.println(map.size());
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
testMap(new SoftSortedMap<Integer, Integer>());
|
||||
testMap(new TreeMap<Integer, Integer>());
|
||||
}
|
||||
}
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
public class SoftValueSortedMapTest {
|
||||
private static void testMap(SortedMap<Integer, Integer> map) {
|
||||
System.out.println("Testing " + map.getClass());
|
||||
for(int i = 0; i < 1000000; i++) {
|
||||
map.put(new Integer(i), new Integer(i));
|
||||
}
|
||||
System.out.println(map.size());
|
||||
byte[] block = new byte[849*1024*1024]; // 10 MB
|
||||
System.out.println(map.size());
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
testMap(new SoftValueSortedMap<Integer, Integer>());
|
||||
testMap(new TreeMap<Integer, Integer>());
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue