HBASE-407 Client should cache region locations in an LRU structure
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@627573 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
77886ac7af
commit
3067b43dc2
|
@ -45,8 +45,11 @@ Hbase Change Log
|
|||
HBASE-436 website: http://hadoop.apache.org/hbase
|
||||
HBASE-417 Factor TableOperation and subclasses into separate files from
|
||||
HMaster (Bryan Duxbury via Stack)
|
||||
|
||||
HBASE-440 Add optional log roll interval so that log files are garbage
|
||||
collected
|
||||
HBASE-407 Keep HRegionLocation information in LRU structure
|
||||
(Bryan Duxbury)
|
||||
|
||||
|
||||
Branch 0.1
|
||||
|
|
|
@ -20,15 +20,12 @@
|
|||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
@ -41,6 +38,7 @@ import org.apache.hadoop.io.Text;
|
|||
import org.apache.hadoop.io.Writable;
|
||||
import org.apache.hadoop.ipc.RemoteException;
|
||||
import org.apache.hadoop.hbase.master.HMasterInterface;
|
||||
import org.apache.hadoop.hbase.util.SoftSortedMap;
|
||||
|
||||
/**
|
||||
* A non-instantiable class that manages connections to multiple tables in
|
||||
|
@ -115,7 +113,9 @@ public class HConnectionManager implements HConstants {
|
|||
|
||||
private HRegionLocation rootRegionLocation;
|
||||
|
||||
private Map<Text, SortedMap<Text, HRegionLocation>> cachedRegionLocations;
|
||||
private Map<Text, SoftSortedMap<Text, HRegionLocation>>
|
||||
cachedRegionLocations = new ConcurrentHashMap<Text,
|
||||
SoftSortedMap<Text, HRegionLocation>>();;
|
||||
|
||||
/**
|
||||
* constructor
|
||||
|
@ -144,9 +144,6 @@ public class HConnectionManager implements HConstants {
|
|||
|
||||
this.master = null;
|
||||
this.masterChecked = false;
|
||||
|
||||
this.cachedRegionLocations =
|
||||
new ConcurrentHashMap<Text, SortedMap<Text, HRegionLocation>>();
|
||||
this.servers = new ConcurrentHashMap<String, HRegionInterface>();
|
||||
}
|
||||
|
||||
|
@ -489,12 +486,12 @@ public class HConnectionManager implements HConstants {
|
|||
*/
|
||||
private HRegionLocation getCachedLocation(Text tableName, Text row) {
|
||||
// find the map of cached locations for this table
|
||||
SortedMap<Text, HRegionLocation> tableLocations =
|
||||
SoftSortedMap<Text, HRegionLocation> tableLocations =
|
||||
cachedRegionLocations.get(tableName);
|
||||
|
||||
// if tableLocations for this table isn't built yet, make one
|
||||
if (tableLocations == null) {
|
||||
tableLocations = new TreeMap<Text, HRegionLocation>();
|
||||
tableLocations = new SoftSortedMap<Text, HRegionLocation>();
|
||||
cachedRegionLocations.put(tableName, tableLocations);
|
||||
}
|
||||
|
||||
|
@ -514,7 +511,7 @@ public class HConnectionManager implements HConstants {
|
|||
|
||||
// cut the cache so that we only get the part that could contain
|
||||
// regions that match our key
|
||||
SortedMap<Text, HRegionLocation> matchingRegions =
|
||||
SoftSortedMap<Text, HRegionLocation> matchingRegions =
|
||||
tableLocations.headMap(row);
|
||||
|
||||
// if that portion of the map is empty, then we're done. otherwise,
|
||||
|
@ -523,7 +520,7 @@ public class HConnectionManager implements HConstants {
|
|||
if (!matchingRegions.isEmpty()) {
|
||||
HRegionLocation possibleRegion =
|
||||
matchingRegions.get(matchingRegions.lastKey());
|
||||
|
||||
|
||||
Text endKey = possibleRegion.getRegionInfo().getEndKey();
|
||||
|
||||
// make sure that the end key is greater than the row we're looking
|
||||
|
@ -551,12 +548,12 @@ public class HConnectionManager implements HConstants {
|
|||
*/
|
||||
private void deleteCachedLocation(Text tableName, Text row){
|
||||
// find the map of cached locations for this table
|
||||
SortedMap<Text, HRegionLocation> tableLocations =
|
||||
SoftSortedMap<Text, HRegionLocation> tableLocations =
|
||||
cachedRegionLocations.get(tableName);
|
||||
|
||||
// if tableLocations for this table isn't built yet, make one
|
||||
if (tableLocations == null) {
|
||||
tableLocations = new TreeMap<Text, HRegionLocation>();
|
||||
tableLocations = new SoftSortedMap<Text, HRegionLocation>();
|
||||
cachedRegionLocations.put(tableName, tableLocations);
|
||||
}
|
||||
|
||||
|
@ -565,7 +562,7 @@ public class HConnectionManager implements HConstants {
|
|||
if (!tableLocations.isEmpty()) {
|
||||
// cut the cache so that we only get the part that could contain
|
||||
// regions that match our key
|
||||
SortedMap<Text, HRegionLocation> matchingRegions =
|
||||
SoftSortedMap<Text, HRegionLocation> matchingRegions =
|
||||
tableLocations.headMap(row);
|
||||
|
||||
// if that portion of the map is empty, then we're done. otherwise,
|
||||
|
@ -599,12 +596,12 @@ public class HConnectionManager implements HConstants {
|
|||
Text startKey = location.getRegionInfo().getStartKey();
|
||||
|
||||
// find the map of cached locations for this table
|
||||
SortedMap<Text, HRegionLocation> tableLocations =
|
||||
SoftSortedMap<Text, HRegionLocation> tableLocations =
|
||||
cachedRegionLocations.get(tableName);
|
||||
|
||||
// if tableLocations for this table isn't built yet, make one
|
||||
if (tableLocations == null) {
|
||||
tableLocations = new TreeMap<Text, HRegionLocation>();
|
||||
tableLocations = new SoftSortedMap<Text, HRegionLocation>();
|
||||
cachedRegionLocations.put(tableName, tableLocations);
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,187 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.lang.ref.SoftReference;
|
||||
import java.lang.ref.ReferenceQueue;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.Comparator;
|
||||
import java.util.Collection;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* A SortedMap implementation that uses SoftReferences internally to make it
|
||||
* play well with the GC when in a low-memory situation.
|
||||
*/
|
||||
public class SoftSortedMap<K,V> implements SortedMap<K,V> {
|
||||
protected static final Log LOG = LogFactory.getLog(SoftSortedMap.class);
|
||||
|
||||
protected SortedMap<K, SoftValue<K,V>> internalMap =
|
||||
new TreeMap<K, SoftValue<K,V>>();
|
||||
|
||||
protected ReferenceQueue referenceQueue = new ReferenceQueue();
|
||||
|
||||
/** Constructor */
|
||||
public SoftSortedMap() {}
|
||||
|
||||
/** For headMap and tailMap support */
|
||||
private SoftSortedMap(SortedMap<K,SoftValue<K,V>> original) {
|
||||
internalMap = original;
|
||||
}
|
||||
|
||||
/* Client methods */
|
||||
public V put(K key, V value) {
|
||||
checkReferences();
|
||||
SoftValue<K,V> oldValue =
|
||||
internalMap.put(key, new SoftValue<K,V>(key, value, referenceQueue));
|
||||
return oldValue == null ? null : oldValue.get();
|
||||
}
|
||||
|
||||
public void putAll(Map map) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
|
||||
public V get(Object key) {
|
||||
checkReferences();
|
||||
SoftValue<K,V> value = internalMap.get(key);
|
||||
return value == null ? null : value.get();
|
||||
}
|
||||
|
||||
public V remove(Object key) {
|
||||
checkReferences();
|
||||
SoftValue<K,V> value = internalMap.remove(key);
|
||||
return value == null ? null : value.get();
|
||||
}
|
||||
|
||||
public boolean containsKey(Object key) {
|
||||
checkReferences();
|
||||
return internalMap.containsKey(key);
|
||||
}
|
||||
|
||||
public boolean containsValue(Object value) {
|
||||
checkReferences();
|
||||
return internalMap.containsValue(value);
|
||||
}
|
||||
|
||||
public K firstKey() {
|
||||
checkReferences();
|
||||
return internalMap.firstKey();
|
||||
}
|
||||
|
||||
public K lastKey() {
|
||||
checkReferences();
|
||||
return internalMap.lastKey();
|
||||
}
|
||||
|
||||
public SoftSortedMap<K,V> headMap(K key) {
|
||||
checkReferences();
|
||||
return new SoftSortedMap<K,V>(internalMap.headMap(key));
|
||||
}
|
||||
|
||||
public SoftSortedMap<K,V> tailMap(K key) {
|
||||
checkReferences();
|
||||
return new SoftSortedMap<K,V>(internalMap.tailMap(key));
|
||||
}
|
||||
|
||||
public SoftSortedMap<K,V> subMap(K fromKey, K toKey) {
|
||||
checkReferences();
|
||||
return new SoftSortedMap<K,V>(internalMap.subMap(fromKey, toKey));
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
checkReferences();
|
||||
return internalMap.isEmpty();
|
||||
}
|
||||
|
||||
public int size() {
|
||||
checkReferences();
|
||||
return internalMap.size();
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
internalMap.clear();
|
||||
}
|
||||
|
||||
public Set<K> keySet() {
|
||||
return internalMap.keySet();
|
||||
}
|
||||
|
||||
public Comparator comparator() {
|
||||
return internalMap.comparator();
|
||||
}
|
||||
|
||||
public Set<Map.Entry<K,V>> entrySet() {
|
||||
Set<Map.Entry<K, SoftValue<K,V>>> entries = internalMap.entrySet();
|
||||
Set<Map.Entry<K, V>> real_entries = new TreeSet<Map.Entry<K,V>>();
|
||||
for(Map.Entry<K, SoftValue<K,V>> entry : entries) {
|
||||
real_entries.add(entry.getValue());
|
||||
}
|
||||
return real_entries;
|
||||
}
|
||||
|
||||
public Collection<V> values() {
|
||||
checkReferences();
|
||||
Collection<SoftValue<K,V>> softValues = internalMap.values();
|
||||
ArrayList<V> hardValues = new ArrayList<V>();
|
||||
for(SoftValue<K,V> softValue : softValues) {
|
||||
hardValues.add(softValue.get());
|
||||
}
|
||||
return hardValues;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the reference queue and delete anything that has since gone away
|
||||
*/
|
||||
private void checkReferences() {
|
||||
SoftValue<K,V> sv;
|
||||
while((sv = (SoftValue<K,V>)referenceQueue.poll()) != null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Reference for key " + sv.key.toString() + " has been cleared.");
|
||||
}
|
||||
internalMap.remove(sv.key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A SoftReference derivative so that we can track down what keys to remove.
|
||||
*/
|
||||
private class SoftValue<K2, V2> extends SoftReference<V2> implements Map.Entry<K2,V2> {
|
||||
K2 key;
|
||||
|
||||
SoftValue(K2 key, V2 value, ReferenceQueue queue) {
|
||||
super(value, queue);
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
public K2 getKey() {return key;}
|
||||
public V2 getValue() {return get();}
|
||||
|
||||
public V2 setValue(V2 value) {
|
||||
throw new RuntimeException("Not implemented");
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* Copyright 2008 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
public class SoftSortedMapTest {
|
||||
private static void testMap(SortedMap<Integer, Integer> map) {
|
||||
System.out.println("Testing " + map.getClass());
|
||||
for(int i = 0; i < 1000000; i++) {
|
||||
map.put(new Integer(i), new Integer(i));
|
||||
}
|
||||
System.out.println(map.size());
|
||||
byte[] block = new byte[849*1024*1024]; // 10 MB
|
||||
System.out.println(map.size());
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
testMap(new SoftSortedMap<Integer, Integer>());
|
||||
testMap(new TreeMap<Integer, Integer>());
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue