From 7ac136887acd01631c855cb9c36fc918bf63c6d4 Mon Sep 17 00:00:00 2001 From: Jim Kellerman Date: Sun, 2 Dec 2007 04:58:03 +0000 Subject: [PATCH] HADOOP-2309 ConcurrentModificationException doing get of all region start keys git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@600240 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../hadoop/hbase/HConnectionManager.java | 15 +++--- src/java/org/apache/hadoop/hbase/HRegion.java | 36 ++++++------- src/java/org/apache/hadoop/hbase/HStore.java | 39 +++++++++----- .../hadoop/hbase/LocalHBaseCluster.java | 22 +++++--- .../hbase/mapred/IndexConfiguration.java | 5 +- src/java/org/onelab/filter/Filter.java | 4 +- .../onelab/filter/RetouchedBloomFilter.java | 40 +++++++------- .../hbase/mapred/TestTableMapReduce.java | 54 +++++++++++++++---- 9 files changed, 140 insertions(+), 76 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6861b7b8ebd..986a0041148 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -47,6 +47,7 @@ Trunk (unreleased changes) (Edward Yoon via Stack) HADOOP-2320 Committed TestGet2 is managled (breaks build). HADOOP-2322 getRow(row, TS) client interface not properly connected + HADOOP-2309 ConcurrentModificationException doing get of all region start keys IMPROVEMENTS HADOOP-2401 Add convenience put method that takes writable diff --git a/src/java/org/apache/hadoop/hbase/HConnectionManager.java b/src/java/org/apache/hadoop/hbase/HConnectionManager.java index ad6f3d78131..f0289a33343 100644 --- a/src/java/org/apache/hadoop/hbase/HConnectionManager.java +++ b/src/java/org/apache/hadoop/hbase/HConnectionManager.java @@ -28,6 +28,8 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -113,10 +115,10 @@ public class HConnectionManager implements HConstants { private Set closedTables; // Set of tables currently being located - private HashSet tablesBeingLocated; + private Set tablesBeingLocated; // Known region HServerAddress.toString() -> HRegionInterface - private HashMap servers; + private Map servers; /** * constructor @@ -145,13 +147,14 @@ public class HConnectionManager implements HConstants { this.master = null; this.masterChecked = false; - this.tablesToServers = Collections.synchronizedMap( - new HashMap>()); + this.tablesToServers = + new ConcurrentHashMap>(); this.closedTables = Collections.synchronizedSet(new HashSet()); - this.tablesBeingLocated = new HashSet(); + this.tablesBeingLocated = Collections.synchronizedSet( + new HashSet()); - this.servers = new HashMap(); + this.servers = new ConcurrentHashMap(); } /** {@inheritDoc} */ diff --git a/src/java/org/apache/hadoop/hbase/HRegion.java b/src/java/org/apache/hadoop/hbase/HRegion.java index c32754966b3..5ef0a1e1ce8 100644 --- a/src/java/org/apache/hadoop/hbase/HRegion.java +++ b/src/java/org/apache/hadoop/hbase/HRegion.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -29,8 +28,8 @@ import java.util.Random; import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; -import java.util.Vector; import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -142,13 +141,13 @@ public class HRegion implements HConstants { LOG.info("starting merge of regions: " + a.getRegionName() + " and " + b.getRegionName() + " into new region " + newRegionInfo.toString()); - Map> byFamily = - new TreeMap>(); + Map> byFamily = + new TreeMap>(); byFamily = filesByFamily(byFamily, a.close()); byFamily = filesByFamily(byFamily, b.close()); - for (Map.Entry> es : byFamily.entrySet()) { + for (Map.Entry> es : byFamily.entrySet()) { Text colFamily = es.getKey(); - Vector srcFiles = es.getValue(); + List srcFiles = es.getValue(); HStoreFile dst = new HStoreFile(conf, merges, HRegionInfo.encodeRegionName(newRegionInfo.getRegionName()), colFamily, Math.abs(rand.nextLong())); @@ -175,12 +174,12 @@ public class HRegion implements HConstants { * @param storeFiles Store files to process. * @return Returns byFamily */ - private static Map> filesByFamily( - Map> byFamily, Vector storeFiles) { + private static Map> filesByFamily( + Map> byFamily, List storeFiles) { for(HStoreFile src: storeFiles) { - Vector v = byFamily.get(src.getColFamily()); + List v = byFamily.get(src.getColFamily()); if(v == null) { - v = new Vector(); + v = new ArrayList(); byFamily.put(src.getColFamily(), v); } v.add(src); @@ -192,11 +191,11 @@ public class HRegion implements HConstants { // Members ////////////////////////////////////////////////////////////////////////////// - volatile Map rowsToLocks = new HashMap(); - volatile Map locksToRows = new HashMap(); - volatile Map stores = new HashMap(); + volatile Map rowsToLocks = new ConcurrentHashMap(); + volatile Map locksToRows = new ConcurrentHashMap(); + volatile Map stores = new ConcurrentHashMap(); volatile Map> targetColumns = - new HashMap>(); + new ConcurrentHashMap>(); final AtomicLong memcacheSize = new AtomicLong(0); @@ -359,7 +358,7 @@ public class HRegion implements HConstants { * * @throws IOException */ - public Vector close() throws IOException { + public List close() throws IOException { return close(false); } @@ -377,7 +376,7 @@ public class HRegion implements HConstants { * * @throws IOException */ - Vector close(boolean abort) throws IOException { + List close(boolean abort) throws IOException { if (isClosed()) { LOG.info("region " + this.regionInfo.getRegionName() + " already closed"); return null; @@ -421,7 +420,7 @@ public class HRegion implements HConstants { internalFlushcache(snapshotMemcaches()); } - Vector result = new Vector(); + List result = new ArrayList(); for (HStore store: stores.values()) { result.addAll(store.close()); } @@ -571,7 +570,7 @@ public class HRegion implements HConstants { // Now close the HRegion. Close returns all store files or null if not // supposed to close (? What to do in this case? Implement abort of close?) // Close also does wait on outstanding rows and calls a flush just-in-case. - Vector hstoreFilesToSplit = close(); + List hstoreFilesToSplit = close(); if (hstoreFilesToSplit == null) { LOG.warn("Close came back null (Implement abort of close?)"); throw new RuntimeException("close returned empty vector of HStoreFiles"); @@ -909,6 +908,7 @@ public class HRegion implements HConstants { // A. Flush memcache to all the HStores. // Keep running vector of all store files that includes both old and the // just-made new flush store file. + for (HStore hstore: stores.values()) { hstore.flushCache(sequenceId); } diff --git a/src/java/org/apache/hadoop/hbase/HStore.java b/src/java/org/apache/hadoop/hbase/HStore.java index 846be27ed67..d2e151d988b 100644 --- a/src/java/org/apache/hadoop/hbase/HStore.java +++ b/src/java/org/apache/hadoop/hbase/HStore.java @@ -99,9 +99,11 @@ class HStore implements HConstants { void snapshot() { this.lock.writeLock().lock(); try { - if (memcache.size() != 0) { - snapshot.putAll(memcache); - memcache.clear(); + synchronized (memcache) { + if (memcache.size() != 0) { + snapshot.putAll(memcache); + memcache.clear(); + } } } finally { this.lock.writeLock().unlock(); @@ -149,9 +151,14 @@ class HStore implements HConstants { List get(final HStoreKey key, final int numVersions) { this.lock.readLock().lock(); try { - ArrayList results = internalGet(memcache, key, numVersions); - results.addAll(results.size(), + List results; + synchronized (memcache) { + results = internalGet(memcache, key, numVersions); + } + synchronized (snapshot) { + results.addAll(results.size(), internalGet(snapshot, key, numVersions - results.size())); + } return results; } finally { @@ -170,8 +177,12 @@ class HStore implements HConstants { void getFull(HStoreKey key, SortedMap results) { this.lock.readLock().lock(); try { - internalGetFull(memcache, key, results); - internalGetFull(snapshot, key, results); + synchronized (memcache) { + internalGetFull(memcache, key, results); + } + synchronized (snapshot) { + internalGetFull(snapshot, key, results); + } } finally { this.lock.readLock().unlock(); @@ -248,11 +259,15 @@ class HStore implements HConstants { List getKeys(final HStoreKey origin, final int versions) { this.lock.readLock().lock(); try { - List results = - internalGetKeys(this.memcache, origin, versions); - results.addAll(results.size(), internalGetKeys(snapshot, origin, - versions == HConstants.ALL_VERSIONS ? versions : - (versions - results.size()))); + List results; + synchronized (memcache) { + results = internalGetKeys(this.memcache, origin, versions); + } + synchronized (snapshot) { + results.addAll(results.size(), internalGetKeys(snapshot, origin, + versions == HConstants.ALL_VERSIONS ? versions : + (versions - results.size()))); + } return results; } finally { diff --git a/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java index 151f5420062..8e83949bb3e 100644 --- a/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java +++ b/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java @@ -104,10 +104,12 @@ public class LocalHBaseCluster implements HConstants { * @return Region server added. */ public RegionServerThread addRegionServer() throws IOException { - RegionServerThread t = new RegionServerThread(new HRegionServer(conf), - this.regionThreads.size()); - this.regionThreads.add(t); - return t; + synchronized (regionThreads) { + RegionServerThread t = new RegionServerThread(new HRegionServer(conf), + this.regionThreads.size()); + this.regionThreads.add(t); + return t; + } } /** runs region servers */ @@ -146,8 +148,10 @@ public class LocalHBaseCluster implements HConstants { * @return Name of region server that just went down. */ public String waitOnRegionServer(int serverNumber) { - RegionServerThread regionServerThread = - this.regionThreads.remove(serverNumber); + RegionServerThread regionServerThread; + synchronized (regionThreads) { + regionServerThread = this.regionThreads.remove(serverNumber); + } while (regionServerThread.isAlive()) { try { LOG.info("Waiting on " + @@ -193,8 +197,10 @@ public class LocalHBaseCluster implements HConstants { */ public String startup() { this.master.start(); - for (RegionServerThread t: this.regionThreads) { - t.start(); + synchronized (regionThreads) { + for (RegionServerThread t: this.regionThreads) { + t.start(); + } } return this.master.getMasterAddress().toString(); } diff --git a/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java b/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java index d88fd5ccb29..3a1535bc306 100644 --- a/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java +++ b/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java @@ -23,7 +23,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; import java.io.StringWriter; -import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.Iterator; import java.util.Map; import java.util.Properties; @@ -105,7 +105,8 @@ public class IndexConfiguration extends Configuration { } } - private HashMap columnMap = new HashMap(); + private Map columnMap = + new ConcurrentHashMap(); public Iterator columnNameIterator() { return columnMap.keySet().iterator(); diff --git a/src/java/org/onelab/filter/Filter.java b/src/java/org/onelab/filter/Filter.java index 4f6a398bfb7..e9c3ec5881c 100644 --- a/src/java/org/onelab/filter/Filter.java +++ b/src/java/org/onelab/filter/Filter.java @@ -52,8 +52,8 @@ package org.onelab.filter; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; +import java.util.List; import org.apache.hadoop.io.WritableComparable; /** @@ -146,7 +146,7 @@ public abstract class Filter implements WritableComparable { * Adds a list of keys to this filter. * @param keys The list of keys. */ - public void add(ArrayList keys){ + public void add(List keys){ if(keys == null) { throw new IllegalArgumentException("ArrayList may not be null"); } diff --git a/src/java/org/onelab/filter/RetouchedBloomFilter.java b/src/java/org/onelab/filter/RetouchedBloomFilter.java index 2d71da6e4b2..7899f9cfc05 100644 --- a/src/java/org/onelab/filter/RetouchedBloomFilter.java +++ b/src/java/org/onelab/filter/RetouchedBloomFilter.java @@ -52,6 +52,8 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Random; /** @@ -76,12 +78,12 @@ implements RemoveScheme { /** * KeyList vector (or ElementList Vector, as defined in the paper) of false positives. */ - ArrayList[] fpVector; + List[] fpVector; /** * KeyList vector of keys recorded in the filter. */ - ArrayList[] keyVector; + List[] keyVector; /** * Ratio vector. @@ -158,7 +160,7 @@ implements RemoveScheme { * Adds a list of false positive information to this retouched Bloom filter. * @param keys The list of false positive. */ - public void addFalsePositive(ArrayList keys){ + public void addFalsePositive(List keys){ if(keys == null) { throw new NullPointerException("ArrayList can not be null"); } @@ -306,8 +308,8 @@ implements RemoveScheme { throw new ArrayIndexOutOfBoundsException(index); } - ArrayList kl = keyVector[index]; - ArrayList fpl = fpVector[index]; + List kl = keyVector[index]; + List fpl = fpVector[index]; // update key list int listSize = kl.size(); @@ -339,7 +341,7 @@ implements RemoveScheme { * @param k The key to remove. * @param vector The counting vector associated to the key. */ - private void removeKey(Key k, ArrayList[] vector) { + private void removeKey(Key k, List[] vector) { if(k == null) { throw new NullPointerException("Key can not be null"); } @@ -369,7 +371,7 @@ implements RemoveScheme { }//end for - i }//end computeRatio() - private double getWeight(ArrayList keyList) { + private double getWeight(List keyList) { double weight = 0.0; for(Key k: keyList) { weight += k.getWeight(); @@ -382,13 +384,13 @@ implements RemoveScheme { */ @SuppressWarnings("unchecked") private void createVector() { - fpVector = new ArrayList[vectorSize]; - keyVector = new ArrayList[vectorSize]; + fpVector = new List[vectorSize]; + keyVector = new List[vectorSize]; ratio = new double[vectorSize]; for(int i = 0; i < vectorSize; i++) { - fpVector[i] = new ArrayList(); - keyVector[i] = new ArrayList(); + fpVector[i] = Collections.synchronizedList(new ArrayList()); + keyVector[i] = Collections.synchronizedList(new ArrayList()); ratio[i] = 0.0; }//end for -i }//end createVector() @@ -422,14 +424,14 @@ implements RemoveScheme { public void write(DataOutput out) throws IOException { super.write(out); for(int i = 0; i < fpVector.length; i++) { - ArrayList list = fpVector[i]; + List list = fpVector[i]; out.writeInt(list.size()); for(Key k: list) { k.write(out); } } for(int i = 0; i < keyVector.length; i++) { - ArrayList list = keyVector[i]; + List list = keyVector[i]; out.writeInt(list.size()); for(Key k: list) { k.write(out); @@ -446,7 +448,7 @@ implements RemoveScheme { super.readFields(in); createVector(); for(int i = 0; i < fpVector.length; i++) { - ArrayList list = fpVector[i]; + List list = fpVector[i]; int size = in.readInt(); for(int j = 0; j < size; j++) { Key k = new Key(); @@ -455,7 +457,7 @@ implements RemoveScheme { } } for(int i = 0; i < keyVector.length; i++) { - ArrayList list = keyVector[i]; + List list = keyVector[i]; int size = in.readInt(); for(int j = 0; j < size; j++) { Key k = new Key(); @@ -478,8 +480,8 @@ implements RemoveScheme { RetouchedBloomFilter other = (RetouchedBloomFilter)o; for(int i = 0; result == 0 && i < fpVector.length; i++) { - ArrayList mylist = fpVector[i]; - ArrayList otherlist = other.fpVector[i]; + List mylist = fpVector[i]; + List otherlist = other.fpVector[i]; for(int j = 0; result == 0 && j < mylist.size(); j++) { result = mylist.get(j).compareTo(otherlist.get(j)); @@ -487,8 +489,8 @@ implements RemoveScheme { } for(int i = 0; result == 0 && i < keyVector.length; i++) { - ArrayList mylist = keyVector[i]; - ArrayList otherlist = other.keyVector[i]; + List mylist = keyVector[i]; + List otherlist = other.keyVector[i]; for(int j = 0; result == 0 && j < mylist.size(); j++) { result = mylist.get(j).compareTo(otherlist.get(j)); diff --git a/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java b/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java index 6a2f214e6e0..5bde61436b9 100644 --- a/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java +++ b/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java @@ -372,27 +372,63 @@ public class TestTableMapReduce extends MultiRegionTable { TreeMap results = new TreeMap(); while(scanner.next(key, results)) { + if (LOG.isDebugEnabled()) { + if (results.size() > 2 ) { + LOG.debug("Too many results, expected 2 got " + results.size()); + } + } byte[] firstValue = null; byte[] secondValue = null; int count = 0; - for(Map.Entry e: results.entrySet()) { if (count == 0) firstValue = e.getValue(); if (count == 1) secondValue = e.getValue(); count++; + if (count == 2) { + break; + } } - // verify second value is the reverse of the first - assertNotNull(firstValue); - assertNotNull(secondValue); - assertEquals(firstValue.length, secondValue.length); - byte[] secondReversed = new byte[secondValue.length]; - for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { - secondReversed[i] = secondValue[j]; + String first = ""; + if (firstValue == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("row=" + key.getRow() + ": first value is null"); + } + fail(); + + } else { + first = new String(firstValue, HConstants.UTF8_ENCODING); + if (LOG.isDebugEnabled()) { + LOG.debug("row=" + key.getRow() + ": first value=" + first); + } + } + + String second = ""; + if (secondValue == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("row=" + key.getRow() + ": second value is null"); + } + fail(); + + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("row=" + key.getRow() + ": second value=" + + new String(secondValue, HConstants.UTF8_ENCODING)); + } + byte[] secondReversed = new byte[secondValue.length]; + for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { + secondReversed[i] = secondValue[j]; + } + second = new String(secondReversed, HConstants.UTF8_ENCODING); + } + if (first.compareTo(second) != 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("second key is not the reverse of first"); + } + fail(); } - assertTrue(Arrays.equals(firstValue, secondReversed)); } } finally {