From 50e5f13ec11de2cc44f6332f5b3806685446dd22 Mon Sep 17 00:00:00 2001 From: anoopsjohn Date: Thu, 25 Sep 2014 17:45:15 +0530 Subject: [PATCH] HBASE-12050 Avoid KeyValueUtil#ensureKeyValue from DefaultMemStore. --- .../org/apache/hadoop/hbase/KeyValueUtil.java | 2 + .../apache/hadoop/hbase/util/ClassSize.java | 6 +- ...eSkipListSet.java => CellSkipListSet.java} | 67 ++-- .../hbase/regionserver/DefaultMemStore.java | 357 +++++++++--------- .../hbase/util/CollectionBackedScanner.java | 34 +- .../apache/hadoop/hbase/io/TestHeapSize.java | 14 +- .../io/encoding/TestPrefixTreeEncoding.java | 14 +- ...pListSet.java => TestCellSkipListSet.java} | 63 ++-- .../regionserver/TestDefaultMemStore.java | 52 +-- .../hbase/regionserver/TestHRegion.java | 8 +- .../hbase/regionserver/TestKeyValueHeap.java | 24 +- .../regionserver/TestMemStoreChunkPool.java | 8 +- .../hadoop/hbase/regionserver/TestStore.java | 10 +- 13 files changed, 328 insertions(+), 331 deletions(-) rename hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/{KeyValueSkipListSet.java => CellSkipListSet.java} (71%) rename hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/{TestKeyValueSkipListSet.java => TestCellSkipListSet.java} (76%) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index d1faa8c1cb7..e2ccb3da20a 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -134,6 +134,8 @@ public class KeyValueUtil { /**************** copy key and value *********************/ public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) { + // TODO when cell instance of KV we can bypass all steps and just do backing single array + // copy(?) int pos = offset; pos = Bytes.putInt(output, pos, keyLength(cell)); pos = Bytes.putInt(output, pos, cell.getValueLength()); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java index 99e7444851e..1bb34e1ec7e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -107,8 +107,8 @@ public class ClassSize { /** Overhead for TimeRangeTracker */ public static final int TIMERANGE_TRACKER; - /** Overhead for KeyValueSkipListSet */ - public static final int KEYVALUE_SKIPLIST_SET; + /** Overhead for CellSkipListSet */ + public static final int CELL_SKIPLIST_SET; /* Are we running on jdk7? */ private static final boolean JDK7; @@ -192,7 +192,7 @@ public class ClassSize { TIMERANGE_TRACKER = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2); - KEYVALUE_SKIPLIST_SET = align(OBJECT + REFERENCE); + CELL_SKIPLIST_SET = align(OBJECT + REFERENCE); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java similarity index 71% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java index 4b2d54a8ede..4c3ab503636 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueSkipListSet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java @@ -18,9 +18,6 @@ */ package org.apache.hadoop.hbase.regionserver; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.KeyValue; - import java.util.Collection; import java.util.Comparator; import java.util.Iterator; @@ -29,8 +26,12 @@ import java.util.SortedSet; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + /** - * A {@link java.util.Set} of {@link KeyValue}s implemented on top of a + * A {@link java.util.Set} of {@link Cell}s implemented on top of a * {@link java.util.concurrent.ConcurrentSkipListMap}. Works like a * {@link java.util.concurrent.ConcurrentSkipListSet} in all but one regard: * An add will overwrite if already an entry for the added key. In other words, @@ -44,96 +45,96 @@ import java.util.concurrent.ConcurrentSkipListMap; * get and set and won't throw ConcurrentModificationException when iterating. */ @InterfaceAudience.Private -public class KeyValueSkipListSet implements NavigableSet { - private final ConcurrentNavigableMap delegatee; +public class CellSkipListSet implements NavigableSet { + private final ConcurrentNavigableMap delegatee; - KeyValueSkipListSet(final KeyValue.KVComparator c) { - this.delegatee = new ConcurrentSkipListMap(c); + CellSkipListSet(final KeyValue.KVComparator c) { + this.delegatee = new ConcurrentSkipListMap(c); } - KeyValueSkipListSet(final ConcurrentNavigableMap m) { + CellSkipListSet(final ConcurrentNavigableMap m) { this.delegatee = m; } - public KeyValue ceiling(KeyValue e) { + public Cell ceiling(Cell e) { throw new UnsupportedOperationException("Not implemented"); } - public Iterator descendingIterator() { + public Iterator descendingIterator() { return this.delegatee.descendingMap().values().iterator(); } - public NavigableSet descendingSet() { + public NavigableSet descendingSet() { throw new UnsupportedOperationException("Not implemented"); } - public KeyValue floor(KeyValue e) { + public Cell floor(Cell e) { throw new UnsupportedOperationException("Not implemented"); } - public SortedSet headSet(final KeyValue toElement) { + public SortedSet headSet(final Cell toElement) { return headSet(toElement, false); } - public NavigableSet headSet(final KeyValue toElement, + public NavigableSet headSet(final Cell toElement, boolean inclusive) { - return new KeyValueSkipListSet(this.delegatee.headMap(toElement, inclusive)); + return new CellSkipListSet(this.delegatee.headMap(toElement, inclusive)); } - public KeyValue higher(KeyValue e) { + public Cell higher(Cell e) { throw new UnsupportedOperationException("Not implemented"); } - public Iterator iterator() { + public Iterator iterator() { return this.delegatee.values().iterator(); } - public KeyValue lower(KeyValue e) { + public Cell lower(Cell e) { throw new UnsupportedOperationException("Not implemented"); } - public KeyValue pollFirst() { + public Cell pollFirst() { throw new UnsupportedOperationException("Not implemented"); } - public KeyValue pollLast() { + public Cell pollLast() { throw new UnsupportedOperationException("Not implemented"); } - public SortedSet subSet(KeyValue fromElement, KeyValue toElement) { + public SortedSet subSet(Cell fromElement, Cell toElement) { throw new UnsupportedOperationException("Not implemented"); } - public NavigableSet subSet(KeyValue fromElement, - boolean fromInclusive, KeyValue toElement, boolean toInclusive) { + public NavigableSet subSet(Cell fromElement, + boolean fromInclusive, Cell toElement, boolean toInclusive) { throw new UnsupportedOperationException("Not implemented"); } - public SortedSet tailSet(KeyValue fromElement) { + public SortedSet tailSet(Cell fromElement) { return tailSet(fromElement, true); } - public NavigableSet tailSet(KeyValue fromElement, boolean inclusive) { - return new KeyValueSkipListSet(this.delegatee.tailMap(fromElement, inclusive)); + public NavigableSet tailSet(Cell fromElement, boolean inclusive) { + return new CellSkipListSet(this.delegatee.tailMap(fromElement, inclusive)); } - public Comparator comparator() { + public Comparator comparator() { throw new UnsupportedOperationException("Not implemented"); } - public KeyValue first() { + public Cell first() { return this.delegatee.get(this.delegatee.firstKey()); } - public KeyValue last() { + public Cell last() { return this.delegatee.get(this.delegatee.lastKey()); } - public boolean add(KeyValue e) { + public boolean add(Cell e) { return this.delegatee.put(e, e) == null; } - public boolean addAll(Collection c) { + public boolean addAll(Collection c) { throw new UnsupportedOperationException("Not implemented"); } @@ -166,7 +167,7 @@ public class KeyValueSkipListSet implements NavigableSet { throw new UnsupportedOperationException("Not implemented"); } - public KeyValue get(KeyValue kv) { + public Cell get(Cell kv) { return this.delegatee.get(kv); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index f77a8bdc1c4..fddfdcaa63d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -73,15 +73,15 @@ public class DefaultMemStore implements MemStore { private Configuration conf; - // MemStore. Use a KeyValueSkipListSet rather than SkipListSet because of the + // MemStore. Use a CellSkipListSet rather than SkipListSet because of the // better semantics. The Map will overwrite if passed a key it already had - // whereas the Set will not add new KV if key is same though value might be + // whereas the Set will not add new Cell if key is same though value might be // different. Value is not important -- just make sure always same // reference passed. - volatile KeyValueSkipListSet kvset; + volatile CellSkipListSet cellSet; // Snapshot of memstore. Made for flusher. - volatile KeyValueSkipListSet snapshot; + volatile CellSkipListSet snapshot; final KeyValue.KVComparator comparator; @@ -114,8 +114,8 @@ public class DefaultMemStore implements MemStore { final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; - this.kvset = new KeyValueSkipListSet(c); - this.snapshot = new KeyValueSkipListSet(c); + this.cellSet = new CellSkipListSet(c); + this.snapshot = new CellSkipListSet(c); timeRangeTracker = new TimeRangeTracker(); snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); @@ -130,11 +130,11 @@ public class DefaultMemStore implements MemStore { } void dump() { - for (KeyValue kv: this.kvset) { - LOG.info(kv); + for (Cell cell: this.cellSet) { + LOG.info(cell); } - for (KeyValue kv: this.snapshot) { - LOG.info(kv); + for (Cell cell: this.snapshot) { + LOG.info(cell); } } @@ -152,9 +152,9 @@ public class DefaultMemStore implements MemStore { } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); this.snapshotSize = keySize(); - if (!this.kvset.isEmpty()) { - this.snapshot = this.kvset; - this.kvset = new KeyValueSkipListSet(this.comparator); + if (!this.cellSet.isEmpty()) { + this.snapshot = this.cellSet; + this.cellSet = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = this.timeRangeTracker; this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys @@ -189,7 +189,7 @@ public class DefaultMemStore implements MemStore { // OK. Passed in snapshot is same as current snapshot. If not-empty, // create a new snapshot and let the old one go. if (!this.snapshot.isEmpty()) { - this.snapshot = new KeyValueSkipListSet(this.comparator); + this.snapshot = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = new TimeRangeTracker(); } this.snapshotSize = 0; @@ -216,7 +216,7 @@ public class DefaultMemStore implements MemStore { */ @Override public Pair add(Cell cell) { - KeyValue toAdd = maybeCloneWithAllocator(KeyValueUtil.ensureKeyValue(cell)); + Cell toAdd = maybeCloneWithAllocator(cell); return new Pair(internalAdd(toAdd), toAdd); } @@ -225,14 +225,14 @@ public class DefaultMemStore implements MemStore { return timeOfOldestEdit; } - private boolean addToKVSet(KeyValue e) { - boolean b = this.kvset.add(e); + private boolean addToCellSet(Cell e) { + boolean b = this.cellSet.add(e); setOldestEditTimeToNow(); return b; } - private boolean removeFromKVSet(KeyValue e) { - boolean b = this.kvset.remove(e); + private boolean removeFromCellSet(Cell e) { + boolean b = this.cellSet.remove(e); setOldestEditTimeToNow(); return b; } @@ -244,39 +244,39 @@ public class DefaultMemStore implements MemStore { } /** - * Internal version of add() that doesn't clone KVs with the + * Internal version of add() that doesn't clone Cells with the * allocator, and doesn't take the lock. * * Callers should ensure they already have the read lock taken */ - private long internalAdd(final KeyValue toAdd) { - long s = heapSizeChange(toAdd, addToKVSet(toAdd)); + private long internalAdd(final Cell toAdd) { + long s = heapSizeChange(toAdd, addToCellSet(toAdd)); timeRangeTracker.includeTimestamp(toAdd); this.size.addAndGet(s); return s; } - private KeyValue maybeCloneWithAllocator(KeyValue kv) { + private Cell maybeCloneWithAllocator(Cell cell) { if (allocator == null) { - return kv; + return cell; } - int len = kv.getLength(); + int len = KeyValueUtil.length(cell); ByteRange alloc = allocator.allocateBytes(len); if (alloc == null) { // The allocation was too large, allocator decided // not to do anything with it. - return kv; + return cell; } assert alloc.getBytes() != null; - alloc.put(0, kv.getBuffer(), kv.getOffset(), len); + KeyValueUtil.appendToByteArray(cell, alloc.getBytes(), alloc.getOffset()); KeyValue newKv = new KeyValue(alloc.getBytes(), alloc.getOffset(), len); - newKv.setSequenceId(kv.getMvccVersion()); + newKv.setSequenceId(cell.getSequenceId()); return newKv; } /** - * Remove n key from the memstore. Only kvs that have the same key and the + * Remove n key from the memstore. Only cells that have the same key and the * same memstoreTS are removed. It is ok to not update timeRangeTracker * in this call. It is possible that we can optimize this method by using * tailMap/iterator, but since this method is called rarely (only for @@ -290,18 +290,17 @@ public class DefaultMemStore implements MemStore { // not the snapshot. The flush of this snapshot to disk has not // yet started because Store.flush() waits for all rwcc transactions to // commit before starting the flush to disk. - KeyValue kv = KeyValueUtil.ensureKeyValue(cell); - KeyValue found = this.snapshot.get(kv); - if (found != null && found.getMvccVersion() == kv.getMvccVersion()) { - this.snapshot.remove(kv); - long sz = heapSizeChange(kv, true); + Cell found = this.snapshot.get(cell); + if (found != null && found.getSequenceId() == cell.getSequenceId()) { + this.snapshot.remove(cell); + long sz = heapSizeChange(cell, true); this.snapshotSize -= sz; } // If the key is in the memstore, delete it. Update this.size. - found = this.kvset.get(kv); - if (found != null && found.getMvccVersion() == kv.getMvccVersion()) { - removeFromKVSet(kv); - long s = heapSizeChange(kv, true); + found = this.cellSet.get(cell); + if (found != null && found.getSequenceId() == cell.getSequenceId()) { + removeFromCellSet(cell); + long s = heapSizeChange(cell, true); this.size.addAndGet(-s); } } @@ -314,20 +313,20 @@ public class DefaultMemStore implements MemStore { @Override public long delete(Cell deleteCell) { long s = 0; - KeyValue toAdd = maybeCloneWithAllocator(KeyValueUtil.ensureKeyValue(deleteCell)); - s += heapSizeChange(toAdd, addToKVSet(toAdd)); + Cell toAdd = maybeCloneWithAllocator(deleteCell); + s += heapSizeChange(toAdd, addToCellSet(toAdd)); timeRangeTracker.includeTimestamp(toAdd); this.size.addAndGet(s); return s; } /** - * @param kv Find the row that comes after this one. If null, we return the + * @param cell Find the row that comes after this one. If null, we return the * first. * @return Next row or null if none found. */ - KeyValue getNextRow(final KeyValue kv) { - return getLowest(getNextRow(kv, this.kvset), getNextRow(kv, this.snapshot)); + Cell getNextRow(final Cell cell) { + return getLowest(getNextRow(cell, this.cellSet), getNextRow(cell, this.snapshot)); } /* @@ -335,7 +334,7 @@ public class DefaultMemStore implements MemStore { * @param b * @return Return lowest of a or b or null if both a and b are null */ - private KeyValue getLowest(final KeyValue a, final KeyValue b) { + private Cell getLowest(final Cell a, final Cell b) { if (a == null) { return b; } @@ -351,17 +350,17 @@ public class DefaultMemStore implements MemStore { * @return Next row or null if none found. If one found, will be a new * KeyValue -- can be destroyed by subsequent calls to this method. */ - private KeyValue getNextRow(final KeyValue key, - final NavigableSet set) { - KeyValue result = null; - SortedSet tail = key == null? set: set.tailSet(key); + private Cell getNextRow(final Cell key, + final NavigableSet set) { + Cell result = null; + SortedSet tail = key == null? set: set.tailSet(key); // Iterate until we fall into the next row; i.e. move off current row - for (KeyValue kv: tail) { - if (comparator.compareRows(kv, key) <= 0) + for (Cell cell: tail) { + if (comparator.compareRows(cell, key) <= 0) continue; // Note: Not suppressing deletes or expired cells. Needs to be handled // by higher up functions. - result = kv; + result = cell; break; } return result; @@ -372,7 +371,7 @@ public class DefaultMemStore implements MemStore { */ @Override public void getRowKeyAtOrBefore(final GetClosestRowBeforeTracker state) { - getRowKeyAtOrBefore(kvset, state); + getRowKeyAtOrBefore(cellSet, state); getRowKeyAtOrBefore(snapshot, state); } @@ -380,7 +379,7 @@ public class DefaultMemStore implements MemStore { * @param set * @param state Accumulates deletes and candidates. */ - private void getRowKeyAtOrBefore(final NavigableSet set, + private void getRowKeyAtOrBefore(final NavigableSet set, final GetClosestRowBeforeTracker state) { if (set.isEmpty()) { return; @@ -401,13 +400,13 @@ public class DefaultMemStore implements MemStore { * @param state * @return True if we found a candidate walking this row. */ - private boolean walkForwardInSingleRow(final SortedSet set, - final KeyValue firstOnRow, final GetClosestRowBeforeTracker state) { + private boolean walkForwardInSingleRow(final SortedSet set, + final Cell firstOnRow, final GetClosestRowBeforeTracker state) { boolean foundCandidate = false; - SortedSet tail = set.tailSet(firstOnRow); + SortedSet tail = set.tailSet(firstOnRow); if (tail.isEmpty()) return foundCandidate; - for (Iterator i = tail.iterator(); i.hasNext();) { - KeyValue kv = i.next(); + for (Iterator i = tail.iterator(); i.hasNext();) { + Cell kv = i.next(); // Did we go beyond the target row? If so break. if (state.isTooFar(kv, firstOnRow)) break; if (state.isExpired(kv)) { @@ -429,17 +428,17 @@ public class DefaultMemStore implements MemStore { * @param set * @param state */ - private void getRowKeyBefore(NavigableSet set, + private void getRowKeyBefore(NavigableSet set, final GetClosestRowBeforeTracker state) { - KeyValue firstOnRow = state.getTargetKey(); + Cell firstOnRow = state.getTargetKey(); for (Member p = memberOfPreviousRow(set, state, firstOnRow); p != null; p = memberOfPreviousRow(p.set, state, firstOnRow)) { // Make sure we don't fall out of our table. - if (!state.isTargetTable(p.kv)) break; + if (!state.isTargetTable(p.cell)) break; // Stop looking if we've exited the better candidate range. - if (!state.isBetterCandidate(p.kv)) break; + if (!state.isBetterCandidate(p.cell)) break; // Make into firstOnRow - firstOnRow = new KeyValue(p.kv.getRowArray(), p.kv.getRowOffset(), p.kv.getRowLength(), + firstOnRow = new KeyValue(p.cell.getRowArray(), p.cell.getRowOffset(), p.cell.getRowLength(), HConstants.LATEST_TIMESTAMP); // If we find something, break; if (walkForwardInSingleRow(p.set, firstOnRow, state)) break; @@ -468,15 +467,14 @@ public class DefaultMemStore implements MemStore { byte[] qualifier, long newValue, long now) { - KeyValue firstKv = KeyValueUtil.createFirstOnRow( - row, family, qualifier); - // Is there a KeyValue in 'snapshot' with the same TS? If so, upgrade the timestamp a bit. - SortedSet snSs = snapshot.tailSet(firstKv); + Cell firstCell = KeyValueUtil.createFirstOnRow(row, family, qualifier); + // Is there a Cell in 'snapshot' with the same TS? If so, upgrade the timestamp a bit. + SortedSet snSs = snapshot.tailSet(firstCell); if (!snSs.isEmpty()) { - KeyValue snKv = snSs.first(); - // is there a matching KV in the snapshot? - if (CellUtil.matchingRow(snKv, firstKv) && CellUtil.matchingQualifier(snKv, firstKv)) { - if (snKv.getTimestamp() == now) { + Cell snc = snSs.first(); + // is there a matching Cell in the snapshot? + if (CellUtil.matchingRow(snc, firstCell) && CellUtil.matchingQualifier(snc, firstCell)) { + if (snc.getTimestamp() == now) { // poop, now += 1; } @@ -486,24 +484,25 @@ public class DefaultMemStore implements MemStore { // logic here: the new ts MUST be at least 'now'. But it could be larger if necessary. // But the timestamp should also be max(now, mostRecentTsInMemstore) - // so we cant add the new KV w/o knowing what's there already, but we also - // want to take this chance to delete some kvs. So two loops (sad) + // so we cant add the new Cell w/o knowing what's there already, but we also + // want to take this chance to delete some cells. So two loops (sad) - SortedSet ss = kvset.tailSet(firstKv); - for (KeyValue kv : ss) { + SortedSet ss = cellSet.tailSet(firstCell); + for (Cell cell : ss) { // if this isnt the row we are interested in, then bail: - if (!CellUtil.matchingColumn(kv, family, qualifier) || !CellUtil.matchingRow(kv, firstKv)) { + if (!CellUtil.matchingColumn(cell, family, qualifier) + || !CellUtil.matchingRow(cell, firstCell)) { break; // rows dont match, bail. } - // if the qualifier matches and it's a put, just RM it out of the kvset. - if (kv.getTypeByte() == KeyValue.Type.Put.getCode() && - kv.getTimestamp() > now && CellUtil.matchingQualifier(firstKv, kv)) { - now = kv.getTimestamp(); + // if the qualifier matches and it's a put, just RM it out of the cellSet. + if (cell.getTypeByte() == KeyValue.Type.Put.getCode() && + cell.getTimestamp() > now && CellUtil.matchingQualifier(firstCell, cell)) { + now = cell.getTimestamp(); } } - // create or update (upsert) a new KeyValue with + // create or update (upsert) a new Cell with // 'now' and a 0 memstoreTS == immediately visible List cells = new ArrayList(1); cells.add(new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue))); @@ -552,37 +551,36 @@ public class DefaultMemStore implements MemStore { * @return change in size of MemStore */ private long upsert(Cell cell, long readpoint) { - // Add the KeyValue to the MemStore + // Add the Cell to the MemStore // Use the internalAdd method here since we (a) already have a lock // and (b) cannot safely use the MSLAB here without potentially // hitting OOME - see TestMemStore.testUpsertMSLAB for a // test that triggers the pathological case if we don't avoid MSLAB // here. - KeyValue kv = KeyValueUtil.ensureKeyValue(cell); - long addedSize = internalAdd(kv); + long addedSize = internalAdd(cell); - // Get the KeyValues for the row/family/qualifier regardless of timestamp. + // Get the Cells for the row/family/qualifier regardless of timestamp. // For this case we want to clean up any other puts - KeyValue firstKv = KeyValueUtil.createFirstOnRow( - kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), - kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), - kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()); - SortedSet ss = kvset.tailSet(firstKv); - Iterator it = ss.iterator(); + Cell firstCell = KeyValueUtil.createFirstOnRow( + cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), + cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), + cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); + SortedSet ss = cellSet.tailSet(firstCell); + Iterator it = ss.iterator(); // versions visible to oldest scanner int versionsVisible = 0; while ( it.hasNext() ) { - KeyValue cur = it.next(); + Cell cur = it.next(); - if (kv == cur) { + if (cell == cur) { // ignore the one just put in continue; } // check that this is the row and column we are interested in, otherwise bail - if (CellUtil.matchingRow(kv, cur) && CellUtil.matchingQualifier(kv, cur)) { + if (CellUtil.matchingRow(cell, cur) && CellUtil.matchingQualifier(cell, cur)) { // only remove Puts that concurrent scanners cannot possibly see if (cur.getTypeByte() == KeyValue.Type.Put.getCode() && - cur.getMvccVersion() <= readpoint) { + cur.getSequenceId() <= readpoint) { if (versionsVisible > 1) { // if we get here we have seen at least one version visible to the oldest scanner, // which means we can prove that no scanner will see this version @@ -607,13 +605,13 @@ public class DefaultMemStore implements MemStore { /* * Immutable data structure to hold member found in set and the set it was - * found in. Include set because it is carrying context. + * found in. Include set because it is carrying context. */ private static class Member { - final KeyValue kv; - final NavigableSet set; - Member(final NavigableSet s, final KeyValue kv) { - this.kv = kv; + final Cell cell; + final NavigableSet set; + Member(final NavigableSet s, final Cell kv) { + this.cell = kv; this.set = s; } } @@ -626,12 +624,12 @@ public class DefaultMemStore implements MemStore { * member in. * @return Null or member of row previous to firstOnRow */ - private Member memberOfPreviousRow(NavigableSet set, - final GetClosestRowBeforeTracker state, final KeyValue firstOnRow) { - NavigableSet head = set.headSet(firstOnRow, false); + private Member memberOfPreviousRow(NavigableSet set, + final GetClosestRowBeforeTracker state, final Cell firstOnRow) { + NavigableSet head = set.headSet(firstOnRow, false); if (head.isEmpty()) return null; - for (Iterator i = head.descendingIterator(); i.hasNext();) { - KeyValue found = i.next(); + for (Iterator i = head.descendingIterator(); i.hasNext();) { + Cell found = i.next(); if (state.isExpired(found)) { i.remove(); continue; @@ -669,32 +667,32 @@ public class DefaultMemStore implements MemStore { * This behaves as if it were a real scanner but does not maintain position. */ protected class MemStoreScanner extends NonLazyKeyValueScanner { - // Next row information for either kvset or snapshot - private KeyValue kvsetNextRow = null; - private KeyValue snapshotNextRow = null; + // Next row information for either cellSet or snapshot + private Cell cellSetNextRow = null; + private Cell snapshotNextRow = null; - // last iterated KVs for kvset and snapshot (to restore iterator state after reseek) - private KeyValue kvsetItRow = null; - private KeyValue snapshotItRow = null; + // last iterated Cells for cellSet and snapshot (to restore iterator state after reseek) + private Cell cellSetItRow = null; + private Cell snapshotItRow = null; // iterator based scanning. - private Iterator kvsetIt; - private Iterator snapshotIt; + private Iterator cellSetIt; + private Iterator snapshotIt; - // The kvset and snapshot at the time of creating this scanner - private KeyValueSkipListSet kvsetAtCreation; - private KeyValueSkipListSet snapshotAtCreation; + // The cellSet and snapshot at the time of creating this scanner + private CellSkipListSet cellSetAtCreation; + private CellSkipListSet snapshotAtCreation; - // the pre-calculated KeyValue to be returned by peek() or next() - private KeyValue theNext; + // the pre-calculated Cell to be returned by peek() or next() + private Cell theNext; // The allocator and snapshot allocator at the time of creating this scanner volatile MemStoreLAB allocatorAtCreation; volatile MemStoreLAB snapshotAllocatorAtCreation; - // A flag represents whether could stop skipping KeyValues for MVCC + // A flag represents whether could stop skipping Cells for MVCC // if have encountered the next row. Only used for reversed scan - private boolean stopSkippingKVsIfNextRow = false; + private boolean stopSkippingCellsIfNextRow = false; private long readPoint; @@ -723,7 +721,7 @@ public class DefaultMemStore implements MemStore { super(); this.readPoint = readPoint; - kvsetAtCreation = kvset; + cellSetAtCreation = cellSet; snapshotAtCreation = snapshot; if (allocator != null) { this.allocatorAtCreation = allocator; @@ -735,17 +733,17 @@ public class DefaultMemStore implements MemStore { } } - private KeyValue getNext(Iterator it) { - KeyValue startKV = theNext; - KeyValue v = null; + private Cell getNext(Iterator it) { + Cell startCell = theNext; + Cell v = null; try { while (it.hasNext()) { v = it.next(); - if (v.getMvccVersion() <= this.readPoint) { + if (v.getSequenceId() <= this.readPoint) { return v; } - if (stopSkippingKVsIfNextRow && startKV != null - && comparator.compareRows(v, startKV) > 0) { + if (stopSkippingCellsIfNextRow && startCell != null + && comparator.compareRows(v, startCell) > 0) { return null; } } @@ -753,11 +751,11 @@ public class DefaultMemStore implements MemStore { return null; } finally { if (v != null) { - // in all cases, remember the last KV iterated to + // in all cases, remember the last Cell iterated to if (it == snapshotIt) { snapshotItRow = v; } else { - kvsetItRow = v; + cellSetItRow = v; } } } @@ -776,27 +774,26 @@ public class DefaultMemStore implements MemStore { close(); return false; } - KeyValue kv = KeyValueUtil.ensureKeyValue(key); // kvset and snapshot will never be null. // if tailSet can't find anything, SortedSet is empty (not null). - kvsetIt = kvsetAtCreation.tailSet(kv).iterator(); - snapshotIt = snapshotAtCreation.tailSet(kv).iterator(); - kvsetItRow = null; + cellSetIt = cellSetAtCreation.tailSet(key).iterator(); + snapshotIt = snapshotAtCreation.tailSet(key).iterator(); + cellSetItRow = null; snapshotItRow = null; - return seekInSubLists(kv); + return seekInSubLists(key); } /** * (Re)initialize the iterators after a seek or a reseek. */ - private synchronized boolean seekInSubLists(KeyValue key){ - kvsetNextRow = getNext(kvsetIt); + private synchronized boolean seekInSubLists(Cell key){ + cellSetNextRow = getNext(cellSetIt); snapshotNextRow = getNext(snapshotIt); // Calculate the next value - theNext = getLowest(kvsetNextRow, snapshotNextRow); + theNext = getLowest(cellSetNextRow, snapshotNextRow); // has data return (theNext != null); @@ -822,37 +819,36 @@ public class DefaultMemStore implements MemStore { get it. So we remember the last keys we iterated to and restore the reseeked set to at least that point. */ - KeyValue kv = KeyValueUtil.ensureKeyValue(key); - kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator(); - snapshotIt = snapshotAtCreation.tailSet(getHighest(kv, snapshotItRow)).iterator(); + cellSetIt = cellSetAtCreation.tailSet(getHighest(key, cellSetItRow)).iterator(); + snapshotIt = snapshotAtCreation.tailSet(getHighest(key, snapshotItRow)).iterator(); - return seekInSubLists(kv); + return seekInSubLists(key); } @Override - public synchronized KeyValue peek() { + public synchronized Cell peek() { //DebugPrint.println(" MS@" + hashCode() + " peek = " + getLowest()); return theNext; } @Override - public synchronized KeyValue next() { + public synchronized Cell next() { if (theNext == null) { return null; } - final KeyValue ret = theNext; + final Cell ret = theNext; // Advance one of the iterators - if (theNext == kvsetNextRow) { - kvsetNextRow = getNext(kvsetIt); + if (theNext == cellSetNextRow) { + cellSetNextRow = getNext(cellSetIt); } else { snapshotNextRow = getNext(snapshotIt); } // Calculate the next value - theNext = getLowest(kvsetNextRow, snapshotNextRow); + theNext = getLowest(cellSetNextRow, snapshotNextRow); //long readpoint = ReadWriteConsistencyControl.getThreadReadPoint(); //DebugPrint.println(" MS@" + hashCode() + " next: " + theNext + " next_next: " + @@ -865,7 +861,7 @@ public class DefaultMemStore implements MemStore { * This uses comparator.compare() to compare the KeyValue using the memstore * comparator. */ - private KeyValue getLowest(KeyValue first, KeyValue second) { + private Cell getLowest(Cell first, Cell second) { if (first == null && second == null) { return null; } @@ -877,11 +873,11 @@ public class DefaultMemStore implements MemStore { } /* - * Returns the higher of the two key values, or null if they are both null. - * This uses comparator.compare() to compare the KeyValue using the memstore + * Returns the higher of the two cells, or null if they are both null. + * This uses comparator.compare() to compare the Cell using the memstore * comparator. */ - private KeyValue getHighest(KeyValue first, KeyValue second) { + private Cell getHighest(Cell first, Cell second) { if (first == null && second == null) { return null; } @@ -893,10 +889,10 @@ public class DefaultMemStore implements MemStore { } public synchronized void close() { - this.kvsetNextRow = null; + this.cellSetNextRow = null; this.snapshotNextRow = null; - this.kvsetIt = null; + this.cellSetIt = null; this.snapshotIt = null; if (allocatorAtCreation != null) { @@ -908,7 +904,7 @@ public class DefaultMemStore implements MemStore { this.snapshotAllocatorAtCreation = null; } - this.kvsetItRow = null; + this.cellSetItRow = null; this.snapshotItRow = null; } @@ -948,47 +944,47 @@ public class DefaultMemStore implements MemStore { */ @Override public synchronized boolean seekToPreviousRow(Cell key) { - KeyValue firstKeyOnRow = KeyValueUtil.createFirstOnRow(key.getRowArray(), key.getRowOffset(), + Cell firstKeyOnRow = KeyValueUtil.createFirstOnRow(key.getRowArray(), key.getRowOffset(), key.getRowLength()); - SortedSet kvHead = kvsetAtCreation.headSet(firstKeyOnRow); - KeyValue kvsetBeforeRow = kvHead.isEmpty() ? null : kvHead.last(); - SortedSet snapshotHead = snapshotAtCreation + SortedSet cellHead = cellSetAtCreation.headSet(firstKeyOnRow); + Cell cellSetBeforeRow = cellHead.isEmpty() ? null : cellHead.last(); + SortedSet snapshotHead = snapshotAtCreation .headSet(firstKeyOnRow); - KeyValue snapshotBeforeRow = snapshotHead.isEmpty() ? null : snapshotHead + Cell snapshotBeforeRow = snapshotHead.isEmpty() ? null : snapshotHead .last(); - KeyValue lastKVBeforeRow = getHighest(kvsetBeforeRow, snapshotBeforeRow); - if (lastKVBeforeRow == null) { + Cell lastCellBeforeRow = getHighest(cellSetBeforeRow, snapshotBeforeRow); + if (lastCellBeforeRow == null) { theNext = null; return false; } - KeyValue firstKeyOnPreviousRow = KeyValueUtil.createFirstOnRow(lastKVBeforeRow.getRowArray(), - lastKVBeforeRow.getRowOffset(), lastKVBeforeRow.getRowLength()); - this.stopSkippingKVsIfNextRow = true; + Cell firstKeyOnPreviousRow = KeyValueUtil.createFirstOnRow(lastCellBeforeRow.getRowArray(), + lastCellBeforeRow.getRowOffset(), lastCellBeforeRow.getRowLength()); + this.stopSkippingCellsIfNextRow = true; seek(firstKeyOnPreviousRow); - this.stopSkippingKVsIfNextRow = false; + this.stopSkippingCellsIfNextRow = false; if (peek() == null || comparator.compareRows(peek(), firstKeyOnPreviousRow) > 0) { - return seekToPreviousRow(lastKVBeforeRow); + return seekToPreviousRow(lastCellBeforeRow); } return true; } @Override public synchronized boolean seekToLastRow() { - KeyValue first = kvsetAtCreation.isEmpty() ? null : kvsetAtCreation + Cell first = cellSetAtCreation.isEmpty() ? null : cellSetAtCreation .last(); - KeyValue second = snapshotAtCreation.isEmpty() ? null + Cell second = snapshotAtCreation.isEmpty() ? null : snapshotAtCreation.last(); - KeyValue higherKv = getHighest(first, second); - if (higherKv == null) { + Cell higherCell = getHighest(first, second); + if (higherCell == null) { return false; } - KeyValue firstKvOnLastRow = KeyValueUtil.createFirstOnRow(higherKv.getRowArray(), - higherKv.getRowOffset(), higherKv.getRowLength()); - if (seek(firstKvOnLastRow)) { + Cell firstCellOnLastRow = KeyValueUtil.createFirstOnRow(higherCell.getRowArray(), + higherCell.getRowOffset(), higherCell.getRowLength()); + if (seek(firstCellOnLastRow)) { return true; } else { - return seekToPreviousRow(higherKv); + return seekToPreviousRow(higherCell); } } @@ -999,19 +995,18 @@ public class DefaultMemStore implements MemStore { public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + ClassSize.ATOMIC_LONG + (2 * ClassSize.TIMERANGE_TRACKER) + - (2 * ClassSize.KEYVALUE_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP)); + (2 * ClassSize.CELL_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP)); /* * Calculate how the MemStore size has changed. Includes overhead of the * backing Map. - * @param kv - * @param notpresent True if the kv was NOT present in the set. + * @param cell + * @param notpresent True if the cell was NOT present in the set. * @return Size */ - static long heapSizeChange(final KeyValue kv, final boolean notpresent) { - return notpresent ? - ClassSize.align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + kv.heapSize()): - 0; + static long heapSizeChange(final Cell cell, final boolean notpresent) { + return notpresent ? ClassSize.align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + + CellUtil.estimatedHeapSizeOf(cell)) : 0; } private long keySize() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java index bf465cafc6c..3b846029ed2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java @@ -35,27 +35,27 @@ import org.apache.hadoop.hbase.regionserver.NonReversedNonLazyKeyValueScanner; */ @InterfaceAudience.Private public class CollectionBackedScanner extends NonReversedNonLazyKeyValueScanner { - final private Iterable data; + final private Iterable data; final KeyValue.KVComparator comparator; - private Iterator iter; - private KeyValue current; + private Iterator iter; + private Cell current; - public CollectionBackedScanner(SortedSet set) { + public CollectionBackedScanner(SortedSet set) { this(set, KeyValue.COMPARATOR); } - public CollectionBackedScanner(SortedSet set, + public CollectionBackedScanner(SortedSet set, KeyValue.KVComparator comparator) { this.comparator = comparator; data = set; init(); } - public CollectionBackedScanner(List list) { + public CollectionBackedScanner(List list) { this(list, KeyValue.COMPARATOR); } - public CollectionBackedScanner(List list, + public CollectionBackedScanner(List list, KeyValue.KVComparator comparator) { Collections.sort(list, comparator); this.comparator = comparator; @@ -64,10 +64,10 @@ public class CollectionBackedScanner extends NonReversedNonLazyKeyValueScanner { } public CollectionBackedScanner(KeyValue.KVComparator comparator, - KeyValue... array) { + Cell... array) { this.comparator = comparator; - List tmp = new ArrayList(array.length); + List tmp = new ArrayList(array.length); Collections.addAll(tmp, array); Collections.sort(tmp, comparator); data = tmp; @@ -82,13 +82,13 @@ public class CollectionBackedScanner extends NonReversedNonLazyKeyValueScanner { } @Override - public KeyValue peek() { + public Cell peek() { return current; } @Override - public KeyValue next() { - KeyValue oldCurrent = current; + public Cell next() { + Cell oldCurrent = current; if(iter.hasNext()){ current = iter.next(); } else { @@ -98,17 +98,17 @@ public class CollectionBackedScanner extends NonReversedNonLazyKeyValueScanner { } @Override - public boolean seek(Cell seekKv) { + public boolean seek(Cell seekCell) { // restart iterator iter = data.iterator(); - return reseek(seekKv); + return reseek(seekCell); } @Override - public boolean reseek(Cell seekKv) { + public boolean reseek(Cell seekCell) { while(iter.hasNext()){ - KeyValue next = iter.next(); - int ret = comparator.compare(next, seekKv); + Cell next = iter.next(); + int ret = comparator.compare(next, seekCell); if(ret >= 0){ current = next; return true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index 7f4ca52024a..2d5060e9203 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -44,10 +44,10 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.LruCachedBlock; import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.regionserver.CellSkipListSet; import org.apache.hadoop.hbase.regionserver.DefaultMemStore; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HStore; -import org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.ClassSize; import org.junit.BeforeClass; @@ -235,10 +235,10 @@ public class TestHeapSize { assertEquals(expected, actual); } - // KeyValueSkipListSet - cl = KeyValueSkipListSet.class; + // CellSkipListSet + cl = CellSkipListSet.class; expected = ClassSize.estimateBase(cl, false); - actual = ClassSize.KEYVALUE_SKIPLIST_SET; + actual = ClassSize.CELL_SKIPLIST_SET; if (expected != actual) { ClassSize.estimateBase(cl, true); assertEquals(expected, actual); @@ -304,14 +304,14 @@ public class TestHeapSize { actual = DefaultMemStore.DEEP_OVERHEAD; expected = ClassSize.estimateBase(cl, false); expected += ClassSize.estimateBase(AtomicLong.class, false); - expected += (2 * ClassSize.estimateBase(KeyValueSkipListSet.class, false)); + expected += (2 * ClassSize.estimateBase(CellSkipListSet.class, false)); expected += (2 * ClassSize.estimateBase(ConcurrentSkipListMap.class, false)); expected += (2 * ClassSize.estimateBase(TimeRangeTracker.class, false)); if(expected != actual) { ClassSize.estimateBase(cl, true); ClassSize.estimateBase(AtomicLong.class, true); - ClassSize.estimateBase(KeyValueSkipListSet.class, true); - ClassSize.estimateBase(KeyValueSkipListSet.class, true); + ClassSize.estimateBase(CellSkipListSet.class, true); + ClassSize.estimateBase(CellSkipListSet.class, true); ClassSize.estimateBase(ConcurrentSkipListMap.class, true); ClassSize.estimateBase(ConcurrentSkipListMap.class, true); ClassSize.estimateBase(TimeRangeTracker.class, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java index 034771cf2ce..333f7e96739 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java @@ -68,7 +68,7 @@ public class TestPrefixTreeEncoding { private static final int NUM_COLS_PER_ROW = 20; private int numBatchesWritten = 0; - private ConcurrentSkipListSet kvset = new ConcurrentSkipListSet( + private ConcurrentSkipListSet kvset = new ConcurrentSkipListSet( KeyValue.COMPARATOR); private static boolean formatRowNum = false; @@ -256,18 +256,18 @@ public class TestPrefixTreeEncoding { private void dumpInputKVSet() { LOG.info("Dumping input keyvalue set in error case:"); - for (KeyValue kv : kvset) { + for (Cell kv : kvset) { System.out.println(kv); } } - private static void generateFixedTestData(ConcurrentSkipListSet kvset, int batchId, + private static void generateFixedTestData(ConcurrentSkipListSet kvset, int batchId, boolean useTags, PrefixTreeCodec encoder, HFileBlockEncodingContext blkEncodingCtx, DataOutputStream userDataStream) throws Exception { generateFixedTestData(kvset, batchId, true, useTags, encoder, blkEncodingCtx, userDataStream); } - private static void generateFixedTestData(ConcurrentSkipListSet kvset, + private static void generateFixedTestData(ConcurrentSkipListSet kvset, int batchId, boolean partial, boolean useTags, PrefixTreeCodec encoder, HFileBlockEncodingContext blkEncodingCtx, DataOutputStream userDataStream) throws Exception { for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) { @@ -286,13 +286,13 @@ public class TestPrefixTreeEncoding { } } encoder.startBlockEncoding(blkEncodingCtx, userDataStream); - for (KeyValue kv : kvset) { + for (Cell kv : kvset) { encoder.encode(kv, blkEncodingCtx, userDataStream); } encoder.endBlockEncoding(blkEncodingCtx, userDataStream, null); } - private static void generateRandomTestData(ConcurrentSkipListSet kvset, + private static void generateRandomTestData(ConcurrentSkipListSet kvset, int batchId, boolean useTags, PrefixTreeCodec encoder, HFileBlockEncodingContext blkEncodingCtx, DataOutputStream userDataStream) throws Exception { Random random = new Random(); @@ -314,7 +314,7 @@ public class TestPrefixTreeEncoding { } } encoder.startBlockEncoding(blkEncodingCtx, userDataStream); - for (KeyValue kv : kvset) { + for (Cell kv : kvset) { encoder.encode(kv, blkEncodingCtx, userDataStream); } encoder.endBlockEncoding(blkEncodingCtx, userDataStream, null); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java similarity index 76% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java index 6f09d2a1c43..9eaed6361f4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueSkipListSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java @@ -21,38 +21,39 @@ package org.apache.hadoop.hbase.regionserver; import java.util.Iterator; import java.util.SortedSet; +import junit.framework.TestCase; + +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.util.Bytes; - -import junit.framework.TestCase; import org.junit.experimental.categories.Category; @Category(SmallTests.class) -public class TestKeyValueSkipListSet extends TestCase { - private final KeyValueSkipListSet kvsls = - new KeyValueSkipListSet(KeyValue.COMPARATOR); +public class TestCellSkipListSet extends TestCase { + private final CellSkipListSet csls = + new CellSkipListSet(KeyValue.COMPARATOR); protected void setUp() throws Exception { super.setUp(); - this.kvsls.clear(); + this.csls.clear(); } public void testAdd() throws Exception { byte [] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); - this.kvsls.add(kv); - assertTrue(this.kvsls.contains(kv)); - assertEquals(1, this.kvsls.size()); - KeyValue first = this.kvsls.first(); + this.csls.add(kv); + assertTrue(this.csls.contains(kv)); + assertEquals(1, this.csls.size()); + Cell first = this.csls.first(); assertTrue(kv.equals(first)); assertTrue(Bytes.equals(kv.getValue(), first.getValue())); // Now try overwritting byte [] overwriteValue = Bytes.toBytes("overwrite"); KeyValue overwrite = new KeyValue(bytes, bytes, bytes, overwriteValue); - this.kvsls.add(overwrite); - assertEquals(1, this.kvsls.size()); - first = this.kvsls.first(); + this.csls.add(overwrite); + assertEquals(1, this.csls.size()); + first = this.csls.first(); assertTrue(Bytes.equals(overwrite.getValue(), first.getValue())); assertFalse(Bytes.equals(overwrite.getValue(), kv.getValue())); } @@ -63,11 +64,11 @@ public class TestKeyValueSkipListSet extends TestCase { byte [] value2 = Bytes.toBytes("2"); final int total = 3; for (int i = 0; i < total; i++) { - this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1)); + this.csls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1)); } // Assert that we added 'total' values and that they are in order int count = 0; - for (KeyValue kv: this.kvsls) { + for (Cell kv: this.csls) { assertEquals("" + count, Bytes.toString(kv.getQualifier())); assertTrue(Bytes.equals(kv.getValue(), value1)); count++; @@ -75,12 +76,12 @@ public class TestKeyValueSkipListSet extends TestCase { assertEquals(total, count); // Now overwrite with a new value. for (int i = 0; i < total; i++) { - this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); + this.csls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); } // Assert that we added 'total' values and that they are in order and that // we are getting back value2 count = 0; - for (KeyValue kv: this.kvsls) { + for (Cell kv: this.csls) { assertEquals("" + count, Bytes.toString(kv.getQualifier())); assertTrue(Bytes.equals(kv.getValue(), value2)); count++; @@ -94,12 +95,12 @@ public class TestKeyValueSkipListSet extends TestCase { byte [] value2 = Bytes.toBytes("2"); final int total = 3; for (int i = 0; i < total; i++) { - this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1)); + this.csls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1)); } // Assert that we added 'total' values and that they are in order int count = 0; - for (Iterator i = this.kvsls.descendingIterator(); i.hasNext();) { - KeyValue kv = i.next(); + for (Iterator i = this.csls.descendingIterator(); i.hasNext();) { + Cell kv = i.next(); assertEquals("" + (total - (count + 1)), Bytes.toString(kv.getQualifier())); assertTrue(Bytes.equals(kv.getValue(), value1)); count++; @@ -107,13 +108,13 @@ public class TestKeyValueSkipListSet extends TestCase { assertEquals(total, count); // Now overwrite with a new value. for (int i = 0; i < total; i++) { - this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); + this.csls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); } // Assert that we added 'total' values and that they are in order and that // we are getting back value2 count = 0; - for (Iterator i = this.kvsls.descendingIterator(); i.hasNext();) { - KeyValue kv = i.next(); + for (Iterator i = this.csls.descendingIterator(); i.hasNext();) { + Cell kv = i.next(); assertEquals("" + (total - (count + 1)), Bytes.toString(kv.getQualifier())); assertTrue(Bytes.equals(kv.getValue(), value2)); count++; @@ -130,22 +131,20 @@ public class TestKeyValueSkipListSet extends TestCase { for (int i = 0; i < total; i++) { KeyValue kv = new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value1); if (i == 1) splitter = kv; - this.kvsls.add(kv); + this.csls.add(kv); } - SortedSet tail = this.kvsls.tailSet(splitter); + SortedSet tail = this.csls.tailSet(splitter); assertEquals(2, tail.size()); - SortedSet head = this.kvsls.headSet(splitter); + SortedSet head = this.csls.headSet(splitter); assertEquals(1, head.size()); // Now ensure that we get back right answer even when we do tail or head. // Now overwrite with a new value. for (int i = 0; i < total; i++) { - this.kvsls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); + this.csls.add(new KeyValue(bytes, bytes, Bytes.toBytes("" + i), value2)); } - tail = this.kvsls.tailSet(splitter); + tail = this.csls.tailSet(splitter); assertTrue(Bytes.equals(tail.first().getValue(), value2)); - head = this.kvsls.headSet(splitter); + head = this.csls.headSet(splitter); assertTrue(Bytes.equals(head.first().getValue(), value2)); } - -} - +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index 383f4244574..cd15249316c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -78,8 +78,8 @@ public class TestDefaultMemStore extends TestCase { byte [] other = Bytes.toBytes("somethingelse"); KeyValue samekey = new KeyValue(bytes, bytes, bytes, other); this.memstore.add(samekey); - KeyValue found = this.memstore.kvset.first(); - assertEquals(1, this.memstore.kvset.size()); + Cell found = this.memstore.cellSet.first(); + assertEquals(1, this.memstore.cellSet.size()); assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found)); } @@ -482,7 +482,7 @@ public class TestDefaultMemStore extends TestCase { m.add(key2); assertTrue("Expected memstore to hold 3 values, actually has " + - m.kvset.size(), m.kvset.size() == 3); + m.cellSet.size(), m.cellSet.size() == 3); } ////////////////////////////////////////////////////////////////////////////// @@ -497,11 +497,11 @@ public class TestDefaultMemStore extends TestCase { // Add more versions to make it a little more interesting. Thread.sleep(1); addRows(this.memstore); - KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY); + Cell closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY); assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty, new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); for (int i = 0; i < ROW_COUNT; i++) { - KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i), + Cell nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i), System.currentTimeMillis())); if (i + 1 == ROW_COUNT) { assertEquals(nr, null); @@ -557,10 +557,10 @@ public class TestDefaultMemStore extends TestCase { memstore.snapshot(); assertEquals(3, memstore.snapshot.size()); //Adding value to "new" memstore - assertEquals(0, memstore.kvset.size()); + assertEquals(0, memstore.cellSet.size()); memstore.add(new KeyValue(row, fam ,qf4, val)); memstore.add(new KeyValue(row, fam ,qf5, val)); - assertEquals(2, memstore.kvset.size()); + assertEquals(2, memstore.cellSet.size()); } ////////////////////////////////////////////////////////////////////////////// @@ -582,7 +582,7 @@ public class TestDefaultMemStore extends TestCase { memstore.add(put2); memstore.add(put3); - assertEquals(3, memstore.kvset.size()); + assertEquals(3, memstore.cellSet.size()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val); memstore.delete(del2); @@ -593,10 +593,10 @@ public class TestDefaultMemStore extends TestCase { expected.add(put2); expected.add(put1); - assertEquals(4, memstore.kvset.size()); + assertEquals(4, memstore.cellSet.size()); int i = 0; - for(KeyValue kv : memstore.kvset) { - assertEquals(expected.get(i++), kv); + for(Cell cell : memstore.cellSet) { + assertEquals(expected.get(i++), cell); } } @@ -616,7 +616,7 @@ public class TestDefaultMemStore extends TestCase { memstore.add(put2); memstore.add(put3); - assertEquals(3, memstore.kvset.size()); + assertEquals(3, memstore.cellSet.size()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val); @@ -629,10 +629,10 @@ public class TestDefaultMemStore extends TestCase { expected.add(put1); - assertEquals(4, memstore.kvset.size()); + assertEquals(4, memstore.cellSet.size()); int i = 0; - for (KeyValue kv: memstore.kvset) { - assertEquals(expected.get(i++), kv); + for (Cell cell: memstore.cellSet) { + assertEquals(expected.get(i++), cell); } } @@ -669,10 +669,10 @@ public class TestDefaultMemStore extends TestCase { - assertEquals(5, memstore.kvset.size()); + assertEquals(5, memstore.cellSet.size()); int i = 0; - for (KeyValue kv: memstore.kvset) { - assertEquals(expected.get(i++), kv); + for (Cell cell: memstore.cellSet) { + assertEquals(expected.get(i++), cell); } } @@ -685,8 +685,8 @@ public class TestDefaultMemStore extends TestCase { memstore.add(new KeyValue(row, fam, qf, ts, val)); KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val); memstore.delete(delete); - assertEquals(2, memstore.kvset.size()); - assertEquals(delete, memstore.kvset.first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } public void testRetainsDeleteVersion() throws IOException { @@ -698,8 +698,8 @@ public class TestDefaultMemStore extends TestCase { "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care"); memstore.delete(delete); - assertEquals(2, memstore.kvset.size()); - assertEquals(delete, memstore.kvset.first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } public void testRetainsDeleteColumn() throws IOException { // add a put to memstore @@ -710,8 +710,8 @@ public class TestDefaultMemStore extends TestCase { KeyValue.Type.DeleteColumn, "dont-care"); memstore.delete(delete); - assertEquals(2, memstore.kvset.size()); - assertEquals(delete, memstore.kvset.first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } public void testRetainsDeleteFamily() throws IOException { // add a put to memstore @@ -722,8 +722,8 @@ public class TestDefaultMemStore extends TestCase { KeyValue.Type.DeleteFamily, "dont-care"); memstore.delete(delete); - assertEquals(2, memstore.kvset.size()); - assertEquals(delete, memstore.kvset.first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } //////////////////////////////////// diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 2e63f1a8636..83aaec164cd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -2188,10 +2188,10 @@ public class TestHRegion { // This is kinda hacky, but better than nothing... long now = System.currentTimeMillis(); DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore; - KeyValue firstKv = memstore.kvset.first(); - assertTrue(firstKv.getTimestamp() <= now); - now = firstKv.getTimestamp(); - for (Cell cell : memstore.kvset) { + Cell firstCell = memstore.cellSet.first(); + assertTrue(firstCell.getTimestamp() <= now); + now = firstCell.getTimestamp(); + for (Cell cell : memstore.cellSet) { assertTrue(cell.getTimestamp() <= now); now = cell.getTimestamp(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java index 850cd94c7c0..4d4b4068a4a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java @@ -70,18 +70,18 @@ public class TestKeyValueHeap extends HBaseTestCase { //1. The "smallest" KeyValue is in the same scanners as current //2. Current scanner gets empty - List l1 = new ArrayList(); + List l1 = new ArrayList(); l1.add(new KeyValue(row1, fam1, col5, data)); l1.add(new KeyValue(row2, fam1, col1, data)); l1.add(new KeyValue(row2, fam1, col2, data)); scanners.add(new Scanner(l1)); - List l2 = new ArrayList(); + List l2 = new ArrayList(); l2.add(new KeyValue(row1, fam1, col1, data)); l2.add(new KeyValue(row1, fam1, col2, data)); scanners.add(new Scanner(l2)); - List l3 = new ArrayList(); + List l3 = new ArrayList(); l3.add(new KeyValue(row1, fam1, col3, data)); l3.add(new KeyValue(row1, fam1, col4, data)); l3.add(new KeyValue(row1, fam2, col1, data)); @@ -132,18 +132,18 @@ public class TestKeyValueHeap extends HBaseTestCase { //1. Seek KeyValue that is not in scanner //2. Check that smallest that is returned from a seek is correct - List l1 = new ArrayList(); + List l1 = new ArrayList(); l1.add(new KeyValue(row1, fam1, col5, data)); l1.add(new KeyValue(row2, fam1, col1, data)); l1.add(new KeyValue(row2, fam1, col2, data)); scanners.add(new Scanner(l1)); - List l2 = new ArrayList(); + List l2 = new ArrayList(); l2.add(new KeyValue(row1, fam1, col1, data)); l2.add(new KeyValue(row1, fam1, col2, data)); scanners.add(new Scanner(l2)); - List l3 = new ArrayList(); + List l3 = new ArrayList(); l3.add(new KeyValue(row1, fam1, col3, data)); l3.add(new KeyValue(row1, fam1, col4, data)); l3.add(new KeyValue(row1, fam2, col1, data)); @@ -178,18 +178,18 @@ public class TestKeyValueHeap extends HBaseTestCase { public void testScannerLeak() throws IOException { // Test for unclosed scanners (HBASE-1927) - List l1 = new ArrayList(); + List l1 = new ArrayList(); l1.add(new KeyValue(row1, fam1, col5, data)); l1.add(new KeyValue(row2, fam1, col1, data)); l1.add(new KeyValue(row2, fam1, col2, data)); scanners.add(new Scanner(l1)); - List l2 = new ArrayList(); + List l2 = new ArrayList(); l2.add(new KeyValue(row1, fam1, col1, data)); l2.add(new KeyValue(row1, fam1, col2, data)); scanners.add(new Scanner(l2)); - List l3 = new ArrayList(); + List l3 = new ArrayList(); l3.add(new KeyValue(row1, fam1, col3, data)); l3.add(new KeyValue(row1, fam1, col4, data)); l3.add(new KeyValue(row1, fam2, col1, data)); @@ -197,7 +197,7 @@ public class TestKeyValueHeap extends HBaseTestCase { l3.add(new KeyValue(row2, fam1, col3, data)); scanners.add(new Scanner(l3)); - List l4 = new ArrayList(); + List l4 = new ArrayList(); scanners.add(new Scanner(l4)); //Creating KeyValueHeap @@ -212,10 +212,10 @@ public class TestKeyValueHeap extends HBaseTestCase { private static class Scanner extends CollectionBackedScanner { private Iterator iter; - private KeyValue current; + private Cell current; private boolean closed = false; - public Scanner(List list) { + public Scanner(List list) { super(list); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java index d73bdb85d55..80133c7be4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java @@ -118,10 +118,10 @@ public class TestMemStoreChunkPool { assertEquals(3, memstore.snapshot.size()); // Adding value to "new" memstore - assertEquals(0, memstore.kvset.size()); + assertEquals(0, memstore.cellSet.size()); memstore.add(new KeyValue(row, fam, qf4, val)); memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.kvset.size()); + assertEquals(2, memstore.cellSet.size()); memstore.clearSnapshot(snapshot.getId()); int chunkCount = chunkPool.getPoolSize(); @@ -155,10 +155,10 @@ public class TestMemStoreChunkPool { assertEquals(3, memstore.snapshot.size()); // Adding value to "new" memstore - assertEquals(0, memstore.kvset.size()); + assertEquals(0, memstore.cellSet.size()); memstore.add(new KeyValue(row, fam, qf4, val)); memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.kvset.size()); + assertEquals(2, memstore.cellSet.size()); // opening scanner before clear the snapshot List scanners = memstore.getScanners(0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index 99834ed0b4c..acfe4ff1ab5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -525,7 +525,7 @@ public class TestStore { this.store.snapshot(); flushStore(store, id++); Assert.assertEquals(storeFilessize, this.store.getStorefiles().size()); - Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).kvset.size()); + Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).cellSet.size()); } private void assertCheck() { @@ -570,7 +570,7 @@ public class TestStore { flushStore(store, id++); Assert.assertEquals(1, this.store.getStorefiles().size()); // from the one we inserted up there, and a new one - Assert.assertEquals(2, ((DefaultMemStore)this.store.memstore).kvset.size()); + Assert.assertEquals(2, ((DefaultMemStore)this.store.memstore).cellSet.size()); // how many key/values for this row are there? Get get = new Get(row); @@ -644,8 +644,8 @@ public class TestStore { } long computedSize=0; - for (KeyValue kv : ((DefaultMemStore)this.store.memstore).kvset) { - long kvsize = DefaultMemStore.heapSizeChange(kv, true); + for (Cell cell : ((DefaultMemStore)this.store.memstore).cellSet) { + long kvsize = DefaultMemStore.heapSizeChange(cell, true); //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize()); computedSize += kvsize; } @@ -676,7 +676,7 @@ public class TestStore { // then flush. flushStore(store, id++); Assert.assertEquals(1, this.store.getStorefiles().size()); - Assert.assertEquals(1, ((DefaultMemStore)this.store.memstore).kvset.size()); + Assert.assertEquals(1, ((DefaultMemStore)this.store.memstore).cellSet.size()); // now increment again: newValue += 1;