HBASE-2520 Cleanup arrays vs Lists of scanners

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@945418 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2010-05-17 23:11:13 +00:00
parent 0491cb140a
commit e553b2c34a
13 changed files with 123 additions and 136 deletions

View File

@ -595,6 +595,7 @@ Release 0.21.0 - Unreleased
(Nicolas Spiegelberg via Stack) (Nicolas Spiegelberg via Stack)
HBASE-2340 Add end-to-end test of sync/flush (Forward-port from branch) HBASE-2340 Add end-to-end test of sync/flush (Forward-port from branch)
HBASE-2555 Get rid of HColumnDescriptor.MAPFILE_INDEX_INTERVAL HBASE-2555 Get rid of HColumnDescriptor.MAPFILE_INDEX_INTERVAL
HBASE-2520 Cleanup arrays vs Lists of scanners (Todd Lipcon via Stack)
NEW FEATURES NEW FEATURES

View File

@ -1962,8 +1962,7 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
Store store = stores.get(entry.getKey()); Store store = stores.get(entry.getKey());
scanners.add(store.getScanner(theScan, entry.getValue())); scanners.add(store.getScanner(theScan, entry.getValue()));
} }
this.storeHeap = this.storeHeap = new KeyValueHeap(scanners, comparator);
new KeyValueHeap(scanners.toArray(new KeyValueScanner[0]), comparator);
} }
private void resetFilters() { private void resetFilters() {

View File

@ -51,9 +51,9 @@ public class KeyValueHeap implements KeyValueScanner, InternalScanner {
* @param scanners * @param scanners
* @param comparator * @param comparator
*/ */
public KeyValueHeap(KeyValueScanner [] scanners, KVComparator comparator) { public KeyValueHeap(List<KeyValueScanner> scanners, KVComparator comparator) {
this.comparator = new KVScannerComparator(comparator); this.comparator = new KVScannerComparator(comparator);
this.heap = new PriorityQueue<KeyValueScanner>(scanners.length, this.heap = new PriorityQueue<KeyValueScanner>(scanners.size(),
this.comparator); this.comparator);
for (KeyValueScanner scanner : scanners) { for (KeyValueScanner scanner : scanners) {
if (scanner.peek() != null) { if (scanner.peek() != null) {
@ -210,4 +210,4 @@ public class KeyValueHeap implements KeyValueScanner, InternalScanner {
public PriorityQueue<KeyValueScanner> getHeap() { public PriorityQueue<KeyValueScanner> getHeap() {
return this.heap; return this.heap;
} }
} }

View File

@ -25,6 +25,7 @@ import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean; import java.lang.management.RuntimeMXBean;
import java.rmi.UnexpectedException; import java.rmi.UnexpectedException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.NavigableSet; import java.util.NavigableSet;
@ -379,12 +380,11 @@ public class MemStore implements HeapSize {
/** /**
* @return scanner on memstore and snapshot in this order. * @return scanner on memstore and snapshot in this order.
*/ */
KeyValueScanner [] getScanners() { List<KeyValueScanner> getScanners() {
this.lock.readLock().lock(); this.lock.readLock().lock();
try { try {
KeyValueScanner [] scanners = new KeyValueScanner[1]; return Collections.<KeyValueScanner>singletonList(
scanners[0] = new MemStoreScanner(); new MemStoreScanner());
return scanners;
} finally { } finally {
this.lock.readLock().unlock(); this.lock.readLock().unlock();
} }

View File

@ -38,7 +38,7 @@ public class MinorCompactingStoreScanner implements KeyValueScanner, InternalSca
private KeyValue.KVComparator comparator; private KeyValue.KVComparator comparator;
MinorCompactingStoreScanner(Store store, MinorCompactingStoreScanner(Store store,
KeyValueScanner [] scanners) { List<KeyValueScanner> scanners) {
comparator = store.comparator; comparator = store.comparator;
KeyValue firstKv = KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW); KeyValue firstKv = KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW);
for (KeyValueScanner scanner : scanners ) { for (KeyValueScanner scanner : scanners ) {
@ -49,7 +49,7 @@ public class MinorCompactingStoreScanner implements KeyValueScanner, InternalSca
} }
MinorCompactingStoreScanner(String cfName, KeyValue.KVComparator comparator, MinorCompactingStoreScanner(String cfName, KeyValue.KVComparator comparator,
KeyValueScanner [] scanners) { List<KeyValueScanner> scanners) {
this.comparator = comparator; this.comparator = comparator;
KeyValue firstKv = KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW); KeyValue firstKv = KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW);

View File

@ -880,16 +880,8 @@ public class Store implements HConstants, HeapSize {
final boolean majorCompaction, final long maxId) final boolean majorCompaction, final long maxId)
throws IOException { throws IOException {
// For each file, obtain a scanner: // For each file, obtain a scanner:
KeyValueScanner [] scanners = new KeyValueScanner[filesToCompact.size()]; List<KeyValueScanner> scanners = StoreFileScanner.getScannersForStoreFiles(
for (int i = 0; i < filesToCompact.size(); ++i) { filesToCompact, false, false);
Reader r = filesToCompact.get(i).getReader();
if (r == null) {
LOG.warn("StoreFile " + filesToCompact.get(i) + " has a null Reader");
continue;
}
// Instantiate HFile.Reader.Scanner to not cache blocks and not use pread
scanners[i] = new StoreFileScanner(r.getScanner(false, false));
}
// Make the instantiation lazy in case compaction produces no product; i.e. // Make the instantiation lazy in case compaction produces no product; i.e.
// where all source cells are expired or deleted. // where all source cells are expired or deleted.
@ -904,9 +896,7 @@ public class Store implements HConstants, HeapSize {
// since scanner.next() can return 'false' but still be delivering data, // since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop. // we have to use a do/while loop.
ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(); ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
boolean more = true; while (scanner.next(kvs)) {
while (more) {
more = scanner.next(kvs);
// output to writer: // output to writer:
for (KeyValue kv : kvs) { for (KeyValue kv : kvs) {
if (writer == null) { if (writer == null) {
@ -1003,6 +993,7 @@ public class Store implements HConstants, HeapSize {
} }
// WARN ugly hack here, but necessary sadly. // WARN ugly hack here, but necessary sadly.
// TODO why is this necessary? need a comment here if it's unintuitive!
ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC()); ReadWriteConsistencyControl.resetThreadReadPoint(region.getRWCC());
// Tell observers that list of StoreFiles has changed. // Tell observers that list of StoreFiles has changed.
@ -1359,7 +1350,7 @@ public class Store implements HConstants, HeapSize {
return size; return size;
} }
/* /**
* Datastructure that holds size and row to split a file around. * Datastructure that holds size and row to split a file around.
* TODO: Take a KeyValue rather than row. * TODO: Take a KeyValue rather than row.
*/ */

View File

@ -20,15 +20,23 @@
package org.apache.hadoop.hbase.regionserver; package org.apache.hadoop.hbase.regionserver;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/** /**
* A KeyValue scanner that iterates over a single HFile * A KeyValue scanner that iterates over a single HFile
*/ */
class StoreFileScanner implements KeyValueScanner { class StoreFileScanner implements KeyValueScanner {
static final Log LOG = LogFactory.getLog(Store.class);
private HFileScanner hfs; private HFileScanner hfs;
private KeyValue cur = null; private KeyValue cur = null;
@ -37,10 +45,33 @@ class StoreFileScanner implements KeyValueScanner {
* Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner} * Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner}
* @param hfs HFile scanner * @param hfs HFile scanner
*/ */
public StoreFileScanner(HFileScanner hfs) { private StoreFileScanner(HFileScanner hfs) {
this.hfs = hfs; this.hfs = hfs;
} }
/**
* Return an array of scanners corresponding to the given
* set of store files.
*/
public static List<KeyValueScanner> getScannersForStoreFiles(
Collection<StoreFile> filesToCompact,
boolean cacheBlocks,
boolean usePread) {
List<KeyValueScanner> scanners =
new ArrayList<KeyValueScanner>(filesToCompact.size());
for (StoreFile file : filesToCompact) {
Reader r = file.getReader();
if (r == null) {
// TODO why can this happen? this seems like something worth
// throwing an exception over!
LOG.error("StoreFile " + file + " has a null Reader");
continue;
}
scanners.add(new StoreFileScanner(r.getScanner(cacheBlocks, usePread)));
}
return scanners;
}
public String toString() { public String toString() {
return "StoreFileScanner[" + hfs.toString() + ", cur=" + cur + "]"; return "StoreFileScanner[" + hfs.toString() + ", cur=" + cur + "]";
} }
@ -104,4 +135,4 @@ class StoreFileScanner implements KeyValueScanner {
// Seeked to the exact key // Seeked to the exact key
return true; return true;
} }
} }

View File

@ -24,8 +24,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -72,8 +70,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
} }
// Combine all seeked scanners with a heap // Combine all seeked scanners with a heap
heap = new KeyValueHeap( heap = new KeyValueHeap(scanners, store.comparator);
scanners.toArray(new KeyValueScanner[scanners.size()]), store.comparator);
this.store.addChangedReaderObserver(this); this.store.addChangedReaderObserver(this);
} }
@ -86,7 +83,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
* @param scan the spec * @param scan the spec
* @param scanners ancilliary scanners * @param scanners ancilliary scanners
*/ */
StoreScanner(Store store, Scan scan, KeyValueScanner [] scanners) { StoreScanner(Store store, Scan scan, List<KeyValueScanner> scanners) {
this.store = store; this.store = store;
this.cacheBlocks = false; this.cacheBlocks = false;
this.isGet = false; this.isGet = false;
@ -107,7 +104,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
StoreScanner(final Scan scan, final byte [] colFamily, final long ttl, StoreScanner(final Scan scan, final byte [] colFamily, final long ttl,
final KeyValue.KVComparator comparator, final KeyValue.KVComparator comparator,
final NavigableSet<byte[]> columns, final NavigableSet<byte[]> columns,
final KeyValueScanner [] scanners) { final List<KeyValueScanner> scanners) {
this.store = null; this.store = null;
this.isGet = false; this.isGet = false;
this.cacheBlocks = scan.getCacheBlocks(); this.cacheBlocks = scan.getCacheBlocks();
@ -125,11 +122,13 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
* @return List of scanners ordered properly. * @return List of scanners ordered properly.
*/ */
private List<KeyValueScanner> getScanners() { private List<KeyValueScanner> getScanners() {
List<KeyValueScanner> scanners = getStoreFileScanners(); // First the store file scanners
KeyValueScanner [] memstorescanners = this.store.memstore.getScanners(); Map<Long, StoreFile> map = this.store.getStorefiles().descendingMap();
for (int i = memstorescanners.length - 1; i >= 0; i--) { List<KeyValueScanner> scanners =
scanners.add(memstorescanners[i]); StoreFileScanner.getScannersForStoreFiles(map.values(),
} cacheBlocks, isGet);
// Then the memstore scanners
scanners.addAll(this.store.memstore.getScanners());
return scanners; return scanners;
} }
@ -231,27 +230,6 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
return next(outResult, -1); return next(outResult, -1);
} }
private List<KeyValueScanner> getStoreFileScanners() {
List<HFileScanner> s =
new ArrayList<HFileScanner>(this.store.getStorefilesCount());
Map<Long, StoreFile> map = this.store.getStorefiles().descendingMap();
for(StoreFile sf : map.values()) {
HFile.Reader r = sf.getReader();
if (r == null) {
LOG.warn("StoreFile " + sf + " has null Reader");
continue;
}
// If isGet, use pread, else false, dont use pread
s.add(r.getScanner(this.cacheBlocks, isGet));
}
List<KeyValueScanner> scanners =
new ArrayList<KeyValueScanner>(s.size()+1);
for(HFileScanner hfs : s) {
scanners.add(new StoreFileScanner(hfs));
}
return scanners;
}
// Implementation of ChangedReadersObserver // Implementation of ChangedReadersObserver
public synchronized void updateReaders() throws IOException { public synchronized void updateReaders() throws IOException {
if (this.closing) return; if (this.closing) return;
@ -269,8 +247,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
} }
// Combine all seeked scanners with a heap // Combine all seeked scanners with a heap
heap = new KeyValueHeap( heap = new KeyValueHeap(scanners, store.comparator);
scanners.toArray(new KeyValueScanner[scanners.size()]), store.comparator);
// Reset the state of the Query Matcher and set to top row // Reset the state of the Query Matcher and set to top row
matcher.reset(); matcher.reset();

View File

@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.KeyValue;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.Collections; import java.util.Collections;
import java.util.List;
/** /**
* A fixture that implements and presents a KeyValueScanner. * A fixture that implements and presents a KeyValueScanner.
@ -50,6 +51,15 @@ public class KeyValueScanFixture implements KeyValueScanner {
Collections.sort(data, this.comparator); Collections.sort(data, this.comparator);
} }
public static List<KeyValueScanner> scanFixture(KeyValue[] ... kvArrays) {
ArrayList<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
for (KeyValue [] kvs : kvArrays) {
scanners.add(new KeyValueScanFixture(KeyValue.COMPARATOR, kvs));
}
return scanners;
}
@Override @Override
public KeyValue peek() { public KeyValue peek() {
return this.current; return this.current;

View File

@ -35,7 +35,7 @@ public class TestKeyValueHeap extends HBaseTestCase
implements HConstants { implements HConstants {
private static final boolean PRINT = false; private static final boolean PRINT = false;
List<Scanner> scanners = new ArrayList<Scanner>(); List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
private byte [] row1; private byte [] row1;
private byte [] fam1; private byte [] fam1;
@ -102,7 +102,7 @@ implements HConstants {
//Creating KeyValueHeap //Creating KeyValueHeap
KeyValueHeap kvh = KeyValueHeap kvh =
new KeyValueHeap(scanners.toArray(new Scanner[0]), KeyValue.COMPARATOR); new KeyValueHeap(scanners, KeyValue.COMPARATOR);
List<KeyValue> actual = new ArrayList<KeyValue>(); List<KeyValue> actual = new ArrayList<KeyValue>();
while(kvh.peek() != null){ while(kvh.peek() != null){
@ -155,7 +155,7 @@ implements HConstants {
//Creating KeyValueHeap //Creating KeyValueHeap
KeyValueHeap kvh = KeyValueHeap kvh =
new KeyValueHeap(scanners.toArray(new Scanner[0]), KeyValue.COMPARATOR); new KeyValueHeap(scanners, KeyValue.COMPARATOR);
KeyValue seekKv = new KeyValue(row2, fam1, null, null); KeyValue seekKv = new KeyValue(row2, fam1, null, null);
kvh.seek(seekKv); kvh.seek(seekKv);
@ -200,13 +200,12 @@ implements HConstants {
scanners.add(new Scanner(l4)); scanners.add(new Scanner(l4));
//Creating KeyValueHeap //Creating KeyValueHeap
KeyValueHeap kvh = KeyValueHeap kvh = new KeyValueHeap(scanners, KeyValue.COMPARATOR);
new KeyValueHeap(scanners.toArray(new Scanner[0]), KeyValue.COMPARATOR);
while(kvh.next() != null); while(kvh.next() != null);
for(Scanner scanner : scanners) { for(KeyValueScanner scanner : scanners) {
assertTrue(scanner.isClosed()); assertTrue(((Scanner)scanner).isClosed());
} }
} }

View File

@ -76,7 +76,7 @@ public class TestMemStore extends TestCase {
*/ */
public void testScanAcrossSnapshot() throws IOException { public void testScanAcrossSnapshot() throws IOException {
int rowCount = addRows(this.memstore); int rowCount = addRows(this.memstore);
KeyValueScanner [] memstorescanners = this.memstore.getScanners(); List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
Scan scan = new Scan(); Scan scan = new Scan();
List<KeyValue> result = new ArrayList<KeyValue>(); List<KeyValue> result = new ArrayList<KeyValue>();
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
@ -95,8 +95,8 @@ public class TestMemStore extends TestCase {
s.close(); s.close();
} }
assertEquals(rowCount, count); assertEquals(rowCount, count);
for (int i = 0; i < memstorescanners.length; i++) { for (KeyValueScanner scanner : memstorescanners) {
memstorescanners[0].close(); scanner.close();
} }
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
@ -123,8 +123,8 @@ public class TestMemStore extends TestCase {
s.close(); s.close();
} }
assertEquals(rowCount, count); assertEquals(rowCount, count);
for (int i = 0; i < memstorescanners.length; i++) { for (KeyValueScanner scanner : memstorescanners) {
memstorescanners[0].close(); scanner.close();
} }
memstorescanners = this.memstore.getScanners(); memstorescanners = this.memstore.getScanners();
// Assert that new values are seen in kvset as we scan. // Assert that new values are seen in kvset as we scan.
@ -190,9 +190,9 @@ public class TestMemStore extends TestCase {
private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) { private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) {
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
KeyValueScanner[] memstorescanners = this.memstore.getScanners(); List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
assertEquals(1, memstorescanners.length); assertEquals(1, memstorescanners.size());
final KeyValueScanner scanner = memstorescanners[0]; final KeyValueScanner scanner = memstorescanners.get(0);
scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW)); scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
assertEquals(kv1, scanner.next()); assertEquals(kv1, scanner.next());
assertEquals(kv2, scanner.next()); assertEquals(kv2, scanner.next());
@ -224,14 +224,14 @@ public class TestMemStore extends TestCase {
memstore.add(kv1); memstore.add(kv1);
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
KeyValueScanner[] s = this.memstore.getScanners(); KeyValueScanner s = this.memstore.getScanners().get(0);
assertScannerResults(s[0], new KeyValue[]{}); assertScannerResults(s, new KeyValue[]{});
rwcc.completeMemstoreInsert(w); rwcc.completeMemstoreInsert(w);
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
s = this.memstore.getScanners(); s = this.memstore.getScanners().get(0);
assertScannerResults(s[0], new KeyValue[]{kv1}); assertScannerResults(s, new KeyValue[]{kv1});
w = rwcc.beginMemstoreInsert(); w = rwcc.beginMemstoreInsert();
KeyValue kv2 = new KeyValue(row, f, q2, v); KeyValue kv2 = new KeyValue(row, f, q2, v);
@ -239,14 +239,14 @@ public class TestMemStore extends TestCase {
memstore.add(kv2); memstore.add(kv2);
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
s = this.memstore.getScanners(); s = this.memstore.getScanners().get(0);
assertScannerResults(s[0], new KeyValue[]{kv1}); assertScannerResults(s, new KeyValue[]{kv1});
rwcc.completeMemstoreInsert(w); rwcc.completeMemstoreInsert(w);
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
s = this.memstore.getScanners(); s = this.memstore.getScanners().get(0);
assertScannerResults(s[0], new KeyValue[]{kv1, kv2}); assertScannerResults(s, new KeyValue[]{kv1, kv2});
} }
private static class ReadOwnWritesTester extends Thread { private static class ReadOwnWritesTester extends Thread {
@ -300,7 +300,7 @@ public class TestMemStore extends TestCase {
// Assert that we can read back // Assert that we can read back
ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
KeyValueScanner s = this.memstore.getScanners()[0]; KeyValueScanner s = this.memstore.getScanners().get(0);
s.seek(kv); s.seek(kv);
KeyValue ret = s.next(); KeyValue ret = s.next();
@ -429,7 +429,7 @@ public class TestMemStore extends TestCase {
InternalScanner scanner = InternalScanner scanner =
new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY, new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY,
Integer.MAX_VALUE, this.memstore.comparator, null, Integer.MAX_VALUE, this.memstore.comparator, null,
new KeyValueScanner[]{memstore.getScanners()[0]}); memstore.getScanners());
List<KeyValue> results = new ArrayList<KeyValue>(); List<KeyValue> results = new ArrayList<KeyValue>();
for (int i = 0; scanner.next(results); i++) { for (int i = 0; scanner.next(results); i++) {
int rowId = startRowId + i; int rowId = startRowId + i;
@ -857,8 +857,7 @@ public class TestMemStore extends TestCase {
static void doScan(MemStore ms, int iteration) { static void doScan(MemStore ms, int iteration) {
long nanos = System.nanoTime(); long nanos = System.nanoTime();
KeyValueScanner [] ss = ms.getScanners(); KeyValueScanner s = ms.getScanners().get(0);
KeyValueScanner s = ss[0];
s.seek(KeyValue.createFirstOnRow(new byte[]{})); s.seek(KeyValue.createFirstOnRow(new byte[]{}));
System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000); System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);

View File

@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.KeyValueTestUtil;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
public class TestMinorCompactingStoreScanner extends TestCase { public class TestMinorCompactingStoreScanner extends TestCase {
@ -45,9 +46,8 @@ public class TestMinorCompactingStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
InternalScanner scan = InternalScanner scan =
new MinorCompactingStoreScanner("cf", KeyValue.COMPARATOR, scanners); new MinorCompactingStoreScanner("cf", KeyValue.COMPARATOR, scanners);
List<KeyValue> results = new ArrayList<KeyValue>(); List<KeyValue> results = new ArrayList<KeyValue>();
@ -77,9 +77,7 @@ public class TestMinorCompactingStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.Delete, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.Delete, "dont-care"),
KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
InternalScanner scan = InternalScanner scan =
new MinorCompactingStoreScanner("cf", KeyValue.COMPARATOR, scanners); new MinorCompactingStoreScanner("cf", KeyValue.COMPARATOR, scanners);
List<KeyValue> results = new ArrayList<KeyValue>(); List<KeyValue> results = new ArrayList<KeyValue>();

View File

@ -28,9 +28,12 @@ import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.NavigableSet; import java.util.NavigableSet;
import java.util.TreeSet; import java.util.TreeSet;
import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture;
public class TestStoreScanner extends TestCase { public class TestStoreScanner extends TestCase {
private static final String CF_STR = "cf"; private static final String CF_STR = "cf";
@ -60,9 +63,10 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = Arrays.<KeyValueScanner>asList(
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs) new KeyValueScanner[] {
}; new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
});
Scan scanSpec = new Scan(Bytes.toBytes(r1)); Scan scanSpec = new Scan(Bytes.toBytes(r1));
scanSpec.setTimeRange(0, 6); scanSpec.setTimeRange(0, 6);
scanSpec.setMaxVersions(); scanSpec.setMaxVersions();
@ -110,9 +114,10 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = Arrays.asList(
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs) new KeyValueScanner[] {
}; new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
});
Scan scanSpec = new Scan(Bytes.toBytes("R1")); Scan scanSpec = new Scan(Bytes.toBytes("R1"));
// this only uses maxVersions (default=1) and TimeRange (default=all) // this only uses maxVersions (default=1) and TimeRange (default=all)
@ -140,10 +145,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR,
kvs)
};
Scan scanSpec = new Scan(Bytes.toBytes("R1")); Scan scanSpec = new Scan(Bytes.toBytes("R1"));
// this only uses maxVersions (default=1) and TimeRange (default=all) // this only uses maxVersions (default=1) and TimeRange (default=all)
@ -175,9 +177,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
Scan scanSpec = new Scan(Bytes.toBytes("R1")); Scan scanSpec = new Scan(Bytes.toBytes("R1"));
StoreScanner scan = StoreScanner scan =
new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
@ -198,9 +198,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
Scan scanSpec = new Scan(Bytes.toBytes("R1")); Scan scanSpec = new Scan(Bytes.toBytes("R1"));
StoreScanner scan = StoreScanner scan =
new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
@ -228,10 +226,8 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs1),
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs2)
};
StoreScanner scan = StoreScanner scan =
new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
getCols("a"), scanners); getCols("a"), scanners);
@ -255,10 +251,8 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs1),
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs2)
};
Scan scanSpec = new Scan(Bytes.toBytes("R1")).setMaxVersions(2); Scan scanSpec = new Scan(Bytes.toBytes("R1")).setMaxVersions(2);
StoreScanner scan = StoreScanner scan =
new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
@ -276,9 +270,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
StoreScanner scan = StoreScanner scan =
new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
null, scanners); null, scanners);
@ -307,9 +299,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no KeyValueTestUtil.create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
StoreScanner scan = StoreScanner scan =
new StoreScanner(new Scan().setMaxVersions(2), CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(new Scan().setMaxVersions(2), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
null, scanners); null, scanners);
@ -337,9 +327,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
StoreScanner scan = StoreScanner scan =
new StoreScanner(new Scan().setMaxVersions(Integer.MAX_VALUE), CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(new Scan().setMaxVersions(Integer.MAX_VALUE), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
null, scanners); null, scanners);
@ -360,9 +348,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs),
};
StoreScanner scan = StoreScanner scan =
new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
null, scanners); null, scanners);
@ -385,9 +371,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
StoreScanner scan = StoreScanner scan =
new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR, new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
getCols("a", "d"), scanners); getCols("a", "d"), scanners);
@ -423,9 +407,7 @@ public class TestStoreScanner extends TestCase {
KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care") KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
}; };
KeyValueScanner [] scanners = new KeyValueScanner[] { List<KeyValueScanner> scanners = scanFixture(kvs);
new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
};
Scan scan = new Scan(); Scan scan = new Scan();
scan.setMaxVersions(1); scan.setMaxVersions(1);
StoreScanner scanner = StoreScanner scanner =