From d92252312c9c42b598d3e79fb805912e9e061e43 Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Sun, 24 Oct 2010 22:36:27 +0000 Subject: [PATCH] HBASE-3082 For ICV gets, first look in MemStore before reading StoreFiles (Prakash via jgray) git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1026910 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + .../hadoop/hbase/regionserver/HRegion.java | 86 ++++++++++++++++++- .../hbase/regionserver/InternalScan.java | 78 +++++++++++++++++ .../hbase/regionserver/StoreScanner.java | 30 +++++-- 4 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java diff --git a/CHANGES.txt b/CHANGES.txt index 5badd1367fc..78003b21b14 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1037,6 +1037,8 @@ Release 0.21.0 - Unreleased HBASE-3132 Print TimestampRange and BloomFilters in HFile pretty print HBASE-2514 RegionServer should refuse to be assigned a region that use LZO when LZO isn't available + HBASE-3082 For ICV gets, first look in MemStore before reading StoreFiles + (prakash via stack) NEW FEATURES HBASE-1961 HBase EC2 scripts diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 4c5395e5281..4954529c4e1 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -2859,6 +2859,86 @@ public class HRegion implements HeapSize { // , Writable{ return new Result(result); } + /** + * An optimized version of {@link #get(Get)} that checks MemStore first for + * the specified query. + *

+ * This is intended for use by increment operations where we have the + * guarantee that versions are never inserted out-of-order so if a value + * exists in MemStore it is the latest value. + *

+ * It only makes sense to use this method without a TimeRange and maxVersions + * equal to 1. + * @param get + * @return result + * @throws IOException + */ + private List getLastIncrement(final Get get) throws IOException { + InternalScan iscan = new InternalScan(get); + + List results = new ArrayList(); + + // memstore scan + iscan.checkOnlyMemStore(); + InternalScanner scanner = null; + try { + scanner = getScanner(iscan); + scanner.next(results); + } finally { + if (scanner != null) + scanner.close(); + } + + // count how many columns we're looking for + int expected = 0; + Map> familyMap = get.getFamilyMap(); + for (NavigableSet qfs : familyMap.values()) { + expected += qfs.size(); + } + + // found everything we were looking for, done + if (results.size() == expected) { + return results; + } + + // still have more columns to find + if (results != null && !results.isEmpty()) { + // subtract what was found in memstore + for (KeyValue kv : results) { + byte [] family = kv.getFamily(); + NavigableSet qfs = familyMap.get(family); + qfs.remove(kv.getQualifier()); + if (qfs.isEmpty()) familyMap.remove(family); + expected--; + } + // make a new get for just what is left + Get newGet = new Get(get.getRow()); + for (Map.Entry> f : familyMap.entrySet()) { + byte [] family = f.getKey(); + for (byte [] qualifier : f.getValue()) { + newGet.addColumn(family, qualifier); + } + } + iscan = new InternalScan(newGet); + } + + // check store files for what is left + List fileResults = new ArrayList(); + iscan.checkOnlyStoreFiles(); + scanner = null; + try { + scanner = getScanner(iscan); + scanner.next(fileResults); + } finally { + if (scanner != null) + scanner.close(); + } + + // combine and return + results.addAll(fileResults); + return results; + } + /* * Do a get based on the get parameter. */ @@ -2905,7 +2985,7 @@ public class HRegion implements HeapSize { // , Writable{ Get get = new Get(row); get.addColumn(family, qualifier); - List results = get(get); + List results = getLastIncrement(get); if (!results.isEmpty()) { KeyValue kv = results.get(0); @@ -2914,7 +2994,7 @@ public class HRegion implements HeapSize { // , Writable{ result += Bytes.toLong(buffer, valueOffset, Bytes.SIZEOF_LONG); } - // bulid the KeyValue now: + // build the KeyValue now: KeyValue newKv = new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTimeMillis(), Bytes.toBytes(result)); @@ -2930,7 +3010,7 @@ public class HRegion implements HeapSize { // , Writable{ // Now request the ICV to the store, this will set the timestamp // appropriately depending on if there is a value in memcache or not. - // returns the + // returns the change in the size of the memstore from operation long size = store.updateColumnValue(row, family, qualifier, result); size = this.memstoreSize.addAndGet(size); diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java new file mode 100644 index 00000000000..db2e02d80a8 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScan.java @@ -0,0 +1,78 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Scan; + +/** + * Special internal-only scanner, currently used for increment operations to + * allow additional server-side arguments for Scan operations. + *

+ * Rather than adding new options/parameters to the public Scan API, this new + * class has been created. + *

+ * Supports adding an option to only read from the MemStore with + * {@link #checkOnlyMemStore()} or to only read from StoreFiles with + * {@link #checkOnlyStoreFiles()}. + */ +class InternalScan extends Scan { + private boolean memOnly = false; + private boolean filesOnly = false; + + /** + * @param get get to model scan after + */ + public InternalScan(Get get) { + super(get); + } + + /** + * StoreFiles will not be scanned. Only MemStore will be scanned. + */ + public void checkOnlyMemStore() { + memOnly = true; + filesOnly = false; + } + + /** + * MemStore will not be scanned. Only StoreFiles will be scanned. + */ + public void checkOnlyStoreFiles() { + memOnly = false; + filesOnly = true; + } + + /** + * Returns true if only the MemStore should be checked. False if not. + * @return true to only check MemStore + */ + public boolean isCheckOnlyMemStore() { + return (memOnly); + } + + /** + * Returns true if only StoreFiles should be checked. False if not. + * @return true if only check StoreFiles + */ + public boolean isCheckOnlyStoreFiles() { + return (filesOnly); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 4775fc86454..202d58a200b 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.client.Scan; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.NavigableSet; @@ -149,22 +150,33 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb */ private List getScanners(Scan scan, final NavigableSet columns) throws IOException { + boolean memOnly; + boolean filesOnly; + if (scan instanceof InternalScan) { + InternalScan iscan = (InternalScan)scan; + memOnly = iscan.isCheckOnlyMemStore(); + filesOnly = iscan.isCheckOnlyStoreFiles(); + } else { + memOnly = false; + filesOnly = false; + } + List scanners = new LinkedList(); // First the store file scanners - List sfScanners = StoreFileScanner + if (memOnly == false) { + List sfScanners = StoreFileScanner .getScannersForStoreFiles(store.getStorefiles(), cacheBlocks, isGet); - List scanners = - new ArrayList(sfScanners.size()+1); - // include only those scan files which pass all filters - for (StoreFileScanner sfs : sfScanners) { - if (sfs.shouldSeek(scan, columns)) { - scanners.add(sfs); + // include only those scan files which pass all filters + for (StoreFileScanner sfs : sfScanners) { + if (sfs.shouldSeek(scan, columns)) { + scanners.add(sfs); + } } } // Then the memstore scanners - if (this.store.memstore.shouldSeek(scan)) { - scanners.addAll(this.store.memstore.getScanners()); + if ((filesOnly == false) && (this.store.memstore.shouldSeek(scan))) { + scanners.addAll(this.store.memstore.getScanners()); } return scanners; }