HBASE-1761 getclosest doesn't understand delete family; manifests as 'HRegionInfo was null or empty in .META' A.K.A the BS problem
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@805063 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
a37b0a523a
commit
eab24df54c
|
@ -335,6 +335,8 @@ Release 0.20.0 - Unreleased
|
||||||
HBASE-1768 REST server has upper limit of 5k PUT
|
HBASE-1768 REST server has upper limit of 5k PUT
|
||||||
HBASE-1766 Add advanced features to HFile.main() to be able to analyze
|
HBASE-1766 Add advanced features to HFile.main() to be able to analyze
|
||||||
storefile problems
|
storefile problems
|
||||||
|
HBASE-1761 getclosest doesn't understand delete family; manifests as
|
||||||
|
"HRegionInfo was null or empty in .META" A.K.A the BS problem
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
|
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
|
||||||
|
|
|
@ -167,6 +167,7 @@ module HBase
|
||||||
raise IOError.new("Table " + tableName + " is enabled. Disable it first")
|
raise IOError.new("Table " + tableName + " is enabled. Disable it first")
|
||||||
else
|
else
|
||||||
@admin.deleteTable(tableName)
|
@admin.deleteTable(tableName)
|
||||||
|
flush(HConstants::META_TABLE_NAME);
|
||||||
major_compact(HConstants::META_TABLE_NAME);
|
major_compact(HConstants::META_TABLE_NAME);
|
||||||
end
|
end
|
||||||
@formatter.footer(now)
|
@formatter.footer(now)
|
||||||
|
|
|
@ -22,6 +22,7 @@ package org.apache.hadoop.hbase;
|
||||||
import java.io.DataInput;
|
import java.io.DataInput;
|
||||||
import java.io.DataOutput;
|
import java.io.DataOutput;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.Comparator;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -291,11 +292,12 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructs KeyValue structure filled with null value.
|
* Constructs KeyValue structure filled with null value.
|
||||||
|
* Sets type to {@link KeyValue.Type#Maximum}
|
||||||
* @param row - row key (arbitrary byte array)
|
* @param row - row key (arbitrary byte array)
|
||||||
* @param timestamp
|
* @param timestamp
|
||||||
*/
|
*/
|
||||||
public KeyValue(final byte [] row, final long timestamp) {
|
public KeyValue(final byte [] row, final long timestamp) {
|
||||||
this(row, timestamp, Type.Put);
|
this(row, timestamp, Type.Maximum);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -309,13 +311,14 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructs KeyValue structure filled with null value.
|
* Constructs KeyValue structure filled with null value.
|
||||||
|
* Sets type to {@link KeyValue.Type#Maximum}
|
||||||
* @param row - row key (arbitrary byte array)
|
* @param row - row key (arbitrary byte array)
|
||||||
* @param family family name
|
* @param family family name
|
||||||
* @param qualifier column qualifier
|
* @param qualifier column qualifier
|
||||||
*/
|
*/
|
||||||
public KeyValue(final byte [] row, final byte [] family,
|
public KeyValue(final byte [] row, final byte [] family,
|
||||||
final byte [] qualifier) {
|
final byte [] qualifier) {
|
||||||
this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put);
|
this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -569,48 +572,9 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
* @return Fully copied clone of this KeyValue
|
* @return Fully copied clone of this KeyValue
|
||||||
*/
|
*/
|
||||||
public KeyValue clone() {
|
public KeyValue clone() {
|
||||||
byte [] bytes = new byte[this.length];
|
byte [] b = new byte[this.length];
|
||||||
System.arraycopy(this.bytes, this.offset, bytes, 0, this.length);
|
System.arraycopy(this.bytes, this.offset, b, 0, this.length);
|
||||||
return new KeyValue(bytes, 0, bytes.length);
|
return new KeyValue(b, 0, b.length);
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clones a row.
|
|
||||||
*
|
|
||||||
* @param timestamp The new time stamp for the row.
|
|
||||||
* @return Clone of bb's key portion with only the row and timestamp filled in.
|
|
||||||
*/
|
|
||||||
public KeyValue cloneRow(final long timestamp) {
|
|
||||||
return new KeyValue(getBuffer(), getRowOffset(), getRowLength(),
|
|
||||||
null, 0, 0, null, 0, 0,
|
|
||||||
timestamp, Type.codeToType(getType()), null, 0, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return Clone of bb's key portion with type set to Type.Maximum. Use this
|
|
||||||
* doing lookups where you are doing getClosest. Using Maximum, you'll be
|
|
||||||
* sure to trip over all of the other key types since Maximum sorts first.
|
|
||||||
*/
|
|
||||||
public KeyValue cloneMaximum() {
|
|
||||||
return createKey(Type.Maximum);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Make a clone with the new type. Does not copy value.
|
|
||||||
*
|
|
||||||
* @param newtype New type to set on clone of this key.
|
|
||||||
* @return Clone of this key with type set to <code>newtype</code>
|
|
||||||
*/
|
|
||||||
private KeyValue createKey(final Type newtype) {
|
|
||||||
int keylength = getKeyLength();
|
|
||||||
int l = keylength + ROW_OFFSET;
|
|
||||||
byte [] other = new byte[l];
|
|
||||||
System.arraycopy(getBuffer(), getOffset(), other, 0, l);
|
|
||||||
// Set value length to zero.
|
|
||||||
Bytes.putInt(other, Bytes.SIZEOF_INT, 0);
|
|
||||||
// Set last byte, the type, to new type
|
|
||||||
other[l - 1] = newtype.getCode();
|
|
||||||
return new KeyValue(other, 0, other.length);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//---------------------------------------------------------------------------
|
//---------------------------------------------------------------------------
|
||||||
|
@ -946,17 +910,27 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return True if Delete KeyValue type.
|
* @return True if a delete type, a {@link KeyValue.Type#Delete} or
|
||||||
|
* a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type.DeleteColumn}
|
||||||
|
* KeyValue type.
|
||||||
*/
|
*/
|
||||||
public boolean isDeleteType() {
|
public boolean isDelete() {
|
||||||
return getType() == Type.Delete.code;
|
int t = getType();
|
||||||
|
return Type.Delete.getCode() <= t && t <= Type.DeleteFamily.getCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return True if DeleteColumn KeyValue type.
|
* @return True if this KV is a {@link KeyValue.Type#Delete} type.
|
||||||
*/
|
*/
|
||||||
public boolean isDeleteColumnType() {
|
public boolean isDeleteType() {
|
||||||
return getType() == Type.DeleteColumn.code;
|
return getType() == Type.Delete.getCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return True if this KV is a delete family type.
|
||||||
|
*/
|
||||||
|
public boolean isDeleteFamily() {
|
||||||
|
return getType() == Type.DeleteFamily.getCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1258,13 +1232,13 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
return index;
|
return index;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/**
|
||||||
* @param b
|
* @param b
|
||||||
* @param delimiter
|
* @param delimiter
|
||||||
* @return Index of delimiter having started from end of <code>b</code> moving
|
* @return Index of delimiter having started from start of <code>b</code>
|
||||||
* leftward.
|
* moving rightward.
|
||||||
*/
|
*/
|
||||||
static int getDelimiter(final byte [] b, int offset, final int length,
|
public static int getDelimiter(final byte [] b, int offset, final int length,
|
||||||
final int delimiter) {
|
final int delimiter) {
|
||||||
if (b == null) {
|
if (b == null) {
|
||||||
throw new NullPointerException();
|
throw new NullPointerException();
|
||||||
|
@ -1279,12 +1253,13 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/**
|
||||||
|
* Find index of passed delimiter walking from end of buffer backwards.
|
||||||
* @param b
|
* @param b
|
||||||
* @param delimiter
|
* @param delimiter
|
||||||
* @return Index of delimiter
|
* @return Index of delimiter
|
||||||
*/
|
*/
|
||||||
static int getDelimiterInReverse(final byte [] b, final int offset,
|
public static int getDelimiterInReverse(final byte [] b, final int offset,
|
||||||
final int length, final int delimiter) {
|
final int length, final int delimiter) {
|
||||||
if (b == null) {
|
if (b == null) {
|
||||||
throw new NullPointerException();
|
throw new NullPointerException();
|
||||||
|
@ -1658,6 +1633,21 @@ public class KeyValue implements Writable, HeapSize {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Comparator that compares row component only of a KeyValue.
|
||||||
|
*/
|
||||||
|
public static class RowComparator implements Comparator<KeyValue> {
|
||||||
|
final KVComparator comparator;
|
||||||
|
|
||||||
|
public RowComparator(final KVComparator c) {
|
||||||
|
this.comparator = c;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int compare(KeyValue left, KeyValue right) {
|
||||||
|
return comparator.compareRows(left, right);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compare key portion of a {@link KeyValue} for keys in <code>.META.</code>
|
* Compare key portion of a {@link KeyValue} for keys in <code>.META.</code>
|
||||||
* table.
|
* table.
|
||||||
|
|
|
@ -577,7 +577,7 @@ public class HConnectionManager implements HConstants {
|
||||||
REGIONINFO_QUALIFIER);
|
REGIONINFO_QUALIFIER);
|
||||||
if (value == null || value.length == 0) {
|
if (value == null || value.length == 0) {
|
||||||
throw new IOException("HRegionInfo was null or empty in " +
|
throw new IOException("HRegionInfo was null or empty in " +
|
||||||
Bytes.toString(parentTable) + ", " + regionInfoRow);
|
Bytes.toString(parentTable) + ", row=" + regionInfoRow);
|
||||||
}
|
}
|
||||||
// convert the row result into the HRegionLocation we need!
|
// convert the row result into the HRegionLocation we need!
|
||||||
HRegionInfo regionInfo = (HRegionInfo) Writables.getWritable(
|
HRegionInfo regionInfo = (HRegionInfo) Writables.getWritable(
|
||||||
|
|
|
@ -333,8 +333,7 @@ public class Put implements HeapSize, Writable, Comparable<Put> {
|
||||||
this.lockId = in.readLong();
|
this.lockId = in.readLong();
|
||||||
this.writeToWAL = in.readBoolean();
|
this.writeToWAL = in.readBoolean();
|
||||||
int numFamilies = in.readInt();
|
int numFamilies = in.readInt();
|
||||||
this.familyMap =
|
if (!this.familyMap.isEmpty()) this.familyMap.clear();
|
||||||
new TreeMap<byte [],List<KeyValue>>(Bytes.BYTES_COMPARATOR);
|
|
||||||
for(int i=0;i<numFamilies;i++) {
|
for(int i=0;i<numFamilies;i++) {
|
||||||
byte [] family = Bytes.readByteArray(in);
|
byte [] family = Bytes.readByteArray(in);
|
||||||
int numKeys = in.readInt();
|
int numKeys = in.readInt();
|
||||||
|
@ -359,7 +358,7 @@ public class Put implements HeapSize, Writable, Comparable<Put> {
|
||||||
out.writeLong(this.lockId);
|
out.writeLong(this.lockId);
|
||||||
out.writeBoolean(this.writeToWAL);
|
out.writeBoolean(this.writeToWAL);
|
||||||
out.writeInt(familyMap.size());
|
out.writeInt(familyMap.size());
|
||||||
for(Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
|
for (Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
|
||||||
Bytes.writeByteArray(out, entry.getKey());
|
Bytes.writeByteArray(out, entry.getKey());
|
||||||
List<KeyValue> keys = entry.getValue();
|
List<KeyValue> keys = entry.getValue();
|
||||||
out.writeInt(keys.size());
|
out.writeInt(keys.size());
|
||||||
|
|
|
@ -0,0 +1,240 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2009 The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.regionserver;
|
||||||
|
|
||||||
|
import java.util.NavigableMap;
|
||||||
|
import java.util.NavigableSet;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
import java.util.TreeSet;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
|
import org.apache.hadoop.hbase.KeyValue.KVComparator;
|
||||||
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* State and utility processing {@link HRegion#getClosestRowBefore(byte[], byte[])}.
|
||||||
|
* Like {@link GetDeleteTracker} and {@link ScanDeleteTracker} but does not
|
||||||
|
* implement the {@link DeleteTracker} interface since state spans rows (There
|
||||||
|
* is no update nor reset method).
|
||||||
|
*/
|
||||||
|
class GetClosestRowBeforeTracker {
|
||||||
|
private final KeyValue targetkey;
|
||||||
|
// Any cell w/ a ts older than this is expired.
|
||||||
|
private final long oldestts;
|
||||||
|
private KeyValue candidate = null;
|
||||||
|
private final KVComparator kvcomparator;
|
||||||
|
// Flag for whether we're doing getclosest on a metaregion.
|
||||||
|
private final boolean metaregion;
|
||||||
|
// Offset and length into targetkey demarking table name (if in a metaregion).
|
||||||
|
private final int rowoffset;
|
||||||
|
private final int tablenamePlusDelimiterLength;
|
||||||
|
|
||||||
|
// Deletes keyed by row. Comparator compares on row portion of KeyValue only.
|
||||||
|
private final NavigableMap<KeyValue, NavigableSet<KeyValue>> deletes;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param c
|
||||||
|
* @param kv Presume first on row: i.e. empty column, maximum timestamp and
|
||||||
|
* a type of Type.Maximum
|
||||||
|
* @param ttl Time to live in ms for this Store
|
||||||
|
* @param metaregion True if this is .META. or -ROOT- region.
|
||||||
|
*/
|
||||||
|
GetClosestRowBeforeTracker(final KVComparator c, final KeyValue kv,
|
||||||
|
final long ttl, final boolean metaregion) {
|
||||||
|
super();
|
||||||
|
this.metaregion = metaregion;
|
||||||
|
this.targetkey = kv;
|
||||||
|
// If we are in a metaregion, then our table name is the prefix on the
|
||||||
|
// targetkey.
|
||||||
|
this.rowoffset = kv.getRowOffset();
|
||||||
|
int l = -1;
|
||||||
|
if (metaregion) {
|
||||||
|
l = KeyValue.getDelimiter(kv.getBuffer(), rowoffset, kv.getRowLength(),
|
||||||
|
HRegionInfo.DELIMITER) - this.rowoffset;
|
||||||
|
}
|
||||||
|
this.tablenamePlusDelimiterLength = metaregion? l + 1: -1;
|
||||||
|
this.oldestts = System.currentTimeMillis() - ttl;
|
||||||
|
this.kvcomparator = c;
|
||||||
|
KeyValue.RowComparator rc = new KeyValue.RowComparator(this.kvcomparator);
|
||||||
|
this.deletes = new TreeMap<KeyValue, NavigableSet<KeyValue>>(rc);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param kv
|
||||||
|
* @return True if this <code>kv</code> is expired.
|
||||||
|
*/
|
||||||
|
boolean isExpired(final KeyValue kv) {
|
||||||
|
return Store.isExpired(kv, this.oldestts);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Add the specified KeyValue to the list of deletes.
|
||||||
|
* @param kv
|
||||||
|
*/
|
||||||
|
private void addDelete(final KeyValue kv) {
|
||||||
|
NavigableSet<KeyValue> rowdeletes = this.deletes.get(kv);
|
||||||
|
if (rowdeletes == null) {
|
||||||
|
rowdeletes = new TreeSet<KeyValue>(this.kvcomparator);
|
||||||
|
this.deletes.put(kv, rowdeletes);
|
||||||
|
}
|
||||||
|
rowdeletes.add(kv);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* @param kv Adds candidate if nearer the target than previous candidate.
|
||||||
|
* @return True if updated candidate.
|
||||||
|
*/
|
||||||
|
private boolean addCandidate(final KeyValue kv) {
|
||||||
|
if (!isDeleted(kv) && isBetterCandidate(kv)) {
|
||||||
|
this.candidate = kv;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean isBetterCandidate(final KeyValue contender) {
|
||||||
|
return this.candidate == null ||
|
||||||
|
(this.kvcomparator.compareRows(this.candidate, contender) < 0 &&
|
||||||
|
this.kvcomparator.compareRows(contender, this.targetkey) <= 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Check if specified KeyValue buffer has been deleted by a previously
|
||||||
|
* seen delete.
|
||||||
|
* @param kv
|
||||||
|
* @return true is the specified KeyValue is deleted, false if not
|
||||||
|
*/
|
||||||
|
private boolean isDeleted(final KeyValue kv) {
|
||||||
|
if (this.deletes.isEmpty()) return false;
|
||||||
|
NavigableSet<KeyValue> rowdeletes = this.deletes.get(kv);
|
||||||
|
if (rowdeletes == null || rowdeletes.isEmpty()) return false;
|
||||||
|
return isDeleted(kv, rowdeletes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the specified KeyValue buffer has been deleted by a previously
|
||||||
|
* seen delete.
|
||||||
|
* @param kv
|
||||||
|
* @param ds
|
||||||
|
* @return True is the specified KeyValue is deleted, false if not
|
||||||
|
*/
|
||||||
|
public boolean isDeleted(final KeyValue kv, final NavigableSet<KeyValue> ds) {
|
||||||
|
if (deletes == null || deletes.isEmpty()) return false;
|
||||||
|
for (KeyValue d: ds) {
|
||||||
|
long kvts = kv.getTimestamp();
|
||||||
|
long dts = d.getTimestamp();
|
||||||
|
if (d.isDeleteFamily()) {
|
||||||
|
if (kvts <= dts) return true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Check column
|
||||||
|
int ret = Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(),
|
||||||
|
kv.getQualifierLength(),
|
||||||
|
d.getBuffer(), d.getQualifierOffset(), d.getQualifierLength());
|
||||||
|
if (ret <= -1) {
|
||||||
|
// This delete is for an earlier column.
|
||||||
|
continue;
|
||||||
|
} else if (ret >= 1) {
|
||||||
|
// Beyond this kv.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// Check Timestamp
|
||||||
|
if (kvts > dts) return false;
|
||||||
|
|
||||||
|
// Check Type
|
||||||
|
switch (KeyValue.Type.codeToType(d.getType())) {
|
||||||
|
case Delete: return kvts == dts;
|
||||||
|
case DeleteColumn: return true;
|
||||||
|
default: continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Handle keys whose values hold deletes.
|
||||||
|
* Add to the set of deletes and then if the candidate keys contain any that
|
||||||
|
* might match, then check for a match and remove it. Implies candidates
|
||||||
|
* is made with a Comparator that ignores key type.
|
||||||
|
* @param kv
|
||||||
|
* @return True if we removed <code>k</code> from <code>candidates</code>.
|
||||||
|
*/
|
||||||
|
boolean handleDeletes(final KeyValue kv) {
|
||||||
|
addDelete(kv);
|
||||||
|
boolean deleted = false;
|
||||||
|
if (!hasCandidate()) return deleted;
|
||||||
|
if (isDeleted(this.candidate)) {
|
||||||
|
this.candidate = null;
|
||||||
|
deleted = true;
|
||||||
|
}
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Do right thing with passed key, add to deletes or add to candidates.
|
||||||
|
* @param kv
|
||||||
|
* @return True if we added a candidate
|
||||||
|
*/
|
||||||
|
boolean handle(final KeyValue kv) {
|
||||||
|
if (kv.isDelete()) {
|
||||||
|
handleDeletes(kv);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return addCandidate(kv);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return True if has candidate
|
||||||
|
*/
|
||||||
|
public boolean hasCandidate() {
|
||||||
|
return this.candidate != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return Best candidate or null.
|
||||||
|
*/
|
||||||
|
public KeyValue getCandidate() {
|
||||||
|
return this.candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public KeyValue getTargetKey() {
|
||||||
|
return this.targetkey;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param kv Current kv
|
||||||
|
* @param First on row kv.
|
||||||
|
* @param state
|
||||||
|
* @return True if we went too far, past the target key.
|
||||||
|
*/
|
||||||
|
boolean isTooFar(final KeyValue kv, final KeyValue firstOnRow) {
|
||||||
|
return this.kvcomparator.compareRows(kv, firstOnRow) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean isTargetTable(final KeyValue kv) {
|
||||||
|
if (!metaregion) return true;
|
||||||
|
// Compare start of keys row. Compare including delimiter. Saves having
|
||||||
|
// to calculate where tablename ends in the candidate kv.
|
||||||
|
return Bytes.compareTo(this.targetkey.getBuffer(), this.rowoffset,
|
||||||
|
this.tablenamePlusDelimiterLength,
|
||||||
|
kv.getBuffer(), kv.getRowOffset(), this.tablenamePlusDelimiterLength) == 0;
|
||||||
|
}
|
||||||
|
}
|
|
@ -38,8 +38,8 @@ import org.apache.hadoop.hbase.util.Bytes;
|
||||||
* This class is NOT thread-safe as queries are never multi-threaded
|
* This class is NOT thread-safe as queries are never multi-threaded
|
||||||
*/
|
*/
|
||||||
public class GetDeleteTracker implements DeleteTracker {
|
public class GetDeleteTracker implements DeleteTracker {
|
||||||
|
private static long UNSET = -1L;
|
||||||
private long familyStamp = -1L;
|
private long familyStamp = UNSET;
|
||||||
protected List<Delete> deletes = null;
|
protected List<Delete> deletes = null;
|
||||||
private List<Delete> newDeletes = new ArrayList<Delete>();
|
private List<Delete> newDeletes = new ArrayList<Delete>();
|
||||||
private Iterator<Delete> iterator;
|
private Iterator<Delete> iterator;
|
||||||
|
@ -64,7 +64,7 @@ public class GetDeleteTracker implements DeleteTracker {
|
||||||
@Override
|
@Override
|
||||||
public void add(byte [] buffer, int qualifierOffset, int qualifierLength,
|
public void add(byte [] buffer, int qualifierOffset, int qualifierLength,
|
||||||
long timestamp, byte type) {
|
long timestamp, byte type) {
|
||||||
if(type == KeyValue.Type.DeleteFamily.getCode()) {
|
if (type == KeyValue.Type.DeleteFamily.getCode()) {
|
||||||
if(timestamp > familyStamp) {
|
if(timestamp > familyStamp) {
|
||||||
familyStamp = timestamp;
|
familyStamp = timestamp;
|
||||||
}
|
}
|
||||||
|
@ -88,14 +88,13 @@ public class GetDeleteTracker implements DeleteTracker {
|
||||||
@Override
|
@Override
|
||||||
public boolean isDeleted(byte [] buffer, int qualifierOffset,
|
public boolean isDeleted(byte [] buffer, int qualifierOffset,
|
||||||
int qualifierLength, long timestamp) {
|
int qualifierLength, long timestamp) {
|
||||||
|
|
||||||
// Check against DeleteFamily
|
// Check against DeleteFamily
|
||||||
if (timestamp <= familyStamp) {
|
if (timestamp <= familyStamp) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if there are other deletes
|
// Check if there are other deletes
|
||||||
if(this.delete == null) {
|
if (this.delete == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,7 +102,7 @@ public class GetDeleteTracker implements DeleteTracker {
|
||||||
int ret = Bytes.compareTo(buffer, qualifierOffset, qualifierLength,
|
int ret = Bytes.compareTo(buffer, qualifierOffset, qualifierLength,
|
||||||
this.delete.buffer, this.delete.qualifierOffset,
|
this.delete.buffer, this.delete.qualifierOffset,
|
||||||
this.delete.qualifierLength);
|
this.delete.qualifierLength);
|
||||||
if(ret <= -1) {
|
if (ret <= -1) {
|
||||||
// Have not reached the next delete yet
|
// Have not reached the next delete yet
|
||||||
return false;
|
return false;
|
||||||
} else if(ret >= 1) {
|
} else if(ret >= 1) {
|
||||||
|
@ -149,10 +148,8 @@ public class GetDeleteTracker implements DeleteTracker {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isEmpty() {
|
public boolean isEmpty() {
|
||||||
if(this.familyStamp == 0L && this.delete == null) {
|
return this.familyStamp == UNSET && this.delete == null &&
|
||||||
return true;
|
this.newDeletes.isEmpty();
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -160,7 +157,7 @@ public class GetDeleteTracker implements DeleteTracker {
|
||||||
this.deletes = null;
|
this.deletes = null;
|
||||||
this.delete = null;
|
this.delete = null;
|
||||||
this.newDeletes = new ArrayList<Delete>();
|
this.newDeletes = new ArrayList<Delete>();
|
||||||
this.familyStamp = 0L;
|
this.familyStamp = UNSET;
|
||||||
this.iterator = null;
|
this.iterator = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -173,7 +170,7 @@ public class GetDeleteTracker implements DeleteTracker {
|
||||||
@Override
|
@Override
|
||||||
public void update() {
|
public void update() {
|
||||||
// If no previous deletes, use new deletes and return
|
// If no previous deletes, use new deletes and return
|
||||||
if(this.deletes == null || this.deletes.size() == 0) {
|
if (this.deletes == null || this.deletes.size() == 0) {
|
||||||
finalize(this.newDeletes);
|
finalize(this.newDeletes);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1022,9 +1022,8 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
|
||||||
* @return map of values
|
* @return map of values
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public Result getClosestRowBefore(final byte [] row,
|
public Result getClosestRowBefore(final byte [] row, final byte [] family)
|
||||||
final byte [] family)
|
throws IOException {
|
||||||
throws IOException{
|
|
||||||
// look across all the HStores for this region and determine what the
|
// look across all the HStores for this region and determine what the
|
||||||
// closest key is across all column families, since the data may be sparse
|
// closest key is across all column families, since the data may be sparse
|
||||||
KeyValue key = null;
|
KeyValue key = null;
|
||||||
|
@ -1038,22 +1037,16 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
|
||||||
if (key == null) {
|
if (key == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
List<KeyValue> results = new ArrayList<KeyValue>();
|
// This will get all results for this store. TODO: Do we need to do this?
|
||||||
// This will get all results for this store. TODO: Do I have to make a
|
|
||||||
// new key?
|
|
||||||
if (!this.comparator.matchingRows(kv, key)) {
|
|
||||||
kv = new KeyValue(key.getRow(), HConstants.LATEST_TIMESTAMP);
|
|
||||||
}
|
|
||||||
Get get = new Get(key.getRow());
|
Get get = new Get(key.getRow());
|
||||||
|
List<KeyValue> results = new ArrayList<KeyValue>();
|
||||||
store.get(get, null, results);
|
store.get(get, null, results);
|
||||||
|
|
||||||
return new Result(results);
|
return new Result(results);
|
||||||
} finally {
|
} finally {
|
||||||
splitsAndClosesLock.readLock().unlock();
|
splitsAndClosesLock.readLock().unlock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//TODO
|
|
||||||
/**
|
/**
|
||||||
* Return an iterator that scans over the HRegion, returning the indicated
|
* Return an iterator that scans over the HRegion, returning the indicated
|
||||||
* columns and rows specified by the {@link Scan}.
|
* columns and rows specified by the {@link Scan}.
|
||||||
|
@ -1166,7 +1159,8 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
|
||||||
Get g = new Get(kv.getRow());
|
Get g = new Get(kv.getRow());
|
||||||
NavigableSet<byte []> qualifiers =
|
NavigableSet<byte []> qualifiers =
|
||||||
new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
|
new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
|
||||||
qualifiers.add(kv.getQualifier());
|
byte [] q = kv.getQualifier();
|
||||||
|
if (q != null && q.length > 0) qualifiers.add(kv.getQualifier());
|
||||||
get(store, g, qualifiers, result);
|
get(store, g, qualifiers, result);
|
||||||
if (result.isEmpty()) {
|
if (result.isEmpty()) {
|
||||||
// Nothing to delete
|
// Nothing to delete
|
||||||
|
@ -2432,6 +2426,7 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
|
||||||
} else {
|
} else {
|
||||||
// Default behavior
|
// Default behavior
|
||||||
Scan scan = new Scan();
|
Scan scan = new Scan();
|
||||||
|
// scan.addFamily(HConstants.CATALOG_FAMILY);
|
||||||
InternalScanner scanner = region.getScanner(scan);
|
InternalScanner scanner = region.getScanner(scan);
|
||||||
try {
|
try {
|
||||||
List<KeyValue> kvs = new ArrayList<KeyValue>();
|
List<KeyValue> kvs = new ArrayList<KeyValue>();
|
||||||
|
@ -2444,6 +2439,7 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
|
||||||
} finally {
|
} finally {
|
||||||
scanner.close();
|
scanner.close();
|
||||||
}
|
}
|
||||||
|
// System.out.println(region.getClosestRowBefore(Bytes.toBytes("GeneratedCSVContent2,E3652782193BC8D66A0BA1629D0FAAAB,9993372036854775807")));
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
region.close();
|
region.close();
|
||||||
|
@ -2481,7 +2477,6 @@ public class HRegion implements HConstants, HeapSize { // , Writable{
|
||||||
printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
|
printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
|
||||||
}
|
}
|
||||||
majorCompact = true;
|
majorCompact = true;
|
||||||
|
|
||||||
}
|
}
|
||||||
Path tableDir = new Path(args[0]);
|
Path tableDir = new Path(args[0]);
|
||||||
HBaseConfiguration c = new HBaseConfiguration();
|
HBaseConfiguration c = new HBaseConfiguration();
|
||||||
|
|
|
@ -29,7 +29,6 @@ import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.NavigableSet;
|
import java.util.NavigableSet;
|
||||||
import java.util.SortedSet;
|
import java.util.SortedSet;
|
||||||
import java.util.TreeSet;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
|
|
||||||
|
@ -56,8 +55,6 @@ import org.apache.hadoop.hbase.util.ClassSize;
|
||||||
public class MemStore implements HeapSize {
|
public class MemStore implements HeapSize {
|
||||||
private static final Log LOG = LogFactory.getLog(MemStore.class);
|
private static final Log LOG = LogFactory.getLog(MemStore.class);
|
||||||
|
|
||||||
private final long ttl;
|
|
||||||
|
|
||||||
// MemStore. Use a KeyValueSkipListSet rather than SkipListSet because of the
|
// MemStore. Use a KeyValueSkipListSet rather than SkipListSet because of the
|
||||||
// better semantics. The Map will overwrite if passed a key it already had
|
// better semantics. The Map will overwrite if passed a key it already had
|
||||||
// whereas the Set will not add new KV if key is same though value might be
|
// whereas the Set will not add new KV if key is same though value might be
|
||||||
|
@ -68,7 +65,7 @@ public class MemStore implements HeapSize {
|
||||||
// Snapshot of memstore. Made for flusher.
|
// Snapshot of memstore. Made for flusher.
|
||||||
volatile KeyValueSkipListSet snapshot;
|
volatile KeyValueSkipListSet snapshot;
|
||||||
|
|
||||||
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
|
final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
|
||||||
|
|
||||||
final KeyValue.KVComparator comparator;
|
final KeyValue.KVComparator comparator;
|
||||||
|
|
||||||
|
@ -85,7 +82,7 @@ public class MemStore implements HeapSize {
|
||||||
* Default constructor. Used for tests.
|
* Default constructor. Used for tests.
|
||||||
*/
|
*/
|
||||||
public MemStore() {
|
public MemStore() {
|
||||||
this(HConstants.FOREVER, KeyValue.COMPARATOR);
|
this(KeyValue.COMPARATOR);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -93,8 +90,7 @@ public class MemStore implements HeapSize {
|
||||||
* @param ttl The TTL for cache entries, in milliseconds.
|
* @param ttl The TTL for cache entries, in milliseconds.
|
||||||
* @param c
|
* @param c
|
||||||
*/
|
*/
|
||||||
public MemStore(final long ttl, final KeyValue.KVComparator c) {
|
public MemStore(final KeyValue.KVComparator c) {
|
||||||
this.ttl = ttl;
|
|
||||||
this.comparator = c;
|
this.comparator = c;
|
||||||
this.comparatorIgnoreTimestamp =
|
this.comparatorIgnoreTimestamp =
|
||||||
this.comparator.getComparatorIgnoringTimestamps();
|
this.comparator.getComparatorIgnoringTimestamps();
|
||||||
|
@ -184,15 +180,15 @@ public class MemStore implements HeapSize {
|
||||||
* @return approximate size of the passed key and value.
|
* @return approximate size of the passed key and value.
|
||||||
*/
|
*/
|
||||||
long add(final KeyValue kv) {
|
long add(final KeyValue kv) {
|
||||||
long size = -1;
|
long s = -1;
|
||||||
this.lock.readLock().lock();
|
this.lock.readLock().lock();
|
||||||
try {
|
try {
|
||||||
size = heapSizeChange(kv, this.kvset.add(kv));
|
s = heapSizeChange(kv, this.kvset.add(kv));
|
||||||
this.size.addAndGet(size);
|
this.size.addAndGet(s);
|
||||||
} finally {
|
} finally {
|
||||||
this.lock.readLock().unlock();
|
this.lock.readLock().unlock();
|
||||||
}
|
}
|
||||||
return size;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -201,7 +197,7 @@ public class MemStore implements HeapSize {
|
||||||
* @return approximate size of the passed key and value.
|
* @return approximate size of the passed key and value.
|
||||||
*/
|
*/
|
||||||
long delete(final KeyValue delete) {
|
long delete(final KeyValue delete) {
|
||||||
long size = 0;
|
long s = 0;
|
||||||
this.lock.readLock().lock();
|
this.lock.readLock().lock();
|
||||||
//Have to find out what we want to do here, to find the fastest way of
|
//Have to find out what we want to do here, to find the fastest way of
|
||||||
//removing things that are under a delete.
|
//removing things that are under a delete.
|
||||||
|
@ -261,17 +257,17 @@ public class MemStore implements HeapSize {
|
||||||
//Delete all the entries effected by the last added delete
|
//Delete all the entries effected by the last added delete
|
||||||
for (KeyValue kv : deletes) {
|
for (KeyValue kv : deletes) {
|
||||||
notpresent = this.kvset.remove(kv);
|
notpresent = this.kvset.remove(kv);
|
||||||
size -= heapSizeChange(kv, notpresent);
|
s -= heapSizeChange(kv, notpresent);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Adding the delete to memstore. Add any value, as long as
|
// Adding the delete to memstore. Add any value, as long as
|
||||||
// same instance each time.
|
// same instance each time.
|
||||||
size += heapSizeChange(delete, this.kvset.add(delete));
|
s += heapSizeChange(delete, this.kvset.add(delete));
|
||||||
} finally {
|
} finally {
|
||||||
this.lock.readLock().unlock();
|
this.lock.readLock().unlock();
|
||||||
}
|
}
|
||||||
this.size.addAndGet(size);
|
this.size.addAndGet(s);
|
||||||
return size;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -325,200 +321,122 @@ public class MemStore implements HeapSize {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param row Row to look for.
|
* @param state
|
||||||
* @param candidateKeys Map of candidate keys (Accumulation over lots of
|
|
||||||
* lookup over stores and memstores)
|
|
||||||
*/
|
*/
|
||||||
void getRowKeyAtOrBefore(final KeyValue row,
|
void getRowKeyAtOrBefore(final GetClosestRowBeforeTracker state) {
|
||||||
final NavigableSet<KeyValue> candidateKeys) {
|
|
||||||
getRowKeyAtOrBefore(row, candidateKeys,
|
|
||||||
new TreeSet<KeyValue>(this.comparator), System.currentTimeMillis());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param kv Row to look for.
|
|
||||||
* @param candidates Map of candidate keys (Accumulation over lots of
|
|
||||||
* lookup over stores and memstores). Pass a Set with a Comparator that
|
|
||||||
* ignores key Type so we can do Set.remove using a delete, i.e. a KeyValue
|
|
||||||
* with a different Type to the candidate key.
|
|
||||||
* @param deletes Pass a Set that has a Comparator that ignores key type.
|
|
||||||
* @param now
|
|
||||||
*/
|
|
||||||
void getRowKeyAtOrBefore(final KeyValue kv,
|
|
||||||
final NavigableSet<KeyValue> candidates,
|
|
||||||
final NavigableSet<KeyValue> deletes, final long now) {
|
|
||||||
this.lock.readLock().lock();
|
this.lock.readLock().lock();
|
||||||
try {
|
try {
|
||||||
getRowKeyAtOrBefore(kvset, kv, candidates, deletes, now);
|
getRowKeyAtOrBefore(kvset, state);
|
||||||
getRowKeyAtOrBefore(snapshot, kv, candidates, deletes, now);
|
getRowKeyAtOrBefore(snapshot, state);
|
||||||
} finally {
|
} finally {
|
||||||
this.lock.readLock().unlock();
|
this.lock.readLock().unlock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* @param set
|
||||||
|
* @param state Accumulates deletes and candidates.
|
||||||
|
*/
|
||||||
private void getRowKeyAtOrBefore(final NavigableSet<KeyValue> set,
|
private void getRowKeyAtOrBefore(final NavigableSet<KeyValue> set,
|
||||||
final KeyValue kv, final NavigableSet<KeyValue> candidates,
|
final GetClosestRowBeforeTracker state) {
|
||||||
final NavigableSet<KeyValue> deletes, final long now) {
|
|
||||||
if (set.isEmpty()) {
|
if (set.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// We want the earliest possible to start searching from. Start before
|
if (!walkForwardInSingleRow(set, state.getTargetKey(), state)) {
|
||||||
// the candidate key in case it turns out a delete came in later.
|
// Found nothing in row. Try backing up.
|
||||||
KeyValue search = candidates.isEmpty()? kv: candidates.first();
|
getRowKeyBefore(set, state);
|
||||||
|
|
||||||
// Get all the entries that come equal or after our search key
|
|
||||||
SortedSet<KeyValue> tail = set.tailSet(search);
|
|
||||||
|
|
||||||
// if there are items in the tail map, there's either a direct match to
|
|
||||||
// the search key, or a range of values between the first candidate key
|
|
||||||
// and the ultimate search key (or the end of the cache)
|
|
||||||
if (!tail.isEmpty() &&
|
|
||||||
this.comparator.compareRows(tail.first(), search) <= 0) {
|
|
||||||
// Keep looking at cells as long as they are no greater than the
|
|
||||||
// ultimate search key and there's still records left in the map.
|
|
||||||
KeyValue deleted = null;
|
|
||||||
KeyValue found = null;
|
|
||||||
for (Iterator<KeyValue> iterator = tail.iterator();
|
|
||||||
iterator.hasNext() && (found == null ||
|
|
||||||
this.comparator.compareRows(found, kv) <= 0);) {
|
|
||||||
found = iterator.next();
|
|
||||||
if (this.comparator.compareRows(found, kv) <= 0) {
|
|
||||||
if (found.isDeleteType()) {
|
|
||||||
Store.handleDeletes(found, candidates, deletes);
|
|
||||||
if (deleted == null) {
|
|
||||||
deleted = found;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (Store.notExpiredAndNotInDeletes(this.ttl, found, now, deletes)) {
|
|
||||||
candidates.add(found);
|
|
||||||
} else {
|
|
||||||
if (deleted == null) {
|
|
||||||
deleted = found;
|
|
||||||
}
|
|
||||||
// TODO: Check this removes the right key.
|
|
||||||
// Its expired. Remove it.
|
|
||||||
iterator.remove();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (candidates.isEmpty() && deleted != null) {
|
|
||||||
getRowKeyBefore(set, deleted, candidates, deletes, now);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// The tail didn't contain any keys that matched our criteria, or was
|
|
||||||
// empty. Examine all the keys that proceed our splitting point.
|
|
||||||
getRowKeyBefore(set, search, candidates, deletes, now);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Get row key that comes before passed <code>search_key</code>
|
* Walk forward in a row from <code>firstOnRow</code>. Presumption is that
|
||||||
* Use when we know search_key is not in the map and we need to search
|
* we have been passed the first possible key on a row. As we walk forward
|
||||||
* earlier in the cache.
|
* we accumulate deletes until we hit a candidate on the row at which point
|
||||||
|
* we return.
|
||||||
* @param set
|
* @param set
|
||||||
* @param search
|
* @param firstOnRow First possible key on this row.
|
||||||
* @param candidates
|
* @param state
|
||||||
* @param deletes Pass a Set that has a Comparator that ignores key type.
|
* @return True if we found a candidate walking this row.
|
||||||
* @param now
|
|
||||||
*/
|
*/
|
||||||
private void getRowKeyBefore(NavigableSet<KeyValue> set,
|
private boolean walkForwardInSingleRow(final SortedSet<KeyValue> set,
|
||||||
KeyValue search, NavigableSet<KeyValue> candidates,
|
final KeyValue firstOnRow, final GetClosestRowBeforeTracker state) {
|
||||||
final NavigableSet<KeyValue> deletes, final long now) {
|
boolean foundCandidate = false;
|
||||||
NavigableSet<KeyValue> head = set.headSet(search, false);
|
SortedSet<KeyValue> tail = set.tailSet(firstOnRow);
|
||||||
// If we tried to create a headMap and got an empty map, then there are
|
if (tail.isEmpty()) return foundCandidate;
|
||||||
// no keys at or before the search key, so we're done.
|
for (Iterator<KeyValue> i = tail.iterator(); i.hasNext();) {
|
||||||
if (head.isEmpty()) {
|
KeyValue kv = i.next();
|
||||||
return;
|
// Did we go beyond the target row? If so break.
|
||||||
}
|
if (state.isTooFar(kv, firstOnRow)) break;
|
||||||
|
if (state.isExpired(kv)) {
|
||||||
// If there aren't any candidate keys at this point, we need to search
|
i.remove();
|
||||||
// backwards until we find at least one candidate or run out of headMap.
|
|
||||||
if (candidates.isEmpty()) {
|
|
||||||
KeyValue lastFound = null;
|
|
||||||
// TODO: Confirm we're iterating in the right order
|
|
||||||
for (Iterator<KeyValue> i = head.descendingIterator();
|
|
||||||
i.hasNext();) {
|
|
||||||
KeyValue found = i.next();
|
|
||||||
// if the last row we found a candidate key for is different than
|
|
||||||
// the row of the current candidate, we can stop looking -- if its
|
|
||||||
// not a delete record.
|
|
||||||
boolean deleted = found.isDeleteType();
|
|
||||||
if (lastFound != null &&
|
|
||||||
this.comparator.matchingRows(lastFound, found) && !deleted) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
// If this isn't a delete, record it as a candidate key. Also
|
|
||||||
// take note of this candidate so that we'll know when
|
|
||||||
// we cross the row boundary into the previous row.
|
|
||||||
if (!deleted) {
|
|
||||||
if (Store.notExpiredAndNotInDeletes(this.ttl, found, now, deletes)) {
|
|
||||||
lastFound = found;
|
|
||||||
candidates.add(found);
|
|
||||||
} else {
|
|
||||||
// Its expired.
|
|
||||||
Store.expiredOrDeleted(set, found);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// We are encountering items in reverse. We may have just added
|
|
||||||
// an item to candidates that this later item deletes. Check. If we
|
|
||||||
// found something in candidates, remove it from the set.
|
|
||||||
if (Store.handleDeletes(found, candidates, deletes)) {
|
|
||||||
remove(set, found);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// If there are already some candidate keys, we only need to consider
|
|
||||||
// the very last row's worth of keys in the headMap, because any
|
|
||||||
// smaller acceptable candidate keys would have caused us to start
|
|
||||||
// our search earlier in the list, and we wouldn't be searching here.
|
|
||||||
SortedSet<KeyValue> rowTail =
|
|
||||||
head.tailSet(head.last().cloneRow(HConstants.LATEST_TIMESTAMP));
|
|
||||||
Iterator<KeyValue> i = rowTail.iterator();
|
|
||||||
do {
|
|
||||||
KeyValue found = i.next();
|
|
||||||
if (found.isDeleteType()) {
|
|
||||||
Store.handleDeletes(found, candidates, deletes);
|
|
||||||
} else {
|
|
||||||
if (ttl == HConstants.FOREVER ||
|
|
||||||
now < found.getTimestamp() + ttl ||
|
|
||||||
!deletes.contains(found)) {
|
|
||||||
candidates.add(found);
|
|
||||||
} else {
|
|
||||||
Store.expiredOrDeleted(set, found);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} while (i.hasNext());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* @param set
|
|
||||||
* @param delete This is a delete record. Remove anything behind this of same
|
|
||||||
* r/c/ts.
|
|
||||||
* @return True if we removed anything.
|
|
||||||
*/
|
|
||||||
private boolean remove(final NavigableSet<KeyValue> set,
|
|
||||||
final KeyValue delete) {
|
|
||||||
SortedSet<KeyValue> s = set.tailSet(delete);
|
|
||||||
if (s.isEmpty()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
boolean removed = false;
|
|
||||||
for (KeyValue kv: s) {
|
|
||||||
if (this.comparatorIgnoreType.compare(kv, delete) == 0) {
|
|
||||||
// Same r/c/ts. Remove it.
|
|
||||||
s.remove(kv);
|
|
||||||
removed = true;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// If we added something, this row is a contender. break.
|
||||||
|
if (state.handle(kv)) {
|
||||||
|
foundCandidate = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
return removed;
|
}
|
||||||
|
return foundCandidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Walk backwards through the passed set a row at a time until we run out of
|
||||||
|
* set or until we get a candidate.
|
||||||
|
* @param set
|
||||||
|
* @param state
|
||||||
|
*/
|
||||||
|
private void getRowKeyBefore(NavigableSet<KeyValue> set,
|
||||||
|
final GetClosestRowBeforeTracker state) {
|
||||||
|
KeyValue firstOnRow = state.getTargetKey();
|
||||||
|
for (Member p = memberOfPreviousRow(set, state, firstOnRow);
|
||||||
|
p != null; p = memberOfPreviousRow(p.set, state, firstOnRow)) {
|
||||||
|
// Make sure we don't fall out of our table.
|
||||||
|
if (!state.isTargetTable(p.kv)) break;
|
||||||
|
// Stop looking if we've exited the better candidate range.
|
||||||
|
if (!state.isBetterCandidate(p.kv)) break;
|
||||||
|
// Make into firstOnRow
|
||||||
|
firstOnRow = new KeyValue(p.kv.getRow(), HConstants.LATEST_TIMESTAMP);
|
||||||
|
// If we find something, break;
|
||||||
|
if (walkForwardInSingleRow(p.set, firstOnRow, state)) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Immutable data structure to hold member found in set and the set it was
|
||||||
|
* found in. Include set because it is carrying context.
|
||||||
|
*/
|
||||||
|
private class Member {
|
||||||
|
final KeyValue kv;
|
||||||
|
final NavigableSet<KeyValue> set;
|
||||||
|
Member(final NavigableSet<KeyValue> s, final KeyValue kv) {
|
||||||
|
this.kv = kv;
|
||||||
|
this.set = s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* @param set Set to walk back in. Pass a first in row or we'll return
|
||||||
|
* same row (loop).
|
||||||
|
* @param state Utility and context.
|
||||||
|
* @param firstOnRow First item on the row after the one we want to find a
|
||||||
|
* member in.
|
||||||
|
* @return Null or member of row previous to <code>firstOnRow</code>
|
||||||
|
*/
|
||||||
|
private Member memberOfPreviousRow(NavigableSet<KeyValue> set,
|
||||||
|
final GetClosestRowBeforeTracker state, final KeyValue firstOnRow) {
|
||||||
|
NavigableSet<KeyValue> head = set.headSet(firstOnRow, false);
|
||||||
|
if (head.isEmpty()) return null;
|
||||||
|
for (Iterator<KeyValue> i = head.descendingIterator(); i.hasNext();) {
|
||||||
|
KeyValue found = i.next();
|
||||||
|
if (state.isExpired(found)) {
|
||||||
|
i.remove();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return new Member(head, found);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -689,7 +607,7 @@ public class MemStore implements HeapSize {
|
||||||
}
|
}
|
||||||
|
|
||||||
public final static long FIXED_OVERHEAD = ClassSize.align(
|
public final static long FIXED_OVERHEAD = ClassSize.align(
|
||||||
ClassSize.OBJECT + Bytes.SIZEOF_LONG + (7 * ClassSize.REFERENCE));
|
ClassSize.OBJECT + (7 * ClassSize.REFERENCE));
|
||||||
|
|
||||||
public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
|
public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
|
||||||
ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
|
ClassSize.REENTRANT_LOCK + ClassSize.ATOMIC_LONG +
|
||||||
|
|
|
@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.KeyValue.KeyComparator;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.io.HeapSize;
|
import org.apache.hadoop.hbase.io.HeapSize;
|
||||||
import org.apache.hadoop.io.SequenceFile;
|
|
||||||
import org.apache.hadoop.hbase.io.hfile.Compression;
|
import org.apache.hadoop.hbase.io.hfile.Compression;
|
||||||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||||
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
|
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
|
||||||
|
@ -58,6 +57,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.ClassSize;
|
import org.apache.hadoop.hbase.util.ClassSize;
|
||||||
import org.apache.hadoop.hbase.util.FSUtils;
|
import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
|
import org.apache.hadoop.io.SequenceFile;
|
||||||
import org.apache.hadoop.util.Progressable;
|
import org.apache.hadoop.util.Progressable;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
|
||||||
|
@ -181,7 +181,7 @@ public class Store implements HConstants, HeapSize {
|
||||||
// second -> ms adjust for user data
|
// second -> ms adjust for user data
|
||||||
this.ttl *= 1000;
|
this.ttl *= 1000;
|
||||||
}
|
}
|
||||||
this.memstore = new MemStore(this.ttl, this.comparator);
|
this.memstore = new MemStore(this.comparator);
|
||||||
this.regionCompactionDir = new Path(HRegion.getCompactionDir(basedir),
|
this.regionCompactionDir = new Path(HRegion.getCompactionDir(basedir),
|
||||||
Integer.toString(info.getEncodedName()));
|
Integer.toString(info.getEncodedName()));
|
||||||
this.storeName = this.family.getName();
|
this.storeName = this.family.getName();
|
||||||
|
@ -1028,293 +1028,144 @@ public class Store implements HConstants, HeapSize {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static boolean isExpired(final KeyValue key, final long oldestTimestamp) {
|
||||||
|
return key.getTimestamp() < oldestTimestamp;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find the key that matches <i>row</i> exactly, or the one that immediately
|
* Find the key that matches <i>row</i> exactly, or the one that immediately
|
||||||
* preceeds it. WARNING: Only use this method on a table where writes occur
|
* preceeds it. WARNING: Only use this method on a table where writes occur
|
||||||
* with stricly increasing timestamps. This method assumes this pattern of
|
* with strictly increasing timestamps. This method assumes this pattern of
|
||||||
* writes in order to make it reasonably performant.
|
* writes in order to make it reasonably performant. Also our search is
|
||||||
* @param targetkey
|
* dependent on the axiom that deletes are for cells that are in the container
|
||||||
* @return Found keyvalue
|
* that follows whether a memstore snapshot or a storefile, not for the
|
||||||
|
* current container: i.e. we'll see deletes before we come across cells we
|
||||||
|
* are to delete. Presumption is that the memstore#kvset is processed before
|
||||||
|
* memstore#snapshot and so on.
|
||||||
|
* @param kv First possible item on targeted row; i.e. empty columns, latest
|
||||||
|
* timestamp and maximum type.
|
||||||
|
* @return Found keyvalue or null if none found.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
KeyValue getRowKeyAtOrBefore(final KeyValue targetkey)
|
KeyValue getRowKeyAtOrBefore(final KeyValue kv)
|
||||||
throws IOException{
|
throws IOException {
|
||||||
// Map of keys that are candidates for holding the row key that
|
GetClosestRowBeforeTracker state = new GetClosestRowBeforeTracker(
|
||||||
// most closely matches what we're looking for. We'll have to update it as
|
this.comparator, kv, this.ttl, this.region.getRegionInfo().isMetaRegion());
|
||||||
// deletes are found all over the place as we go along before finally
|
|
||||||
// reading the best key out of it at the end. Use a comparator that
|
|
||||||
// ignores key types. Otherwise, we can't remove deleted items doing
|
|
||||||
// set.remove because of the differing type between insert and delete.
|
|
||||||
NavigableSet<KeyValue> candidates =
|
|
||||||
new TreeSet<KeyValue>(this.comparator.getComparatorIgnoringType());
|
|
||||||
|
|
||||||
// Keep a list of deleted cell keys. We need this because as we go through
|
|
||||||
// the store files, the cell with the delete marker may be in one file and
|
|
||||||
// the old non-delete cell value in a later store file. If we don't keep
|
|
||||||
// around the fact that the cell was deleted in a newer record, we end up
|
|
||||||
// returning the old value if user is asking for more than one version.
|
|
||||||
// This List of deletes should not be large since we are only keeping rows
|
|
||||||
// and columns that match those set on the scanner and which have delete
|
|
||||||
// values. If memory usage becomes an issue, could redo as bloom filter.
|
|
||||||
NavigableSet<KeyValue> deletes =
|
|
||||||
new TreeSet<KeyValue>(this.comparatorIgnoringType);
|
|
||||||
long now = System.currentTimeMillis();
|
|
||||||
this.lock.readLock().lock();
|
this.lock.readLock().lock();
|
||||||
try {
|
try {
|
||||||
// First go to the memstore. Pick up deletes and candidates.
|
// First go to the memstore. Pick up deletes and candidates.
|
||||||
this.memstore.getRowKeyAtOrBefore(targetkey, candidates, deletes, now);
|
this.memstore.getRowKeyAtOrBefore(state);
|
||||||
|
// Check if match, if we got a candidate on the asked for 'kv' row.
|
||||||
// Process each store file. Run through from newest to oldest.
|
// Process each store file. Run through from newest to oldest.
|
||||||
Map<Long, StoreFile> m = this.storefiles.descendingMap();
|
Map<Long, StoreFile> m = this.storefiles.descendingMap();
|
||||||
for (Map.Entry<Long, StoreFile> e: m.entrySet()) {
|
for (Map.Entry<Long, StoreFile> e : m.entrySet()) {
|
||||||
// Update the candidate keys from the current map file
|
// Update the candidate keys from the current map file
|
||||||
rowAtOrBeforeFromStoreFile(e.getValue(), targetkey, candidates,
|
rowAtOrBeforeFromStoreFile(e.getValue(), state);
|
||||||
deletes, now);
|
|
||||||
}
|
}
|
||||||
// Return the best key from candidateKeys
|
return state.getCandidate();
|
||||||
return candidates.isEmpty()? null: candidates.last();
|
|
||||||
} finally {
|
} finally {
|
||||||
this.lock.readLock().unlock();
|
this.lock.readLock().unlock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Check an individual MapFile for the row at or before a given key
|
* Check an individual MapFile for the row at or before a given row.
|
||||||
* and timestamp
|
|
||||||
* @param f
|
* @param f
|
||||||
* @param targetkey
|
* @param state
|
||||||
* @param candidates Pass a Set with a Comparator that
|
|
||||||
* ignores key Type so we can do Set.remove using a delete, i.e. a KeyValue
|
|
||||||
* with a different Type to the candidate key.
|
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
private void rowAtOrBeforeFromStoreFile(final StoreFile f,
|
private void rowAtOrBeforeFromStoreFile(final StoreFile f,
|
||||||
final KeyValue targetkey, final NavigableSet<KeyValue> candidates,
|
final GetClosestRowBeforeTracker state)
|
||||||
final NavigableSet<KeyValue> deletes, final long now)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// if there aren't any candidate keys yet, we'll do some things different
|
Reader r = f.getReader();
|
||||||
if (candidates.isEmpty()) {
|
if (r == null) {
|
||||||
rowAtOrBeforeCandidate(f, targetkey, candidates, deletes, now);
|
LOG.warn("StoreFile " + f + " has a null Reader");
|
||||||
} else {
|
return;
|
||||||
rowAtOrBeforeWithCandidates(f, targetkey, candidates, deletes, now);
|
}
|
||||||
|
// TODO: Cache these keys rather than make each time?
|
||||||
|
byte [] fk = r.getFirstKey();
|
||||||
|
KeyValue firstKV = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
|
||||||
|
byte [] lk = r.getLastKey();
|
||||||
|
KeyValue lastKV = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
|
||||||
|
KeyValue firstOnRow = state.getTargetKey();
|
||||||
|
if (this.comparator.compareRows(lastKV, firstOnRow) < 0) {
|
||||||
|
// If last key in file is not of the target table, no candidates in this
|
||||||
|
// file. Return.
|
||||||
|
if (!state.isTargetTable(lastKV)) return;
|
||||||
|
// If the row we're looking for is past the end of file, set search key to
|
||||||
|
// last key. TODO: Cache last and first key rather than make each time.
|
||||||
|
firstOnRow = new KeyValue(lastKV.getRow(), HConstants.LATEST_TIMESTAMP);
|
||||||
|
}
|
||||||
|
HFileScanner scanner = r.getScanner();
|
||||||
|
// Seek scanner. If can't seek it, return.
|
||||||
|
if (!seekToScanner(scanner, firstOnRow, firstKV)) return;
|
||||||
|
// If we found candidate on firstOnRow, just return. THIS WILL NEVER HAPPEN!
|
||||||
|
// Unlikely that there'll be an instance of actual first row in table.
|
||||||
|
if (walkForwardInSingleRow(scanner, firstOnRow, state)) return;
|
||||||
|
// If here, need to start backing up.
|
||||||
|
while (scanner.seekBefore(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(),
|
||||||
|
firstOnRow.getKeyLength())) {
|
||||||
|
KeyValue kv = scanner.getKeyValue();
|
||||||
|
if (!state.isTargetTable(kv)) break;
|
||||||
|
if (!state.isBetterCandidate(kv)) break;
|
||||||
|
// Make new first on row.
|
||||||
|
firstOnRow = new KeyValue(kv.getRow(), HConstants.LATEST_TIMESTAMP);
|
||||||
|
// Seek scanner. If can't seek it, break.
|
||||||
|
if (!seekToScanner(scanner, firstOnRow, firstKV)) break;
|
||||||
|
// If we find something, break;
|
||||||
|
if (walkForwardInSingleRow(scanner, firstOnRow, state)) break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* @param ttlSetting
|
* Seek the file scanner to firstOnRow or first entry in file.
|
||||||
* @param hsk
|
* @param scanner
|
||||||
* @param now
|
* @param firstOnRow
|
||||||
* @param deletes A Set whose Comparator ignores Type.
|
* @param firstKV
|
||||||
* @return True if key has not expired and is not in passed set of deletes.
|
* @return True if we successfully seeked scanner.
|
||||||
*/
|
|
||||||
static boolean notExpiredAndNotInDeletes(final long ttl,
|
|
||||||
final KeyValue key, final long now, final Set<KeyValue> deletes) {
|
|
||||||
return !isExpired(key, now-ttl) && (deletes == null || deletes.isEmpty() ||
|
|
||||||
!deletes.contains(key));
|
|
||||||
}
|
|
||||||
|
|
||||||
static boolean isExpired(final KeyValue key, final long oldestTimestamp) {
|
|
||||||
return key.getTimestamp() < oldestTimestamp;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Find a candidate for row that is at or before passed key, searchkey, in hfile.
|
|
||||||
* @param f
|
|
||||||
* @param targetkey Key to go search the hfile with.
|
|
||||||
* @param candidates
|
|
||||||
* @param now
|
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @see {@link #rowAtOrBeforeCandidate(HStoreKey, org.apache.hadoop.io.MapFile.Reader, byte[], SortedMap, long)}
|
|
||||||
*/
|
*/
|
||||||
private void rowAtOrBeforeCandidate(final StoreFile f,
|
private boolean seekToScanner(final HFileScanner scanner,
|
||||||
final KeyValue targetkey, final NavigableSet<KeyValue> candidates,
|
final KeyValue firstOnRow, final KeyValue firstKV)
|
||||||
final NavigableSet<KeyValue> deletes, final long now)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
KeyValue search = targetkey;
|
KeyValue kv = firstOnRow;
|
||||||
// If the row we're looking for is past the end of this mapfile, set the
|
// If firstOnRow < firstKV, set to firstKV
|
||||||
// search key to be the last key. If its a deleted key, then we'll back
|
if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV;
|
||||||
// up to the row before and return that.
|
int result = scanner.seekTo(kv.getBuffer(), kv.getKeyOffset(),
|
||||||
// TODO: Cache last key as KV over in the file.
|
kv.getKeyLength());
|
||||||
Reader r = f.getReader();
|
return result >= 0;
|
||||||
if (r == null) {
|
|
||||||
LOG.warn("StoreFile " + f + " has a null Reader");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
byte [] lastkey = r.getLastKey();
|
|
||||||
KeyValue lastKeyValue =
|
|
||||||
KeyValue.createKeyValueFromKey(lastkey, 0, lastkey.length);
|
|
||||||
if (this.comparator.compareRows(lastKeyValue, targetkey) < 0) {
|
|
||||||
search = lastKeyValue;
|
|
||||||
}
|
|
||||||
KeyValue knownNoGoodKey = null;
|
|
||||||
HFileScanner scanner = r.getScanner();
|
|
||||||
for (boolean foundCandidate = false; !foundCandidate;) {
|
|
||||||
// Seek to the exact row, or the one that would be immediately before it
|
|
||||||
int result = scanner.seekTo(search.getBuffer(), search.getKeyOffset(),
|
|
||||||
search.getKeyLength());
|
|
||||||
if (result < 0) {
|
|
||||||
// Not in file.
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
KeyValue deletedOrExpiredRow = null;
|
|
||||||
KeyValue kv = null;
|
|
||||||
do {
|
|
||||||
kv = scanner.getKeyValue();
|
|
||||||
if (this.comparator.compareRows(kv, search) <= 0) {
|
|
||||||
if (!kv.isDeleteType()) {
|
|
||||||
if (handleNonDelete(kv, now, deletes, candidates)) {
|
|
||||||
foundCandidate = true;
|
|
||||||
// NOTE! Continue.
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
deletes.add(kv);
|
|
||||||
if (deletedOrExpiredRow == null) {
|
|
||||||
deletedOrExpiredRow = kv;
|
|
||||||
}
|
|
||||||
} else if (this.comparator.compareRows(kv, search) > 0) {
|
|
||||||
// if the row key we just read is beyond the key we're searching for,
|
|
||||||
// then we're done.
|
|
||||||
break;
|
|
||||||
} else {
|
|
||||||
// So, the row key doesn't match, but we haven't gone past the row
|
|
||||||
// we're seeking yet, so this row is a candidate for closest
|
|
||||||
// (assuming that it isn't a delete).
|
|
||||||
if (!kv.isDeleteType()) {
|
|
||||||
if (handleNonDelete(kv, now, deletes, candidates)) {
|
|
||||||
foundCandidate = true;
|
|
||||||
// NOTE: Continue
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
deletes.add(kv);
|
|
||||||
if (deletedOrExpiredRow == null) {
|
|
||||||
deletedOrExpiredRow = kv;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} while(scanner.next() && (knownNoGoodKey == null ||
|
|
||||||
this.comparator.compare(kv, knownNoGoodKey) < 0));
|
|
||||||
|
|
||||||
// If we get here and have no candidates but we did find a deleted or
|
|
||||||
// expired candidate, we need to look at the key before that
|
|
||||||
if (!foundCandidate && deletedOrExpiredRow != null) {
|
|
||||||
knownNoGoodKey = deletedOrExpiredRow;
|
|
||||||
if (!scanner.seekBefore(deletedOrExpiredRow.getBuffer(),
|
|
||||||
deletedOrExpiredRow.getKeyOffset(),
|
|
||||||
deletedOrExpiredRow.getKeyLength())) {
|
|
||||||
// Not in file -- what can I do now but break?
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
search = scanner.getKeyValue();
|
|
||||||
} else {
|
|
||||||
// No candidates and no deleted or expired candidates. Give up.
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Arriving here just means that we consumed the whole rest of the map
|
/*
|
||||||
// without going "past" the key we're searching for. we can just fall
|
* When we come in here, we are probably at the kv just before we break into
|
||||||
// through here.
|
* the row that firstOnRow is on. Usually need to increment one time to get
|
||||||
}
|
* on to the row we are interested in.
|
||||||
|
* @param scanner
|
||||||
private void rowAtOrBeforeWithCandidates(final StoreFile f,
|
* @param firstOnRow
|
||||||
final KeyValue targetkey,
|
* @param state
|
||||||
final NavigableSet<KeyValue> candidates,
|
* @return True we found a candidate.
|
||||||
final NavigableSet<KeyValue> deletes, final long now)
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private boolean walkForwardInSingleRow(final HFileScanner scanner,
|
||||||
|
final KeyValue firstOnRow, final GetClosestRowBeforeTracker state)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// if there are already candidate keys, we need to start our search
|
boolean foundCandidate = false;
|
||||||
// at the earliest possible key so that we can discover any possible
|
|
||||||
// deletes for keys between the start and the search key. Back up to start
|
|
||||||
// of the row in case there are deletes for this candidate in this mapfile
|
|
||||||
// BUT do not backup before the first key in the store file.
|
|
||||||
KeyValue firstCandidateKey = candidates.first();
|
|
||||||
KeyValue search = null;
|
|
||||||
if (this.comparator.compareRows(firstCandidateKey, targetkey) < 0) {
|
|
||||||
search = targetkey;
|
|
||||||
} else {
|
|
||||||
search = firstCandidateKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Seek to the exact row, or the one that would be immediately before it
|
|
||||||
Reader r = f.getReader();
|
|
||||||
if (r == null) {
|
|
||||||
LOG.warn("StoreFile " + f + " has a null Reader");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
HFileScanner scanner = r.getScanner();
|
|
||||||
int result = scanner.seekTo(search.getBuffer(), search.getKeyOffset(),
|
|
||||||
search.getKeyLength());
|
|
||||||
if (result < 0) {
|
|
||||||
// Key is before start of this file. Return.
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
do {
|
do {
|
||||||
KeyValue kv = scanner.getKeyValue();
|
KeyValue kv = scanner.getKeyValue();
|
||||||
// if we have an exact match on row, and it's not a delete, save this
|
// If we are not in the row, skip.
|
||||||
// as a candidate key
|
if (this.comparator.compareRows(kv, firstOnRow) < 0) continue;
|
||||||
if (this.comparator.matchingRows(kv, targetkey)) {
|
// Did we go beyond the target row? If so break.
|
||||||
handleKey(kv, now, deletes, candidates);
|
if (state.isTooFar(kv, firstOnRow)) break;
|
||||||
} else if (this.comparator.compareRows(kv, targetkey) > 0 ) {
|
if (state.isExpired(kv)) {
|
||||||
// if the row key we just read is beyond the key we're searching for,
|
continue;
|
||||||
// then we're done.
|
}
|
||||||
|
// If we added something, this row is a contender. break.
|
||||||
|
if (state.handle(kv)) {
|
||||||
|
foundCandidate = true;
|
||||||
break;
|
break;
|
||||||
} else {
|
|
||||||
// So, the row key doesn't match, but we haven't gone past the row
|
|
||||||
// we're seeking yet, so this row is a candidate for closest
|
|
||||||
// (assuming that it isn't a delete).
|
|
||||||
handleKey(kv, now, deletes, candidates);
|
|
||||||
}
|
}
|
||||||
} while(scanner.next());
|
} while(scanner.next());
|
||||||
}
|
return foundCandidate;
|
||||||
|
|
||||||
/*
|
|
||||||
* Used calculating keys at or just before a passed key.
|
|
||||||
* @param readkey
|
|
||||||
* @param now
|
|
||||||
* @param deletes Set with Comparator that ignores key type.
|
|
||||||
* @param candidate Set with Comprator that ignores key type.
|
|
||||||
*/
|
|
||||||
private void handleKey(final KeyValue readkey, final long now,
|
|
||||||
final NavigableSet<KeyValue> deletes,
|
|
||||||
final NavigableSet<KeyValue> candidates) {
|
|
||||||
if (!readkey.isDeleteType()) {
|
|
||||||
handleNonDelete(readkey, now, deletes, candidates);
|
|
||||||
} else {
|
|
||||||
handleDeletes(readkey, candidates, deletes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Used calculating keys at or just before a passed key.
|
|
||||||
* @param readkey
|
|
||||||
* @param now
|
|
||||||
* @param deletes Set with Comparator that ignores key type.
|
|
||||||
* @param candidates Set with Comparator that ignores key type.
|
|
||||||
* @return True if we added a candidate.
|
|
||||||
*/
|
|
||||||
private boolean handleNonDelete(final KeyValue readkey, final long now,
|
|
||||||
final NavigableSet<KeyValue> deletes,
|
|
||||||
final NavigableSet<KeyValue> candidates) {
|
|
||||||
if (notExpiredAndNotInDeletes(this.ttl, readkey, now, deletes)) {
|
|
||||||
candidates.add(readkey);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle keys whose values hold deletes.
|
|
||||||
* Add to the set of deletes and then if the candidate keys contain any that
|
|
||||||
* might match, then check for a match and remove it. Implies candidates
|
|
||||||
* is made with a Comparator that ignores key type.
|
|
||||||
* @param k
|
|
||||||
* @param candidates
|
|
||||||
* @param deletes
|
|
||||||
* @return True if we removed <code>k</code> from <code>candidates</code>.
|
|
||||||
*/
|
|
||||||
static boolean handleDeletes(final KeyValue k,
|
|
||||||
final NavigableSet<KeyValue> candidates,
|
|
||||||
final NavigableSet<KeyValue> deletes) {
|
|
||||||
deletes.add(k);
|
|
||||||
return candidates.remove(k);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -145,7 +145,6 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
|
||||||
* @return true if there are more rows, false if scanner is done
|
* @return true if there are more rows, false if scanner is done
|
||||||
*/
|
*/
|
||||||
public synchronized boolean next(List<KeyValue> outResult) throws IOException {
|
public synchronized boolean next(List<KeyValue> outResult) throws IOException {
|
||||||
List<KeyValue> results = new ArrayList<KeyValue>();
|
|
||||||
KeyValue peeked = this.heap.peek();
|
KeyValue peeked = this.heap.peek();
|
||||||
if (peeked == null) {
|
if (peeked == null) {
|
||||||
close();
|
close();
|
||||||
|
@ -153,6 +152,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
|
||||||
}
|
}
|
||||||
matcher.setRow(peeked.getRow());
|
matcher.setRow(peeked.getRow());
|
||||||
KeyValue kv;
|
KeyValue kv;
|
||||||
|
List<KeyValue> results = new ArrayList<KeyValue>();
|
||||||
while((kv = this.heap.peek()) != null) {
|
while((kv = this.heap.peek()) != null) {
|
||||||
QueryMatcher.MatchCode qcode = matcher.match(kv);
|
QueryMatcher.MatchCode qcode = matcher.match(kv);
|
||||||
switch(qcode) {
|
switch(qcode) {
|
||||||
|
@ -162,7 +162,6 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
case DONE:
|
case DONE:
|
||||||
|
|
||||||
// copy jazz
|
// copy jazz
|
||||||
outResult.addAll(results);
|
outResult.addAll(results);
|
||||||
return true;
|
return true;
|
||||||
|
@ -198,7 +197,6 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
|
||||||
if (!results.isEmpty()) {
|
if (!results.isEmpty()) {
|
||||||
// copy jazz
|
// copy jazz
|
||||||
outResult.addAll(results);
|
outResult.addAll(results);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -242,7 +242,7 @@ public class Bytes {
|
||||||
* @return String made from <code>b</code>
|
* @return String made from <code>b</code>
|
||||||
*/
|
*/
|
||||||
public static String toString(final byte [] b) {
|
public static String toString(final byte [] b) {
|
||||||
if(b == null) {
|
if (b == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return toString(b, 0, b.length);
|
return toString(b, 0, b.length);
|
||||||
|
|
|
@ -62,8 +62,7 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
|
protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
|
||||||
protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
|
protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
|
||||||
protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
|
protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
|
||||||
protected static final byte [][] COLUMNS = {fam1,
|
protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
|
||||||
fam2, fam3};
|
|
||||||
|
|
||||||
private boolean localfs = false;
|
private boolean localfs = false;
|
||||||
protected Path testDir = null;
|
protected Path testDir = null;
|
||||||
|
|
|
@ -258,8 +258,8 @@ public class TestCompaction extends HBaseTestCase {
|
||||||
|
|
||||||
private void createSmallerStoreFile(final HRegion region) throws IOException {
|
private void createSmallerStoreFile(final HRegion region) throws IOException {
|
||||||
HRegionIncommon loader = new HRegionIncommon(region);
|
HRegionIncommon loader = new HRegionIncommon(region);
|
||||||
addContent(loader, Bytes.toString(COLUMN_FAMILY),
|
addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
|
||||||
("bbb").getBytes(), null);
|
"bbb").getBytes(), null);
|
||||||
loader.flushcache();
|
loader.flushcache();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,350 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2009 The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.regionserver;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||||
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
|
import org.apache.hadoop.hbase.client.TestGet;
|
||||||
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.hadoop.hbase.util.Writables;
|
||||||
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link TestGet} is a medley of tests of get all done up as a single test.
|
||||||
|
* This class
|
||||||
|
*/
|
||||||
|
public class TestGetClosestAtOrBefore extends HBaseTestCase implements HConstants {
|
||||||
|
static final Log LOG = LogFactory.getLog(TestGetClosestAtOrBefore.class);
|
||||||
|
private MiniDFSCluster miniHdfs;
|
||||||
|
|
||||||
|
private static final byte [] T00 = Bytes.toBytes("000");
|
||||||
|
private static final byte [] T10 = Bytes.toBytes("010");
|
||||||
|
private static final byte [] T11 = Bytes.toBytes("011");
|
||||||
|
private static final byte [] T12 = Bytes.toBytes("012");
|
||||||
|
private static final byte [] T20 = Bytes.toBytes("020");
|
||||||
|
private static final byte [] T30 = Bytes.toBytes("030");
|
||||||
|
private static final byte [] T31 = Bytes.toBytes("031");
|
||||||
|
private static final byte [] T35 = Bytes.toBytes("035");
|
||||||
|
private static final byte [] T40 = Bytes.toBytes("040");
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
this.miniHdfs = new MiniDFSCluster(this.conf, 1, true, null);
|
||||||
|
// Set the hbase.rootdir to be the home directory in mini dfs.
|
||||||
|
this.conf.set(HConstants.HBASE_DIR,
|
||||||
|
this.miniHdfs.getFileSystem().getHomeDirectory().toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testUsingMetaAndBinary() throws IOException {
|
||||||
|
FileSystem filesystem = FileSystem.get(conf);
|
||||||
|
Path rootdir = filesystem.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
|
||||||
|
filesystem.mkdirs(rootdir);
|
||||||
|
// Up flush size else we bind up when we use default catalog flush of 16k.
|
||||||
|
HRegionInfo.FIRST_META_REGIONINFO.getTableDesc().
|
||||||
|
setMemStoreFlushSize(64 * 1024 * 1024);
|
||||||
|
HRegion mr = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO,
|
||||||
|
rootdir, this.conf);
|
||||||
|
// Write rows for three tables 'A', 'B', and 'C'.
|
||||||
|
for (char c = 'A'; c < 'D'; c++) {
|
||||||
|
HTableDescriptor htd = new HTableDescriptor("" + c);
|
||||||
|
final int last = 128;
|
||||||
|
final int interval = 2;
|
||||||
|
for (int i = 0; i <= last; i += interval) {
|
||||||
|
HRegionInfo hri = new HRegionInfo(htd,
|
||||||
|
i == 0? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i),
|
||||||
|
i == last? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i + interval));
|
||||||
|
Put put = new Put(hri.getRegionName());
|
||||||
|
put.add(CATALOG_FAMILY, REGIONINFO_QUALIFIER, Writables.getBytes(hri));
|
||||||
|
mr.put(put, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
InternalScanner s = mr.getScanner(new Scan());
|
||||||
|
try {
|
||||||
|
List<KeyValue> keys = new ArrayList<KeyValue>();
|
||||||
|
while(s.next(keys)) {
|
||||||
|
LOG.info(keys);
|
||||||
|
keys.clear();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
s.close();
|
||||||
|
}
|
||||||
|
findRow(mr, 'C', 44, 44);
|
||||||
|
findRow(mr, 'C', 45, 44);
|
||||||
|
findRow(mr, 'C', 46, 46);
|
||||||
|
findRow(mr, 'C', 43, 42);
|
||||||
|
mr.flushcache();
|
||||||
|
findRow(mr, 'C', 44, 44);
|
||||||
|
findRow(mr, 'C', 45, 44);
|
||||||
|
findRow(mr, 'C', 46, 46);
|
||||||
|
findRow(mr, 'C', 43, 42);
|
||||||
|
// Now delete 'C' and make sure I don't get entries from 'B'.
|
||||||
|
byte [] firstRowInC = HRegionInfo.createRegionName(Bytes.toBytes("" + 'C'),
|
||||||
|
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES);
|
||||||
|
Scan scan = new Scan(firstRowInC);
|
||||||
|
s = mr.getScanner(scan);
|
||||||
|
try {
|
||||||
|
List<KeyValue> keys = new ArrayList<KeyValue>();
|
||||||
|
while (s.next(keys)) {
|
||||||
|
mr.delete(new Delete(keys.get(0).getRow()), null, false);
|
||||||
|
keys.clear();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
s.close();
|
||||||
|
}
|
||||||
|
// Assert we get null back (pass -1).
|
||||||
|
findRow(mr, 'C', 44, -1);
|
||||||
|
findRow(mr, 'C', 45, -1);
|
||||||
|
findRow(mr, 'C', 46, -1);
|
||||||
|
findRow(mr, 'C', 43, -1);
|
||||||
|
mr.flushcache();
|
||||||
|
findRow(mr, 'C', 44, -1);
|
||||||
|
findRow(mr, 'C', 45, -1);
|
||||||
|
findRow(mr, 'C', 46, -1);
|
||||||
|
findRow(mr, 'C', 43, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* @param mr
|
||||||
|
* @param table
|
||||||
|
* @param rowToFind
|
||||||
|
* @param answer Pass -1 if we're not to find anything.
|
||||||
|
* @return Row found.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private byte [] findRow(final HRegion mr, final char table,
|
||||||
|
final int rowToFind, final int answer)
|
||||||
|
throws IOException {
|
||||||
|
byte [] tableb = Bytes.toBytes("" + table);
|
||||||
|
// Find the row.
|
||||||
|
byte [] tofindBytes = Bytes.toBytes((short)rowToFind);
|
||||||
|
byte [] metaKey = HRegionInfo.createRegionName(tableb, tofindBytes,
|
||||||
|
HConstants.NINES);
|
||||||
|
LOG.info("find=" + new String(metaKey));
|
||||||
|
Result r = mr.getClosestRowBefore(metaKey);
|
||||||
|
if (answer == -1) {
|
||||||
|
assertNull(r);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
assertTrue(Bytes.compareTo(Bytes.toBytes((short)answer),
|
||||||
|
extractRowFromMetaRow(r.getRow())) == 0);
|
||||||
|
return r.getRow();
|
||||||
|
}
|
||||||
|
|
||||||
|
private byte [] extractRowFromMetaRow(final byte [] b) {
|
||||||
|
int firstDelimiter = KeyValue.getDelimiter(b, 0, b.length,
|
||||||
|
HRegionInfo.DELIMITER);
|
||||||
|
int lastDelimiter = KeyValue.getDelimiterInReverse(b, 0, b.length,
|
||||||
|
HRegionInfo.DELIMITER);
|
||||||
|
int length = lastDelimiter - firstDelimiter - 1;
|
||||||
|
byte [] row = new byte[length];
|
||||||
|
System.arraycopy(b, firstDelimiter + 1, row, 0, length);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test file of multiple deletes and with deletes as final key.
|
||||||
|
* @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
|
||||||
|
*/
|
||||||
|
public void testGetClosestRowBefore3() throws IOException{
|
||||||
|
HRegion region = null;
|
||||||
|
byte [] c0 = COLUMNS[0];
|
||||||
|
byte [] c1 = COLUMNS[1];
|
||||||
|
try {
|
||||||
|
HTableDescriptor htd = createTableDescriptor(getName());
|
||||||
|
region = createNewHRegion(htd, null, null);
|
||||||
|
|
||||||
|
Put p = new Put(T00);
|
||||||
|
p.add(c0, c0, T00);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
p = new Put(T10);
|
||||||
|
p.add(c0, c0, T10);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
p = new Put(T20);
|
||||||
|
p.add(c0, c0, T20);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
Result r = region.getClosestRowBefore(T20, c0);
|
||||||
|
assertTrue(Bytes.equals(T20, r.getRow()));
|
||||||
|
|
||||||
|
Delete d = new Delete(T20);
|
||||||
|
d.deleteColumn(c0, c0);
|
||||||
|
region.delete(d, null, false);
|
||||||
|
|
||||||
|
r = region.getClosestRowBefore(T20, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
p = new Put(T30);
|
||||||
|
p.add(c0, c0, T30);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
r = region.getClosestRowBefore(T30, c0);
|
||||||
|
assertTrue(Bytes.equals(T30, r.getRow()));
|
||||||
|
|
||||||
|
d = new Delete(T30);
|
||||||
|
d.deleteColumn(c0, c0);
|
||||||
|
region.delete(d, null, false);
|
||||||
|
|
||||||
|
r = region.getClosestRowBefore(T30, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
r = region.getClosestRowBefore(T31, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
region.flushcache();
|
||||||
|
|
||||||
|
// try finding "010" after flush
|
||||||
|
r = region.getClosestRowBefore(T30, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
r = region.getClosestRowBefore(T31, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
// Put into a different column family. Should make it so I still get t10
|
||||||
|
p = new Put(T20);
|
||||||
|
p.add(c1, c1, T20);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
r = region.getClosestRowBefore(T30, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
r = region.getClosestRowBefore(T31, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
region.flushcache();
|
||||||
|
|
||||||
|
r = region.getClosestRowBefore(T30, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
r = region.getClosestRowBefore(T31, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
// Now try combo of memcache and mapfiles. Delete the t20 COLUMS[1]
|
||||||
|
// in memory; make sure we get back t10 again.
|
||||||
|
d = new Delete(T20);
|
||||||
|
d.deleteColumn(c1, c1);
|
||||||
|
region.delete(d, null, false);
|
||||||
|
r = region.getClosestRowBefore(T30, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
// Ask for a value off the end of the file. Should return t10.
|
||||||
|
r = region.getClosestRowBefore(T31, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
region.flushcache();
|
||||||
|
r = region.getClosestRowBefore(T31, c0);
|
||||||
|
assertTrue(Bytes.equals(T10, r.getRow()));
|
||||||
|
|
||||||
|
// Ok. Let the candidate come out of hfile but have delete of
|
||||||
|
// the candidate be in memory.
|
||||||
|
p = new Put(T11);
|
||||||
|
p.add(c0, c0, T11);
|
||||||
|
region.put(p);
|
||||||
|
d = new Delete(T10);
|
||||||
|
d.deleteColumn(c1, c1);
|
||||||
|
r = region.getClosestRowBefore(T12, c0);
|
||||||
|
assertTrue(Bytes.equals(T11, r.getRow()));
|
||||||
|
} finally {
|
||||||
|
if (region != null) {
|
||||||
|
try {
|
||||||
|
region.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
region.getLog().closeAndDelete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** For HBASE-694 */
|
||||||
|
public void testGetClosestRowBefore2() throws IOException{
|
||||||
|
HRegion region = null;
|
||||||
|
byte [] c0 = COLUMNS[0];
|
||||||
|
try {
|
||||||
|
HTableDescriptor htd = createTableDescriptor(getName());
|
||||||
|
region = createNewHRegion(htd, null, null);
|
||||||
|
|
||||||
|
Put p = new Put(T10);
|
||||||
|
p.add(c0, c0, T10);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
p = new Put(T30);
|
||||||
|
p.add(c0, c0, T30);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
p = new Put(T40);
|
||||||
|
p.add(c0, c0, T40);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
// try finding "035"
|
||||||
|
Result r = region.getClosestRowBefore(T35, c0);
|
||||||
|
assertTrue(Bytes.equals(T30, r.getRow()));
|
||||||
|
|
||||||
|
region.flushcache();
|
||||||
|
|
||||||
|
// try finding "035"
|
||||||
|
r = region.getClosestRowBefore(T35, c0);
|
||||||
|
assertTrue(Bytes.equals(T30, r.getRow()));
|
||||||
|
|
||||||
|
p = new Put(T20);
|
||||||
|
p.add(c0, c0, T20);
|
||||||
|
region.put(p);
|
||||||
|
|
||||||
|
// try finding "035"
|
||||||
|
r = region.getClosestRowBefore(T35, c0);
|
||||||
|
assertTrue(Bytes.equals(T30, r.getRow()));
|
||||||
|
|
||||||
|
region.flushcache();
|
||||||
|
|
||||||
|
// try finding "035"
|
||||||
|
r = region.getClosestRowBefore(T35, c0);
|
||||||
|
assertTrue(Bytes.equals(T30, r.getRow()));
|
||||||
|
} finally {
|
||||||
|
if (region != null) {
|
||||||
|
try {
|
||||||
|
region.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
region.getLog().closeAndDelete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void tearDown() throws Exception {
|
||||||
|
if (this.miniHdfs != null) {
|
||||||
|
this.miniHdfs.shutdown();
|
||||||
|
}
|
||||||
|
super.tearDown();
|
||||||
|
}
|
||||||
|
}
|
|
@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.regionserver;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
|
@ -35,7 +34,6 @@ import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.KeyValue;
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.UnknownScannerException;
|
|
||||||
import org.apache.hadoop.hbase.client.Delete;
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
|
|
|
@ -42,7 +42,7 @@ public class TestMemStore extends TestCase {
|
||||||
private final Log LOG = LogFactory.getLog(this.getClass());
|
private final Log LOG = LogFactory.getLog(this.getClass());
|
||||||
private MemStore memstore;
|
private MemStore memstore;
|
||||||
private static final int ROW_COUNT = 10;
|
private static final int ROW_COUNT = 10;
|
||||||
private static final int QUALIFIER_COUNT = 10;
|
private static final int QUALIFIER_COUNT = ROW_COUNT;
|
||||||
private static final byte [] FAMILY = Bytes.toBytes("column");
|
private static final byte [] FAMILY = Bytes.toBytes("column");
|
||||||
private static final byte [] CONTENTS_BASIC = Bytes.toBytes("contents:basic");
|
private static final byte [] CONTENTS_BASIC = Bytes.toBytes("contents:basic");
|
||||||
private static final String CONTENTSTR = "contentstr";
|
private static final String CONTENTSTR = "contentstr";
|
||||||
|
@ -82,6 +82,8 @@ public class TestMemStore extends TestCase {
|
||||||
while (s.next(result)) {
|
while (s.next(result)) {
|
||||||
LOG.info(result);
|
LOG.info(result);
|
||||||
count++;
|
count++;
|
||||||
|
// Row count is same as column count.
|
||||||
|
assertEquals(rowCount, result.size());
|
||||||
result.clear();
|
result.clear();
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
@ -98,6 +100,8 @@ public class TestMemStore extends TestCase {
|
||||||
// Assert the stuff is coming out in right order.
|
// Assert the stuff is coming out in right order.
|
||||||
assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
|
assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
|
||||||
count++;
|
count++;
|
||||||
|
// Row count is same as column count.
|
||||||
|
assertEquals(rowCount, result.size());
|
||||||
if (count == 2) {
|
if (count == 2) {
|
||||||
this.memstore.snapshot();
|
this.memstore.snapshot();
|
||||||
LOG.info("Snapshotted");
|
LOG.info("Snapshotted");
|
||||||
|
@ -108,6 +112,34 @@ public class TestMemStore extends TestCase {
|
||||||
s.close();
|
s.close();
|
||||||
}
|
}
|
||||||
assertEquals(rowCount, count);
|
assertEquals(rowCount, count);
|
||||||
|
// Assert that new values are seen in kvset as we scan.
|
||||||
|
long ts = System.currentTimeMillis();
|
||||||
|
s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
|
||||||
|
this.memstore.comparator, null, memstorescanners);
|
||||||
|
count = 0;
|
||||||
|
int snapshotIndex = 5;
|
||||||
|
try {
|
||||||
|
while (s.next(result)) {
|
||||||
|
LOG.info(result);
|
||||||
|
// Assert the stuff is coming out in right order.
|
||||||
|
assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
|
||||||
|
// Row count is same as column count.
|
||||||
|
// TODO PUTBACK assertEquals("count=" + count + ", result=" + result,
|
||||||
|
// rowCount, result.size());
|
||||||
|
count++;
|
||||||
|
if (count == snapshotIndex) {
|
||||||
|
this.memstore.snapshot();
|
||||||
|
this.memstore.clearSnapshot(this.memstore.getSnapshot());
|
||||||
|
// Added more rows into kvset.
|
||||||
|
addRows(this.memstore, ts);
|
||||||
|
LOG.info("Snapshotted, cleared it and then added values");
|
||||||
|
}
|
||||||
|
result.clear();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
s.close();
|
||||||
|
}
|
||||||
|
assertEquals(rowCount, count);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -126,7 +158,7 @@ public class TestMemStore extends TestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testMultipleVersionsSimple() throws Exception {
|
public void testMultipleVersionsSimple() throws Exception {
|
||||||
MemStore m = new MemStore(HConstants.FOREVER, KeyValue.COMPARATOR);
|
MemStore m = new MemStore(KeyValue.COMPARATOR);
|
||||||
byte [] row = Bytes.toBytes("testRow");
|
byte [] row = Bytes.toBytes("testRow");
|
||||||
byte [] family = Bytes.toBytes("testFamily");
|
byte [] family = Bytes.toBytes("testFamily");
|
||||||
byte [] qf = Bytes.toBytes("testQualifier");
|
byte [] qf = Bytes.toBytes("testQualifier");
|
||||||
|
@ -146,7 +178,7 @@ public class TestMemStore extends TestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testBinary() throws IOException {
|
public void testBinary() throws IOException {
|
||||||
MemStore mc = new MemStore(HConstants.FOREVER, KeyValue.ROOT_COMPARATOR);
|
MemStore mc = new MemStore(KeyValue.ROOT_COMPARATOR);
|
||||||
final int start = 43;
|
final int start = 43;
|
||||||
final int end = 46;
|
final int end = 46;
|
||||||
for (int k = start; k <= end; k++) {
|
for (int k = start; k <= end; k++) {
|
||||||
|
@ -180,65 +212,6 @@ public class TestMemStore extends TestCase {
|
||||||
//////////////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////////////
|
||||||
// Get tests
|
// Get tests
|
||||||
//////////////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////////////
|
||||||
/** For HBASE-528 */
|
|
||||||
public void testGetRowKeyAtOrBefore() {
|
|
||||||
// set up some test data
|
|
||||||
byte [] t10 = Bytes.toBytes("010");
|
|
||||||
byte [] t20 = Bytes.toBytes("020");
|
|
||||||
byte [] t30 = Bytes.toBytes("030");
|
|
||||||
byte [] t35 = Bytes.toBytes("035");
|
|
||||||
byte [] t40 = Bytes.toBytes("040");
|
|
||||||
|
|
||||||
memstore.add(getKV(t10, "t10 bytes".getBytes()));
|
|
||||||
memstore.add(getKV(t20, "t20 bytes".getBytes()));
|
|
||||||
memstore.add(getKV(t30, "t30 bytes".getBytes()));
|
|
||||||
memstore.add(getKV(t35, "t35 bytes".getBytes()));
|
|
||||||
// write a delete in there to see if things still work ok
|
|
||||||
memstore.add(getDeleteKV(t35));
|
|
||||||
memstore.add(getKV(t40, "t40 bytes".getBytes()));
|
|
||||||
|
|
||||||
NavigableSet<KeyValue> results = null;
|
|
||||||
|
|
||||||
// try finding "015"
|
|
||||||
results =
|
|
||||||
new TreeSet<KeyValue>(this.memstore.comparator.getComparatorIgnoringType());
|
|
||||||
KeyValue t15 = new KeyValue(Bytes.toBytes("015"),
|
|
||||||
System.currentTimeMillis());
|
|
||||||
memstore.getRowKeyAtOrBefore(t15, results);
|
|
||||||
KeyValue kv = results.last();
|
|
||||||
assertTrue(KeyValue.COMPARATOR.compareRows(kv, t10) == 0);
|
|
||||||
|
|
||||||
// try "020", we should get that row exactly
|
|
||||||
results =
|
|
||||||
new TreeSet<KeyValue>(this.memstore.comparator.getComparatorIgnoringType());
|
|
||||||
memstore.getRowKeyAtOrBefore(new KeyValue(t20, System.currentTimeMillis()),
|
|
||||||
results);
|
|
||||||
assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t20) == 0);
|
|
||||||
|
|
||||||
// try "030", we should get that row exactly
|
|
||||||
results =
|
|
||||||
new TreeSet<KeyValue>(this.memstore.comparator.getComparatorIgnoringType());
|
|
||||||
memstore.getRowKeyAtOrBefore(new KeyValue(t30, System.currentTimeMillis()),
|
|
||||||
results);
|
|
||||||
assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t30) == 0);
|
|
||||||
|
|
||||||
// try "038", should skip the deleted "035" and give "030"
|
|
||||||
results =
|
|
||||||
new TreeSet<KeyValue>(this.memstore.comparator.getComparatorIgnoringType());
|
|
||||||
byte [] t38 = Bytes.toBytes("038");
|
|
||||||
memstore.getRowKeyAtOrBefore(new KeyValue(t38, System.currentTimeMillis()),
|
|
||||||
results);
|
|
||||||
assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t30) == 0);
|
|
||||||
|
|
||||||
// try "050", should get stuff from "040"
|
|
||||||
results =
|
|
||||||
new TreeSet<KeyValue>(this.memstore.comparator.getComparatorIgnoringType());
|
|
||||||
byte [] t50 = Bytes.toBytes("050");
|
|
||||||
memstore.getRowKeyAtOrBefore(new KeyValue(t50, System.currentTimeMillis()),
|
|
||||||
results);
|
|
||||||
assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t40) == 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/** Test getNextRow from memstore
|
/** Test getNextRow from memstore
|
||||||
* @throws InterruptedException
|
* @throws InterruptedException
|
||||||
|
@ -606,8 +579,19 @@ public class TestMemStore extends TestCase {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
private int addRows(final MemStore hmc) {
|
private int addRows(final MemStore hmc) {
|
||||||
|
return addRows(hmc, HConstants.LATEST_TIMESTAMP);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds {@link #ROW_COUNT} rows and {@link #COLUMNS_COUNT}
|
||||||
|
* @param hmc Instance to add rows to.
|
||||||
|
* @return How many rows we added.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private int addRows(final MemStore hmc, final long ts) {
|
||||||
for (int i = 0; i < ROW_COUNT; i++) {
|
for (int i = 0; i < ROW_COUNT; i++) {
|
||||||
long timestamp = System.currentTimeMillis();
|
long timestamp = ts == HConstants.LATEST_TIMESTAMP?
|
||||||
|
System.currentTimeMillis(): ts;
|
||||||
for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
|
for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
|
||||||
byte [] row = Bytes.toBytes(i);
|
byte [] row = Bytes.toBytes(i);
|
||||||
byte [] qf = makeQualifier(i, ii);
|
byte [] qf = makeQualifier(i, ii);
|
||||||
|
|
|
@ -406,5 +406,4 @@ public class TestStore extends TestCase {
|
||||||
long storeTs = results.get(0).getTimestamp();
|
long storeTs = results.get(0).getTimestamp();
|
||||||
assertTrue(icvTs != storeTs);
|
assertTrue(icvTs != storeTs);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
Loading…
Reference in New Issue