Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14557 MapReduce WALPlayer issue with NoTagsKeyValue.

This commit is contained in:
anoopsjohn 2015-10-30 21:18:09 +05:30
parent 23fa18184c
commit b0ad82191c
4 changed files with 26 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
View File

@ -437,14 +437,28 @@ public class KeyValueUtil {
/*************** misc **********************************/
/**
* @param cell
* @return <code>cell</code> if it is an instance of {@link KeyValue} else we will return a
* new {@link KeyValue} instance made from <code>cell</code>
* @return <code>cell</code> if it is an object of class {@link KeyValue} else we will return a
* new {@link KeyValue} instance made from <code>cell</code> Note: Even if the cell is an
* object of any of the subclass of {@link KeyValue}, we will create a new
* {@link KeyValue} object wrapping same buffer. This API is used only with MR based tools
* which expect the type to be exactly KeyValue. That is the reason for doing this way.
* @deprecated without any replacement.
*/
@Deprecated
public static KeyValue ensureKeyValue(final Cell cell) {
if (cell == null) return null;
return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
if (cell instanceof KeyValue) {
if (cell.getClass().getName().equals(KeyValue.class.getName())) {
return (KeyValue) cell;
}
// Cell is an Object of any of the sub classes of KeyValue. Make a new KeyValue wrapping the
// same byte[]
KeyValue kv = (KeyValue) cell;
KeyValue newKv = new KeyValue(kv.bytes, kv.offset, kv.length);
newKv.setSequenceId(kv.getSequenceId());
return newKv;
}
return copyToNewKeyValue(cell);
}
@Deprecated

4
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
View File

@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KeyOnlyKeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.ByteBufferedKeyOnlyKeyValue;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.io.HeapSize;
@ -350,9 +349,8 @@ public class HFileBlockIndex {
if (index == -1) {
// This has to be changed
// For now change this to key value
KeyValue kv = KeyValueUtil.ensureKeyValue(key);
throw new IOException("The key "
+ Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength())
+ CellUtil.getCellKeyAsString(key)
+ " is before the" + " first key of the non-root index block " + block);
}

8
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
View File

@ -172,12 +172,12 @@ public class MemStoreWrapper {
}
/**
* Adds a KeyValue into the memstore.
* @param kv The KeyValue to be added.
* Adds a Cell into the memstore.
* @param cell The Cell to be added.
* @throws IOException
*/
public void addToMemstore(KeyValue kv) throws IOException {
memstore.add(kv);
public void addToMemstore(Cell cell) throws IOException {
memstore.add(cell);
// flush the memstore if it's full.
flushMemStoreIfNecessary();
}

10
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java
View File

@ -331,13 +331,13 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
}
// write the hfile name
writer.append(mobFileName.getFileName(), MobConstants.EMPTY_STRING);
Set<KeyValue> kvs = new HashSet<KeyValue>();
Set<Cell> kvs = new HashSet<Cell>();
for (KeyValue kv : values) {
if (kv.getValueLength() > Bytes.SIZEOF_INT) {
mobFileStat.addValidSize(Bytes.toInt(kv.getValueArray(), kv.getValueOffset(),
Bytes.SIZEOF_INT));
}
kvs.add(kv.createKeyOnly(false));
kvs.add(kv);
}
// If the mob file is a invalid one or a small one, merge it into new/bigger ones.
if (mobFileStat.needClean() || (mergeSmall && mobFileStat.needMerge())) {
@ -351,11 +351,9 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY));
Cell cell;
while (null != (cell = scanner.next())) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
KeyValue keyOnly = kv.createKeyOnly(false);
if (kvs.contains(keyOnly)) {
if (kvs.contains(cell)) {
// write the KeyValue existing in HBase to the memstore.
memstore.addToMemstore(kv);
memstore.addToMemstore(cell);
memstoreUpdated = true;
}
}