Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18641 Include block content verification logic used in lruCache in bucketCache 

Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Biju Nair 2017-08-24 13:14:57 -04:00 committed by tedyu
parent 359fed7b4b
commit af21572427
3 changed files with 28 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.io.hfile;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentSkipListMap;
@ -26,6 +27,7 @@ import java.util.concurrent.ConcurrentSkipListSet;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram;
import org.apache.hadoop.hbase.util.Bytes;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.JsonMappingException;
@ -172,6 +174,15 @@ public class BlockCacheUtil {
return cbsbf;
}
public static int compareCacheBlock(Cacheable left, Cacheable right) {
ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
left.serialize(l);
ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
right.serialize(r);
return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
r.array(), r.arrayOffset(), r.limit());
}
/**
* Use one of these to keep a running account of cached blocks by file. Throw it away when done.
* This is different than metrics in that it is stats on current state of a cache.

13
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
View File

@ -379,7 +379,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
LruCachedBlock cb = map.get(cacheKey);
if (cb != null) {
// compare the contents, if they are not equal, we are in big trouble
if (compare(buf, cb.getBuffer()) != 0) {
if (BlockCacheUtil.compareCacheBlock(buf, cb.getBuffer()) != 0) {
throw new RuntimeException("Cached block contents differ, which should not have happened."
+ "cacheKey:" + cacheKey);
}
@ -439,15 +439,6 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
}
}
private int compare(Cacheable left, Cacheable right) {
ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
left.serialize(l);
ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
right.serialize(r);
return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
r.array(), r.arrayOffset(), r.limit());
}
/**
* Cache the block with the specified name and buffer.
* <p>
@ -589,7 +580,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
boolean wait = getCurrentSize() < acceptableSize();
boolean inMemory = block.getPriority() == BlockPriority.MEMORY;
((BucketCache) victimHandler).cacheBlockWithWait(block.getCacheKey(), block.getBuffer(),
inMemory, wait);
inMemory, true, wait);
} else {
victimHandler.cacheBlock(block.getCacheKey(), block.getBuffer());
}

17
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
View File

@ -411,7 +411,7 @@ public class BucketCache implements BlockCache, HeapSize {
@Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory,
final boolean cacheDataInL1) {
cacheBlockWithWait(cacheKey, cachedItem, inMemory, wait_when_cache);
cacheBlockWithWait(cacheKey, cachedItem, inMemory, cacheDataInL1, wait_when_cache);
}
/**
@ -422,13 +422,26 @@ public class BucketCache implements BlockCache, HeapSize {
* @param wait if true, blocking wait when queue is full
*/
public void cacheBlockWithWait(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory,
boolean wait) {
boolean cacheDataInL1, boolean wait) {
if (LOG.isTraceEnabled()) LOG.trace("Caching key=" + cacheKey + ", item=" + cachedItem);
if (!cacheEnabled) {
return;
}
if (backingMap.containsKey(cacheKey)) {
/*
* Compare already cached block only if lruBlockCache is not used to cache data blocks
*/
if (!cacheDataInL1) {
Cacheable existingBlock = getBlock(cacheKey, false, false, false);
if (BlockCacheUtil.compareCacheBlock(cachedItem, existingBlock) != 0) {
throw new RuntimeException("Cached block contents differ, which should not have happened."
+ "cacheKey:" + cacheKey);
}
}
String msg = "Caching an already cached block: " + cacheKey;
msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
LOG.warn(msg);
return;
}