Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-4589 CacheOnWrite broken in some cases because it can conflict with evictOnClose (jgray) 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1183541 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Gray 2011-10-14 22:41:45 +00:00
parent 8313263677
commit 4d776c28b9
11 changed files with 43 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -363,6 +363,8 @@ Release 0.92.0 - Unreleased
Hadoop 0.23 (todd)
HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of
regions
HBASE-4589 CacheOnWrite broken in some cases because it can conflict
with evictOnClose (jgray)
TESTS
HBASE-4450 test for number of blocks read: to serve as baseline for expected

3
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
View File

@ -322,6 +322,9 @@ public class HFile {
DataInput getBloomFilterMetadata() throws IOException;
Path getPath();
/** Close method with optional evictOnClose */
void close(boolean evictOnClose) throws IOException;
}
private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,

7
src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
View File

@ -348,7 +348,12 @@ public class HFileReaderV1 extends AbstractHFileReader {
@Override
public void close() throws IOException {
if (cacheConf.shouldEvictOnClose()) {
close(cacheConf.shouldEvictOnClose());
}
@Override
public void close(boolean evictOnClose) throws IOException {
if (evictOnClose) {
int numEvicted = 0;
for (int i = 0; i < dataBlockIndexReader.getRootBlockCount(); i++) {
if (cacheConf.getBlockCache().evictBlock(HFile.getBlockCacheKey(name,

6
src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
View File

@ -293,7 +293,11 @@ public class HFileReaderV2 extends AbstractHFileReader {
@Override
public void close() throws IOException {
if (cacheConf.shouldEvictOnClose()) {
close(cacheConf.shouldEvictOnClose());
}
public void close(boolean evictOnClose) throws IOException {
if (evictOnClose) {
int numEvicted = cacheConf.getBlockCache().evictBlocksByPrefix(name
+ HFile.CACHE_KEY_SEPARATOR);
LOG.debug("On close of file " + name + " evicted " + numEvicted

5
src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
View File

@ -26,6 +26,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@ -42,6 +44,7 @@ import org.apache.hadoop.io.Writable;
* Writes HFile format version 2.
*/
public class HFileWriterV2 extends AbstractHFileWriter {
static final Log LOG = LogFactory.getLog(HFileWriterV2.class);
/** Inline block writers for multi-level block index and compound Blooms. */
private List<InlineBlockWriter> inlineBlockWriters =
@ -174,6 +177,8 @@ public class HFileWriterV2 extends AbstractHFileWriter {
// Meta data block index writer
metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter();
LOG.debug("HFileWriter initialized with " + cacheConf);
}
/**

2
src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
View File

@ -409,7 +409,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
}
} finally {
if (halfWriter != null) halfWriter.close();
if (halfReader != null) halfReader.close();
if (halfReader != null) halfReader.close(cacheConf.shouldEvictOnClose());
}
}

4
src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
View File

@ -421,7 +421,7 @@ public class Store implements HeapSize {
storefiles = ImmutableList.of();
for (StoreFile f: result) {
f.closeReader();
f.closeReader(true);
}
LOG.debug("closed " + this.storeNameStr);
return result;
@ -1215,7 +1215,7 @@ public class Store implements HeapSize {
throw e;
} finally {
if (storeFile != null) {
storeFile.closeReader();
storeFile.closeReader(false);
}
}
}

12
src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
View File

@ -526,11 +526,13 @@ public class StoreFile {
}
/**
* @param b
* @throws IOException
*/
public synchronized void closeReader() throws IOException {
public synchronized void closeReader(boolean evictOnClose)
throws IOException {
if (this.reader != null) {
this.reader.close();
this.reader.close(evictOnClose);
this.reader = null;
}
}
@ -540,7 +542,7 @@ public class StoreFile {
* @throws IOException
*/
public void deleteReader() throws IOException {
closeReader();
closeReader(true);
this.fs.delete(getPath(), true);
}
@ -1011,8 +1013,8 @@ public class StoreFile {
return reader.getScanner(cacheBlocks, pread);
}
public void close() throws IOException {
reader.close();
public void close(boolean evictOnClose) throws IOException {
reader.close(evictOnClose);
}
public boolean shouldSeek(Scan scan, final SortedSet<byte[]> columns) {

2
src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
View File

@ -268,7 +268,7 @@ public class TestCompoundBloomFilter {
}
}
r.close();
r.close(true); // end of test so evictOnClose
}
private boolean isInBloom(StoreFileScanner scanner, byte[] row, BloomType bt,

2
src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
View File

@ -101,7 +101,7 @@ public class TestFSErrorsExposed {
LOG.info("Got expected exception", ioe);
assertTrue(ioe.getMessage().contains("Fault"));
}
reader.close();
reader.close(true); // end of test so evictOnClose
}
/**

20
src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
View File

@ -321,10 +321,10 @@ public class TestStoreFile extends HBaseTestCase {
assertTrue(count == 0);
} finally {
if (top != null) {
top.close();
top.close(true); // evict since we are about to delete the file
}
if (bottom != null) {
bottom.close();
bottom.close(true); // evict since we are about to delete the file
}
fs.delete(f.getPath(), true);
}
@ -370,7 +370,7 @@ public class TestStoreFile extends HBaseTestCase {
if (exists) falsePos++;
}
}
reader.close();
reader.close(true); // evict because we are about to delete the file
fs.delete(f, true);
assertEquals("False negatives: " + falseNeg, 0, falseNeg);
int maxFalsePos = (int) (2 * 2000 * err);
@ -466,7 +466,7 @@ public class TestStoreFile extends HBaseTestCase {
}
}
}
reader.close();
reader.close(true); // evict because we are about to delete the file
fs.delete(f, true);
System.out.println(bt[x].toString());
System.out.println(" False negatives: " + falseNeg);
@ -677,7 +677,7 @@ public class TestStoreFile extends HBaseTestCase {
assertEquals(startEvicted, cs.getEvictedCount());
startMiss += 3;
scanner.close();
reader.close();
reader.close(cacheConf.shouldEvictOnClose());
// Now write a StoreFile with three blocks, with cache on write on
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
@ -697,7 +697,7 @@ public class TestStoreFile extends HBaseTestCase {
assertEquals(startEvicted, cs.getEvictedCount());
startHit += 3;
scanner.close();
reader.close();
reader.close(cacheConf.shouldEvictOnClose());
// Let's read back the two files to ensure the blocks exactly match
hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
@ -730,9 +730,9 @@ public class TestStoreFile extends HBaseTestCase {
assertEquals(startEvicted, cs.getEvictedCount());
startHit += 6;
scannerOne.close();
readerOne.close();
readerOne.close(cacheConf.shouldEvictOnClose());
scannerTwo.close();
readerTwo.close();
readerTwo.close(cacheConf.shouldEvictOnClose());
// Let's close the first file with evict on close turned on
conf.setBoolean("hbase.rs.evictblocksonclose", true);
@ -740,7 +740,7 @@ public class TestStoreFile extends HBaseTestCase {
hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
StoreFile.BloomType.NONE);
reader = hsf.createReader();
reader.close();
reader.close(cacheConf.shouldEvictOnClose());
// We should have 3 new evictions
assertEquals(startHit, cs.getHitCount());
@ -754,7 +754,7 @@ public class TestStoreFile extends HBaseTestCase {
hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
StoreFile.BloomType.NONE);
reader = hsf.createReader();
reader.close();
reader.close(cacheConf.shouldEvictOnClose());
// We expect no changes
assertEquals(startHit, cs.getHitCount());