HBASE-4932 Block cache can be mistakenly instantiated by tools
Summary: disable block cache for HFile writers created for tools Test Plan: ran the following tests that call getWriterFactory test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Reviewers: dhruba, jgray, nspiegelberg, JIRA, mbautin Reviewed By: nspiegelberg CC: HBase Diffs Facebook Group, mbautin, khemani, nspiegelberg Differential Revision: 573 git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1210697 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2e4ae76616
commit
806ccf03dd
|
@ -259,9 +259,14 @@ public class HFile {
|
|||
|
||||
/**
|
||||
* Returns the factory to be used to create {@link HFile} writers.
|
||||
* Disables block cache access for all writers created through the
|
||||
* returned factory.
|
||||
*/
|
||||
public static final WriterFactory getWriterFactory(Configuration conf) {
|
||||
return HFile.getWriterFactory(conf, new CacheConfig(conf));
|
||||
public static final WriterFactory getWriterFactoryNoCache(Configuration
|
||||
conf) {
|
||||
Configuration tempConf = new Configuration(conf);
|
||||
tempConf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
|
||||
return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -173,7 +173,7 @@ public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable,
|
|||
String compression = compressionMap.get(family);
|
||||
compression = compression == null ? defaultCompression : compression;
|
||||
wl.writer =
|
||||
HFile.getWriterFactory(conf).createWriter(fs,
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(fs,
|
||||
StoreFile.getUniqueFile(fs, familydir), blocksize,
|
||||
compression, KeyValue.KEY_COMPARATOR);
|
||||
this.writers.put(family, wl);
|
||||
|
|
|
@ -106,7 +106,7 @@ public class CompressionTest {
|
|||
throws Exception {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
HFile.Writer writer =
|
||||
HFile.getWriterFactory(conf).createWriter(
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(
|
||||
fs, path, HFile.DEFAULT_BLOCKSIZE, codec, null);
|
||||
writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval"));
|
||||
writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval"));
|
||||
|
|
|
@ -190,7 +190,7 @@ public class HFilePerformanceEvaluation {
|
|||
@Override
|
||||
void setUp() throws Exception {
|
||||
writer =
|
||||
HFile.getWriterFactory(conf).createWriter(this.fs,
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(this.fs,
|
||||
this.mf, RFILE_BLOCKSIZE, (Compression.Algorithm) null, null);
|
||||
}
|
||||
|
||||
|
|
|
@ -162,7 +162,7 @@ public class TestHFilePerformance extends TestCase {
|
|||
if ("HFile".equals(fileType)){
|
||||
System.out.println("HFile write method: ");
|
||||
HFile.Writer writer =
|
||||
HFile.getWriterFactory(conf).createWriter(fout,
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(fout,
|
||||
minBlockSize, codecName, null);
|
||||
|
||||
// Writing value in one shot.
|
||||
|
|
|
@ -122,7 +122,7 @@ public class TestHFileSeek extends TestCase {
|
|||
FSDataOutputStream fout = createFSOutput(path, fs);
|
||||
try {
|
||||
Writer writer =
|
||||
HFile.getWriterFactory(conf).createWriter(fout,
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(fout,
|
||||
options.minBlockSize, options.compress, null);
|
||||
try {
|
||||
BytesWritable key = new BytesWritable();
|
||||
|
|
|
@ -47,7 +47,7 @@ public class TestSeekTo extends HBaseTestCase {
|
|||
FSDataOutputStream fout = this.fs.create(ncTFile);
|
||||
int blocksize = toKV("a").getLength() * 3;
|
||||
HFile.Writer writer =
|
||||
HFile.getWriterFactory(conf).createWriter(fout,
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(fout,
|
||||
blocksize, "none", null);
|
||||
// 4 bytes * 3 * 2 for each key/value +
|
||||
// 3 for keys, 15 for values = 42 (woot)
|
||||
|
|
|
@ -197,7 +197,7 @@ public class TestWALReplay {
|
|||
HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
|
||||
Path f = new Path(basedir, "hfile");
|
||||
HFile.Writer writer =
|
||||
HFile.getWriterFactory(conf).createWriter(this.fs, f);
|
||||
HFile.getWriterFactoryNoCache(conf).createWriter(this.fs, f);
|
||||
byte [] family = htd.getFamilies().iterator().next().getName();
|
||||
byte [] row = Bytes.toBytes(tableNameStr);
|
||||
writer.append(new KeyValue(row, family, family, row));
|
||||
|
|
Loading…
Reference in New Issue