HBASE-4520 Better handling of Bloom filter type discrepancy between HFile and CF config

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1178893 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2011-10-04 18:27:54 +00:00
parent 6232eae926
commit ea5ddc5ff7
2 changed files with 37 additions and 12 deletions

View File

@ -581,6 +581,8 @@ Release 0.92.0 - Unreleased
the Hlog (dhruba borthakur) the Hlog (dhruba borthakur)
HBASE-4526 special case for stopping master in hbase-daemon.sh is no longer HBASE-4526 special case for stopping master in hbase-daemon.sh is no longer
required (Roman Shaposhnik) required (Roman Shaposhnik)
HBASE-4520 Better handling of Bloom filter type discrepancy between HFile
and CF config (Mikhail Bautin)
TASKS TASKS

View File

@ -182,7 +182,12 @@ public class StoreFile {
// Used making file ids. // Used making file ids.
private final static Random rand = new Random(); private final static Random rand = new Random();
private final Configuration conf; private final Configuration conf;
private final BloomType bloomType;
/**
* Bloom filter type specified in column family configuration. Does not
* necessarily correspond to the Bloom filter type present in the HFile.
*/
private final BloomType cfBloomType;
// the last modification time stamp // the last modification time stamp
private long modificationTimeStamp = 0L; private long modificationTimeStamp = 0L;
@ -195,14 +200,18 @@ public class StoreFile {
* @param p The path of the file. * @param p The path of the file.
* @param blockcache <code>true</code> if the block cache is enabled. * @param blockcache <code>true</code> if the block cache is enabled.
* @param conf The current configuration. * @param conf The current configuration.
* @param bt The bloom type to use for this store file * @param cfBloomType The bloom type to use for this store file as specified
* by column family configuration. This may or may not be the same
* as the Bloom filter type actually present in the HFile, because
* column family configuration might change. If this is
* {@link BloomType#NONE}, the existing Bloom filter is ignored.
* @throws IOException When opening the reader fails. * @throws IOException When opening the reader fails.
*/ */
StoreFile(final FileSystem fs, StoreFile(final FileSystem fs,
final Path p, final Path p,
final boolean blockcache, final boolean blockcache,
final Configuration conf, final Configuration conf,
final BloomType bt, final BloomType cfBloomType,
final boolean inMemory) final boolean inMemory)
throws IOException { throws IOException {
this.conf = conf; this.conf = conf;
@ -214,14 +223,13 @@ public class StoreFile {
this.reference = Reference.read(fs, p); this.reference = Reference.read(fs, p);
this.referencePath = getReferredToFile(this.path); this.referencePath = getReferredToFile(this.path);
} }
// ignore if the column family config says "no bloom filter"
// even if there is one in the hfile.
if (BloomFilterFactory.isBloomEnabled(conf)) { if (BloomFilterFactory.isBloomEnabled(conf)) {
this.bloomType = bt; this.cfBloomType = cfBloomType;
} else { } else {
LOG.info("Ignoring bloom filter check for file " + path + ": " + LOG.info("Ignoring bloom filter check for file " + path + ": " +
"bloomType=" + bt + " (disabled in config)"); "cfBloomType=" + cfBloomType + " (disabled in config)");
this.bloomType = BloomType.NONE; this.cfBloomType = BloomType.NONE;
} }
// cache the modification time stamp of this store file // cache the modification time stamp of this store file
@ -501,8 +509,9 @@ public class StoreFile {
computeHDFSBlockDistribution(); computeHDFSBlockDistribution();
// Load up indices and fileinfo. // Load up indices and fileinfo. This also loads Bloom filter type.
metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo()); metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
// Read in our metadata. // Read in our metadata.
byte [] b = metadataMap.get(MAX_SEQ_ID_KEY); byte [] b = metadataMap.get(MAX_SEQ_ID_KEY);
if (b != null) { if (b != null) {
@ -534,8 +543,18 @@ public class StoreFile {
this.majorCompaction = new AtomicBoolean(false); this.majorCompaction = new AtomicBoolean(false);
} }
if (this.bloomType != BloomType.NONE) { BloomType hfileBloomType = reader.getBloomFilterType();
this.reader.loadBloomfilter(); if (cfBloomType != BloomType.NONE) {
reader.loadBloomfilter();
if (hfileBloomType != cfBloomType) {
LOG.info("HFile Bloom filter type for "
+ reader.getHFileReader().getName() + ": " + hfileBloomType
+ ", but " + cfBloomType + " specified in column family "
+ "configuration");
}
} else if (hfileBloomType != BloomType.NONE) {
LOG.info("Bloom filter turned off by CF config for "
+ reader.getHFileReader().getName());
} }
try { try {
@ -899,6 +918,9 @@ public class StoreFile {
break; break;
case NONE: case NONE:
newKey = false; newKey = false;
break;
default:
throw new IOException("Invalid Bloom filter type: " + bloomType);
} }
} }
if (newKey) { if (newKey) {
@ -930,7 +952,8 @@ public class StoreFile {
bloomKeyLen = bloomKey.length; bloomKeyLen = bloomKey.length;
break; break;
default: default:
throw new IOException("Invalid Bloom filter type: " + bloomType); throw new IOException("Invalid Bloom filter type: " + bloomType +
" (ROW or ROWCOL expected)");
} }
bloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen); bloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
if (lastBloomKey != null if (lastBloomKey != null