diff --git a/CHANGES.txt b/CHANGES.txt index c14b0329110..dd7cb93ae2c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -581,6 +581,8 @@ Release 0.92.0 - Unreleased the Hlog (dhruba borthakur) HBASE-4526 special case for stopping master in hbase-daemon.sh is no longer required (Roman Shaposhnik) + HBASE-4520 Better handling of Bloom filter type discrepancy between HFile + and CF config (Mikhail Bautin) TASKS diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index fa629c7bf01..abb59318647 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -182,7 +182,12 @@ public class StoreFile { // Used making file ids. private final static Random rand = new Random(); private final Configuration conf; - private final BloomType bloomType; + + /** + * Bloom filter type specified in column family configuration. Does not + * necessarily correspond to the Bloom filter type present in the HFile. + */ + private final BloomType cfBloomType; // the last modification time stamp private long modificationTimeStamp = 0L; @@ -195,14 +200,18 @@ public class StoreFile { * @param p The path of the file. * @param blockcache true if the block cache is enabled. * @param conf The current configuration. - * @param bt The bloom type to use for this store file + * @param cfBloomType The bloom type to use for this store file as specified + * by column family configuration. This may or may not be the same + * as the Bloom filter type actually present in the HFile, because + * column family configuration might change. If this is + * {@link BloomType#NONE}, the existing Bloom filter is ignored. * @throws IOException When opening the reader fails. */ StoreFile(final FileSystem fs, final Path p, final boolean blockcache, final Configuration conf, - final BloomType bt, + final BloomType cfBloomType, final boolean inMemory) throws IOException { this.conf = conf; @@ -214,14 +223,13 @@ public class StoreFile { this.reference = Reference.read(fs, p); this.referencePath = getReferredToFile(this.path); } - // ignore if the column family config says "no bloom filter" - // even if there is one in the hfile. + if (BloomFilterFactory.isBloomEnabled(conf)) { - this.bloomType = bt; + this.cfBloomType = cfBloomType; } else { LOG.info("Ignoring bloom filter check for file " + path + ": " + - "bloomType=" + bt + " (disabled in config)"); - this.bloomType = BloomType.NONE; + "cfBloomType=" + cfBloomType + " (disabled in config)"); + this.cfBloomType = BloomType.NONE; } // cache the modification time stamp of this store file @@ -501,8 +509,9 @@ public class StoreFile { computeHDFSBlockDistribution(); - // Load up indices and fileinfo. + // Load up indices and fileinfo. This also loads Bloom filter type. metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo()); + // Read in our metadata. byte [] b = metadataMap.get(MAX_SEQ_ID_KEY); if (b != null) { @@ -534,8 +543,18 @@ public class StoreFile { this.majorCompaction = new AtomicBoolean(false); } - if (this.bloomType != BloomType.NONE) { - this.reader.loadBloomfilter(); + BloomType hfileBloomType = reader.getBloomFilterType(); + if (cfBloomType != BloomType.NONE) { + reader.loadBloomfilter(); + if (hfileBloomType != cfBloomType) { + LOG.info("HFile Bloom filter type for " + + reader.getHFileReader().getName() + ": " + hfileBloomType + + ", but " + cfBloomType + " specified in column family " + + "configuration"); + } + } else if (hfileBloomType != BloomType.NONE) { + LOG.info("Bloom filter turned off by CF config for " + + reader.getHFileReader().getName()); } try { @@ -899,6 +918,9 @@ public class StoreFile { break; case NONE: newKey = false; + break; + default: + throw new IOException("Invalid Bloom filter type: " + bloomType); } } if (newKey) { @@ -930,7 +952,8 @@ public class StoreFile { bloomKeyLen = bloomKey.length; break; default: - throw new IOException("Invalid Bloom filter type: " + bloomType); + throw new IOException("Invalid Bloom filter type: " + bloomType + + " (ROW or ROWCOL expected)"); } bloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen); if (lastBloomKey != null