HBASE-4520 Better handling of Bloom filter type discrepancy between HFile and CF config
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1178893 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
6232eae926
commit
ea5ddc5ff7
|
@ -581,6 +581,8 @@ Release 0.92.0 - Unreleased
|
||||||
the Hlog (dhruba borthakur)
|
the Hlog (dhruba borthakur)
|
||||||
HBASE-4526 special case for stopping master in hbase-daemon.sh is no longer
|
HBASE-4526 special case for stopping master in hbase-daemon.sh is no longer
|
||||||
required (Roman Shaposhnik)
|
required (Roman Shaposhnik)
|
||||||
|
HBASE-4520 Better handling of Bloom filter type discrepancy between HFile
|
||||||
|
and CF config (Mikhail Bautin)
|
||||||
|
|
||||||
|
|
||||||
TASKS
|
TASKS
|
||||||
|
|
|
@ -182,7 +182,12 @@ public class StoreFile {
|
||||||
// Used making file ids.
|
// Used making file ids.
|
||||||
private final static Random rand = new Random();
|
private final static Random rand = new Random();
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
private final BloomType bloomType;
|
|
||||||
|
/**
|
||||||
|
* Bloom filter type specified in column family configuration. Does not
|
||||||
|
* necessarily correspond to the Bloom filter type present in the HFile.
|
||||||
|
*/
|
||||||
|
private final BloomType cfBloomType;
|
||||||
|
|
||||||
// the last modification time stamp
|
// the last modification time stamp
|
||||||
private long modificationTimeStamp = 0L;
|
private long modificationTimeStamp = 0L;
|
||||||
|
@ -195,14 +200,18 @@ public class StoreFile {
|
||||||
* @param p The path of the file.
|
* @param p The path of the file.
|
||||||
* @param blockcache <code>true</code> if the block cache is enabled.
|
* @param blockcache <code>true</code> if the block cache is enabled.
|
||||||
* @param conf The current configuration.
|
* @param conf The current configuration.
|
||||||
* @param bt The bloom type to use for this store file
|
* @param cfBloomType The bloom type to use for this store file as specified
|
||||||
|
* by column family configuration. This may or may not be the same
|
||||||
|
* as the Bloom filter type actually present in the HFile, because
|
||||||
|
* column family configuration might change. If this is
|
||||||
|
* {@link BloomType#NONE}, the existing Bloom filter is ignored.
|
||||||
* @throws IOException When opening the reader fails.
|
* @throws IOException When opening the reader fails.
|
||||||
*/
|
*/
|
||||||
StoreFile(final FileSystem fs,
|
StoreFile(final FileSystem fs,
|
||||||
final Path p,
|
final Path p,
|
||||||
final boolean blockcache,
|
final boolean blockcache,
|
||||||
final Configuration conf,
|
final Configuration conf,
|
||||||
final BloomType bt,
|
final BloomType cfBloomType,
|
||||||
final boolean inMemory)
|
final boolean inMemory)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
|
@ -214,14 +223,13 @@ public class StoreFile {
|
||||||
this.reference = Reference.read(fs, p);
|
this.reference = Reference.read(fs, p);
|
||||||
this.referencePath = getReferredToFile(this.path);
|
this.referencePath = getReferredToFile(this.path);
|
||||||
}
|
}
|
||||||
// ignore if the column family config says "no bloom filter"
|
|
||||||
// even if there is one in the hfile.
|
|
||||||
if (BloomFilterFactory.isBloomEnabled(conf)) {
|
if (BloomFilterFactory.isBloomEnabled(conf)) {
|
||||||
this.bloomType = bt;
|
this.cfBloomType = cfBloomType;
|
||||||
} else {
|
} else {
|
||||||
LOG.info("Ignoring bloom filter check for file " + path + ": " +
|
LOG.info("Ignoring bloom filter check for file " + path + ": " +
|
||||||
"bloomType=" + bt + " (disabled in config)");
|
"cfBloomType=" + cfBloomType + " (disabled in config)");
|
||||||
this.bloomType = BloomType.NONE;
|
this.cfBloomType = BloomType.NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
// cache the modification time stamp of this store file
|
// cache the modification time stamp of this store file
|
||||||
|
@ -501,8 +509,9 @@ public class StoreFile {
|
||||||
|
|
||||||
computeHDFSBlockDistribution();
|
computeHDFSBlockDistribution();
|
||||||
|
|
||||||
// Load up indices and fileinfo.
|
// Load up indices and fileinfo. This also loads Bloom filter type.
|
||||||
metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
|
metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
|
||||||
|
|
||||||
// Read in our metadata.
|
// Read in our metadata.
|
||||||
byte [] b = metadataMap.get(MAX_SEQ_ID_KEY);
|
byte [] b = metadataMap.get(MAX_SEQ_ID_KEY);
|
||||||
if (b != null) {
|
if (b != null) {
|
||||||
|
@ -534,8 +543,18 @@ public class StoreFile {
|
||||||
this.majorCompaction = new AtomicBoolean(false);
|
this.majorCompaction = new AtomicBoolean(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.bloomType != BloomType.NONE) {
|
BloomType hfileBloomType = reader.getBloomFilterType();
|
||||||
this.reader.loadBloomfilter();
|
if (cfBloomType != BloomType.NONE) {
|
||||||
|
reader.loadBloomfilter();
|
||||||
|
if (hfileBloomType != cfBloomType) {
|
||||||
|
LOG.info("HFile Bloom filter type for "
|
||||||
|
+ reader.getHFileReader().getName() + ": " + hfileBloomType
|
||||||
|
+ ", but " + cfBloomType + " specified in column family "
|
||||||
|
+ "configuration");
|
||||||
|
}
|
||||||
|
} else if (hfileBloomType != BloomType.NONE) {
|
||||||
|
LOG.info("Bloom filter turned off by CF config for "
|
||||||
|
+ reader.getHFileReader().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -899,6 +918,9 @@ public class StoreFile {
|
||||||
break;
|
break;
|
||||||
case NONE:
|
case NONE:
|
||||||
newKey = false;
|
newKey = false;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IOException("Invalid Bloom filter type: " + bloomType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (newKey) {
|
if (newKey) {
|
||||||
|
@ -930,7 +952,8 @@ public class StoreFile {
|
||||||
bloomKeyLen = bloomKey.length;
|
bloomKeyLen = bloomKey.length;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new IOException("Invalid Bloom filter type: " + bloomType);
|
throw new IOException("Invalid Bloom filter type: " + bloomType +
|
||||||
|
" (ROW or ROWCOL expected)");
|
||||||
}
|
}
|
||||||
bloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
|
bloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
|
||||||
if (lastBloomKey != null
|
if (lastBloomKey != null
|
||||||
|
|
Loading…
Reference in New Issue