Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13762 Use the same HFileContext with store files in mob files. (Jingcheng)

This commit is contained in:
anoopsjohn 2015-05-25 23:27:45 +05:30
parent 5428c9fdd3
commit 6388b3baf6
3 changed files with 18 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
View File

@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.Cipher;
import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.Encryption;
import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.master.TableLockManager;
@ -72,7 +71,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.security.EncryptionUtil; import org.apache.hadoop.hbase.security.EncryptionUtil;
import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Threads;
@ -493,7 +492,8 @@ public class MobUtils {
.withCompressTags(family.isCompressTags()).withChecksumType(HStore.getChecksumType(conf)) .withCompressTags(family.isCompressTags()).withChecksumType(HStore.getChecksumType(conf))
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize()) .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize())
.withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()) .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding())
.withEncryptionContext(cryptoContext).build(); .withEncryptionContext(cryptoContext).withCreateTime(EnvironmentEdgeManager.currentTime())
.build();
Path tempPath = new Path(basePath, UUID.randomUUID().toString().replaceAll("-", "")); Path tempPath = new Path(basePath, UUID.randomUUID().toString().replaceAll("-", ""));
StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs).withFilePath(tempPath) StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs).withFilePath(tempPath)
.withComparator(CellComparator.COMPARATOR).withBloomType(family.getBloomFilterType()) .withComparator(CellComparator.COMPARATOR).withBloomType(family.getBloomFilterType())
@ -574,10 +574,12 @@ public class MobUtils {
throws IOException { throws IOException {
HFileContext hFileContext = new HFileContextBuilder().withCompression(compression) HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
.withIncludesMvcc(true).withIncludesTags(true) .withIncludesMvcc(true).withIncludesTags(true)
.withChecksumType(ChecksumType.getDefaultChecksumType()) .withCompressTags(family.isCompressTags())
.withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).withBlockSize(family.getBlocksize()) .withChecksumType(HStore.getChecksumType(conf))
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(family.getBlocksize())
.withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()) .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding())
.withEncryptionContext(cryptoContext).build(); .withEncryptionContext(cryptoContext)
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs) StoreFile.Writer w = new StoreFile.WriterBuilder(conf, cacheConfig, fs)
.withFilePath(new Path(basePath, mobFileName.getFileName())) .withFilePath(new Path(basePath, mobFileName.getFileName()))

13
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
View File

@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.master.TableLockManager;
@ -60,7 +59,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController; import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.IdLock; import org.apache.hadoop.hbase.util.IdLock;
@ -250,11 +249,13 @@ public class HMobStore extends HStore {
final CacheConfig writerCacheConf = mobCacheConfig; final CacheConfig writerCacheConf = mobCacheConfig;
HFileContext hFileContext = new HFileContextBuilder().withCompression(compression) HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
.withIncludesMvcc(true).withIncludesTags(true) .withIncludesMvcc(true).withIncludesTags(true)
.withChecksumType(ChecksumType.getDefaultChecksumType()) .withCompressTags(family.isCompressTags())
.withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM) .withChecksumType(checksumType)
.withBlockSize(getFamily().getBlocksize()) .withBytesPerCheckSum(bytesPerChecksum)
.withBlockSize(blocksize)
.withHBaseCheckSum(true).withDataBlockEncoding(getFamily().getDataBlockEncoding()) .withHBaseCheckSum(true).withDataBlockEncoding(getFamily().getDataBlockEncoding())
.withEncryptionContext(cryptoContext).build(); .withEncryptionContext(cryptoContext)
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf, region.getFilesystem()) StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf, region.getFilesystem())
.withFilePath(new Path(basePath, mobFileName.getFileName())) .withFilePath(new Path(basePath, mobFileName.getFileName()))

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -172,12 +172,12 @@ public class HStore implements Store {
private final Set<ChangedReadersObserver> changedReaderObservers = private final Set<ChangedReadersObserver> changedReaderObservers =
Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>()); Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>());
private final int blocksize; protected final int blocksize;
private HFileDataBlockEncoder dataBlockEncoder; private HFileDataBlockEncoder dataBlockEncoder;
/** Checksum configuration */ /** Checksum configuration */
private ChecksumType checksumType; protected ChecksumType checksumType;
private int bytesPerChecksum; protected int bytesPerChecksum;
// Comparing KeyValues // Comparing KeyValues
private final CellComparator comparator; private final CellComparator comparator;