From 8a09d95c4635689dfe48faa7aeae4ac611488d08 Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 19 May 2015 16:10:36 -0700 Subject: [PATCH] HBASE-13693 [HBase MOB] Mob files are not encrypting (Ashutosh Jindai) --- .../hadoop/hbase/regionserver/HMobStore.java | 4 +- .../hadoop/hbase/regionserver/HStore.java | 2 +- .../hbase/regionserver/TestHMobStore.java | 89 ++++++++++++++++++- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index 6016b39f27d..b4d221350ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -1,5 +1,4 @@ /** - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -251,7 +250,8 @@ public class HMobStore extends HStore { .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE) .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM) .withBlockSize(getFamily().getBlocksize()) - .withHBaseCheckSum(true).withDataBlockEncoding(getFamily().getDataBlockEncoding()).build(); + .withHBaseCheckSum(true).withDataBlockEncoding(getFamily().getDataBlockEncoding()) + .withEncryptionContext(cryptoContext).build(); StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf, region.getFilesystem()) .withFilePath(new Path(basePath, mobFileName.getFileName())) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index e0bb7cf180f..ad7ce987a75 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -193,7 +193,7 @@ public class HStore implements Store { private long blockingFileCount; private int compactionCheckMultiplier; - private Encryption.Context cryptoContext = Encryption.Context.NONE; + protected Encryption.Context cryptoContext = Encryption.Context.NONE; private volatile long flushedCellsCount = 0; private volatile long compactedCellsCount = 0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java index 39fd410a6b2..0be79bf144c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java @@ -19,7 +19,10 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.security.Key; +import java.security.SecureRandom; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.Iterator; @@ -27,31 +30,43 @@ import java.util.List; import java.util.NavigableSet; import java.util.concurrent.ConcurrentSkipListSet; +import javax.crypto.spec.SecretKeySpec; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.HarFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; +import org.apache.hadoop.hbase.io.crypto.aes.AES; +import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.regionserver.StoreFile.Reader; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.regionserver.compactions.NoLimitCompactionThroughputController; +import org.apache.hadoop.hbase.security.EncryptionUtil; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.junit.Assert; import org.junit.Before; @@ -469,4 +484,74 @@ public class TestHMobStore { storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class)); storeFlushCtx.commit(Mockito.mock(MonitoredTask.class)); } + + @Test + public void testMOBStoreEncryption() throws Exception { + final Configuration conf = TEST_UTIL.getConfiguration(); + + conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); + SecureRandom rng = new SecureRandom(); + byte[] keyBytes = new byte[AES.KEY_LENGTH]; + rng.nextBytes(keyBytes); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Key cfKey = new SecretKeySpec(keyBytes, algorithm); + + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setMobEnabled(true); + hcd.setMobThreshold(100); + hcd.setMaxVersions(4); + hcd.setEncryptionType(algorithm); + hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, + conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); + + init(name.getMethodName(), conf, hcd, false); + + this.store.add(new KeyValue(row, family, qf1, 1, value)); + this.store.add(new KeyValue(row, family, qf2, 1, value)); + this.store.add(new KeyValue(row, family, qf3, 1, value)); + flush(1); + + this.store.add(new KeyValue(row, family, qf4, 1, value)); + this.store.add(new KeyValue(row, family, qf5, 1, value)); + this.store.add(new KeyValue(row, family, qf6, 1, value)); + flush(2); + + Collection storefiles = this.store.getStorefiles(); + checkMobHFileEncrytption(storefiles); + + // Scan the values + Scan scan = new Scan(get); + InternalScanner scanner = (InternalScanner) store.getScanner(scan, + scan.getFamilyMap().get(store.getFamily().getName()), + 0); + + List results = new ArrayList(); + scanner.next(results); + Collections.sort(results, KeyValue.COMPARATOR); + scanner.close(); + Assert.assertEquals(expected.size(), results.size()); + for(int i=0; i storefiles) { + StoreFile storeFile = storefiles.iterator().next(); + HFile.Reader reader = storeFile.getReader().getHFileReader(); + byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); + Assert.assertTrue(null != encryptionKey); + Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() + .equals(HConstants.CIPHER_AES)); + } + }