HBASE-12085 mob status should print human readable numbers.(Jingcheng Du)
This commit is contained in:
parent
a0d9e18ccf
commit
629042f4ce
|
@ -123,6 +123,12 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
|
||||||
public static final String ENCRYPTION = "ENCRYPTION";
|
public static final String ENCRYPTION = "ENCRYPTION";
|
||||||
public static final String ENCRYPTION_KEY = "ENCRYPTION_KEY";
|
public static final String ENCRYPTION_KEY = "ENCRYPTION_KEY";
|
||||||
|
|
||||||
|
public static final String IS_MOB = "IS_MOB";
|
||||||
|
public static final byte[] IS_MOB_BYTES = Bytes.toBytes(IS_MOB);
|
||||||
|
public static final String MOB_THRESHOLD = "MOB_THRESHOLD";
|
||||||
|
public static final byte[] MOB_THRESHOLD_BYTES = Bytes.toBytes(MOB_THRESHOLD);
|
||||||
|
public static final long DEFAULT_MOB_THRESHOLD = 100 * 1024; // 100k
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Default compression type.
|
* Default compression type.
|
||||||
*/
|
*/
|
||||||
|
@ -260,6 +266,8 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
|
||||||
}
|
}
|
||||||
RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(ENCRYPTION)));
|
RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(ENCRYPTION)));
|
||||||
RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(ENCRYPTION_KEY)));
|
RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(ENCRYPTION_KEY)));
|
||||||
|
RESERVED_KEYWORDS.add(new ImmutableBytesWritable(IS_MOB_BYTES));
|
||||||
|
RESERVED_KEYWORDS.add(new ImmutableBytesWritable(MOB_THRESHOLD_BYTES));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final int UNINITIALIZED = -1;
|
private static final int UNINITIALIZED = -1;
|
||||||
|
@ -1084,6 +1092,10 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
|
||||||
/* TTL for now, we can add more as we neeed */
|
/* TTL for now, we can add more as we neeed */
|
||||||
if (key.equals(HColumnDescriptor.TTL)) {
|
if (key.equals(HColumnDescriptor.TTL)) {
|
||||||
unit = Unit.TIME_INTERVAL;
|
unit = Unit.TIME_INTERVAL;
|
||||||
|
} else if (key.equals(HColumnDescriptor.MOB_THRESHOLD)) {
|
||||||
|
unit = Unit.LONG;
|
||||||
|
} else if (key.equals(HColumnDescriptor.IS_MOB)) {
|
||||||
|
unit = Unit.BOOLEAN;
|
||||||
} else {
|
} else {
|
||||||
unit = Unit.NONE;
|
unit = Unit.NONE;
|
||||||
}
|
}
|
||||||
|
@ -1376,4 +1388,46 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
|
||||||
setValue(Bytes.toBytes(ENCRYPTION_KEY), keyBytes);
|
setValue(Bytes.toBytes(ENCRYPTION_KEY), keyBytes);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the mob threshold of the family.
|
||||||
|
* If the size of a cell value is larger than this threshold, it's regarded as a mob.
|
||||||
|
* The default threshold is 1024*100(100K)B.
|
||||||
|
* @return The mob threshold.
|
||||||
|
*/
|
||||||
|
public long getMobThreshold() {
|
||||||
|
byte[] threshold = getValue(MOB_THRESHOLD_BYTES);
|
||||||
|
return threshold != null && threshold.length == Bytes.SIZEOF_LONG ? Bytes.toLong(threshold)
|
||||||
|
: DEFAULT_MOB_THRESHOLD;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the mob threshold of the family.
|
||||||
|
* @param threshold The mob threshold.
|
||||||
|
* @return this (for chained invocation)
|
||||||
|
*/
|
||||||
|
public HColumnDescriptor setMobThreshold(long threshold) {
|
||||||
|
setValue(MOB_THRESHOLD_BYTES, Bytes.toBytes(threshold));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets whether the mob is enabled for the family.
|
||||||
|
* @return True if the mob is enabled for the family.
|
||||||
|
*/
|
||||||
|
public boolean isMobEnabled() {
|
||||||
|
byte[] isMobEnabled = getValue(IS_MOB_BYTES);
|
||||||
|
return isMobEnabled != null && isMobEnabled.length == Bytes.SIZEOF_BOOLEAN
|
||||||
|
&& Bytes.toBoolean(isMobEnabled);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Enables the mob for the family.
|
||||||
|
* @param isMobEnabled Whether to enable the mob for the family.
|
||||||
|
* @return this (for chained invocation)
|
||||||
|
*/
|
||||||
|
public HColumnDescriptor setMobEnabled(boolean isMobEnabled) {
|
||||||
|
setValue(IS_MOB_BYTES, Bytes.toBytes(isMobEnabled));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,6 +29,8 @@ public class PrettyPrinter {
|
||||||
|
|
||||||
public enum Unit {
|
public enum Unit {
|
||||||
TIME_INTERVAL,
|
TIME_INTERVAL,
|
||||||
|
LONG,
|
||||||
|
BOOLEAN,
|
||||||
NONE
|
NONE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -38,6 +40,14 @@ public class PrettyPrinter {
|
||||||
case TIME_INTERVAL:
|
case TIME_INTERVAL:
|
||||||
human.append(humanReadableTTL(Long.valueOf(value)));
|
human.append(humanReadableTTL(Long.valueOf(value)));
|
||||||
break;
|
break;
|
||||||
|
case LONG:
|
||||||
|
byte[] longBytes = Bytes.toBytesBinary(value);
|
||||||
|
human.append(String.valueOf(Bytes.toLong(longBytes)));
|
||||||
|
break;
|
||||||
|
case BOOLEAN:
|
||||||
|
byte[] booleanBytes = Bytes.toBytesBinary(value);
|
||||||
|
human.append(String.valueOf(Bytes.toBoolean(booleanBytes)));
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
human.append(value);
|
human.append(value);
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,6 @@ import java.util.List;
|
||||||
import org.apache.commons.cli.CommandLine;
|
import org.apache.commons.cli.CommandLine;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
|
import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
|
||||||
import org.apache.hadoop.hbase.util.LoadTestTool;
|
import org.apache.hadoop.hbase.util.LoadTestTool;
|
||||||
|
@ -118,8 +117,8 @@ public class IntegrationTestIngestWithMOB extends IntegrationTestIngest {
|
||||||
admin.disableTable(tableName);
|
admin.disableTable(tableName);
|
||||||
for (HColumnDescriptor columnDescriptor : tableDesc.getFamilies()) {
|
for (HColumnDescriptor columnDescriptor : tableDesc.getFamilies()) {
|
||||||
if(Arrays.equals(columnDescriptor.getName(), mobColumnFamily)) {
|
if(Arrays.equals(columnDescriptor.getName(), mobColumnFamily)) {
|
||||||
columnDescriptor.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
columnDescriptor.setMobEnabled(true);
|
||||||
columnDescriptor.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes((long) threshold));
|
columnDescriptor.setMobThreshold((long) threshold);
|
||||||
admin.modifyColumn(tableName, columnDescriptor);
|
admin.modifyColumn(tableName, columnDescriptor);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,8 +29,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.TableDescriptors;
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
import org.apache.hadoop.hbase.mob.ExpiredMobFileCleaner;
|
import org.apache.hadoop.hbase.mob.ExpiredMobFileCleaner;
|
||||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
import org.apache.hadoop.hbase.mob.MobConstants;
|
||||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
|
||||||
import org.apache.hadoop.hbase.util.Threads;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class ExpiredMobFileCleanerChore for running cleaner regularly to remove the expired
|
* The Class ExpiredMobFileCleanerChore for running cleaner regularly to remove the expired
|
||||||
|
@ -57,7 +55,7 @@ public class ExpiredMobFileCleanerChore extends Chore {
|
||||||
Map<String, HTableDescriptor> map = htds.getAll();
|
Map<String, HTableDescriptor> map = htds.getAll();
|
||||||
for (HTableDescriptor htd : map.values()) {
|
for (HTableDescriptor htd : map.values()) {
|
||||||
for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
|
for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
|
||||||
if (MobUtils.isMobFamily(hcd) && hcd.getMinVersions() == 0) {
|
if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) {
|
||||||
cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd);
|
cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -160,7 +160,7 @@ public class DeleteTableHandler extends TableEventHandler {
|
||||||
HColumnDescriptor[] hcds = hTableDescriptor.getColumnFamilies();
|
HColumnDescriptor[] hcds = hTableDescriptor.getColumnFamilies();
|
||||||
boolean hasMob = false;
|
boolean hasMob = false;
|
||||||
for (HColumnDescriptor hcd : hcds) {
|
for (HColumnDescriptor hcd : hcds) {
|
||||||
if (MobUtils.isMobFamily(hcd)) {
|
if (hcd.isMobEnabled()) {
|
||||||
hasMob = true;
|
hasMob = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,7 @@ public class DefaultMobCompactor extends DefaultCompactor {
|
||||||
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
|
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
|
||||||
}
|
}
|
||||||
mobStore = (HMobStore) store;
|
mobStore = (HMobStore) store;
|
||||||
mobSizeThreshold = MobUtils.getMobThreshold(store.getFamily());
|
mobSizeThreshold = store.getFamily().getMobThreshold();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -69,7 +69,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
|
||||||
|
|
||||||
public DefaultMobStoreFlusher(Configuration conf, Store store) throws IOException {
|
public DefaultMobStoreFlusher(Configuration conf, Store store) throws IOException {
|
||||||
super(conf, store);
|
super(conf, store);
|
||||||
mobCellValueSizeThreshold = MobUtils.getMobThreshold(store.getFamily());
|
mobCellValueSizeThreshold = store.getFamily().getMobThreshold();
|
||||||
this.targetPath = MobUtils.getMobFamilyPath(conf, store.getTableName(),
|
this.targetPath = MobUtils.getMobFamilyPath(conf, store.getTableName(),
|
||||||
store.getColumnFamilyName());
|
store.getColumnFamilyName());
|
||||||
if (!this.store.getFileSystem().exists(targetPath)) {
|
if (!this.store.getFileSystem().exists(targetPath)) {
|
||||||
|
|
|
@ -99,7 +99,7 @@ public class ExpiredMobFileCleaner extends Configured implements Tool {
|
||||||
try {
|
try {
|
||||||
HTableDescriptor htd = admin.getTableDescriptor(tn);
|
HTableDescriptor htd = admin.getTableDescriptor(tn);
|
||||||
HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
|
HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
|
||||||
if (family == null || !MobUtils.isMobFamily(family)) {
|
if (family == null || !family.isMobEnabled()) {
|
||||||
throw new IOException("Column family " + familyName + " is not a MOB column family");
|
throw new IOException("Column family " + familyName + " is not a MOB column family");
|
||||||
}
|
}
|
||||||
if (family.getMinVersions() > 0) {
|
if (family.getMinVersions() > 0) {
|
||||||
|
|
|
@ -32,10 +32,6 @@ import org.apache.hadoop.hbase.util.Bytes;
|
||||||
@InterfaceStability.Evolving
|
@InterfaceStability.Evolving
|
||||||
public class MobConstants {
|
public class MobConstants {
|
||||||
|
|
||||||
public static final byte[] IS_MOB = Bytes.toBytes("IS_MOB");
|
|
||||||
public static final byte[] MOB_THRESHOLD = Bytes.toBytes("MOB_THRESHOLD");
|
|
||||||
public static final long DEFAULT_MOB_THRESHOLD = 100 * 1024; // 100k
|
|
||||||
|
|
||||||
public static final String MOB_SCAN_RAW = "hbase.mob.scan.raw";
|
public static final String MOB_SCAN_RAW = "hbase.mob.scan.raw";
|
||||||
public static final String MOB_CACHE_BLOCKS = "hbase.mob.cache.blocks";
|
public static final String MOB_CACHE_BLOCKS = "hbase.mob.cache.blocks";
|
||||||
public static final String MOB_SCAN_REF_ONLY = "hbase.mob.scan.ref.only";
|
public static final String MOB_SCAN_REF_ONLY = "hbase.mob.scan.ref.only";
|
||||||
|
|
|
@ -73,29 +73,6 @@ public class MobUtils {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Indicates whether the column family is a mob one.
|
|
||||||
* @param hcd The descriptor of a column family.
|
|
||||||
* @return True if this column family is a mob one, false if it's not.
|
|
||||||
*/
|
|
||||||
public static boolean isMobFamily(HColumnDescriptor hcd) {
|
|
||||||
byte[] isMob = hcd.getValue(MobConstants.IS_MOB);
|
|
||||||
return isMob != null && isMob.length == 1 && Bytes.toBoolean(isMob);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Gets the mob threshold.
|
|
||||||
* If the size of a cell value is larger than this threshold, it's regarded as a mob.
|
|
||||||
* The default threshold is 1024*100(100K)B.
|
|
||||||
* @param hcd The descriptor of a column family.
|
|
||||||
* @return The threshold.
|
|
||||||
*/
|
|
||||||
public static long getMobThreshold(HColumnDescriptor hcd) {
|
|
||||||
byte[] threshold = hcd.getValue(MobConstants.MOB_THRESHOLD);
|
|
||||||
return threshold != null && threshold.length == Bytes.SIZEOF_LONG ? Bytes.toLong(threshold)
|
|
||||||
: MobConstants.DEFAULT_MOB_THRESHOLD;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Formats a date to a string.
|
* Formats a date to a string.
|
||||||
* @param date The date.
|
* @param date The date.
|
||||||
|
|
|
@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
@ -68,7 +67,7 @@ public class Sweeper extends Configured implements Tool {
|
||||||
TableName tn = TableName.valueOf(tableName);
|
TableName tn = TableName.valueOf(tableName);
|
||||||
HTableDescriptor htd = admin.getTableDescriptor(tn);
|
HTableDescriptor htd = admin.getTableDescriptor(tn);
|
||||||
HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
|
HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
|
||||||
if (family == null || !MobUtils.isMobFamily(family)) {
|
if (family == null || !family.isMobEnabled()) {
|
||||||
throw new IOException("Column family " + familyName + " is not a MOB column family");
|
throw new IOException("Column family " + familyName + " is not a MOB column family");
|
||||||
}
|
}
|
||||||
SweepJob job = new SweepJob(conf, fs);
|
SweepJob job = new SweepJob(conf, fs);
|
||||||
|
|
|
@ -113,7 +113,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||||
import org.apache.hadoop.hbase.ipc.CallerDisconnectedException;
|
import org.apache.hadoop.hbase.ipc.CallerDisconnectedException;
|
||||||
import org.apache.hadoop.hbase.ipc.RpcCallContext;
|
import org.apache.hadoop.hbase.ipc.RpcCallContext;
|
||||||
import org.apache.hadoop.hbase.ipc.RpcServer;
|
import org.apache.hadoop.hbase.ipc.RpcServer;
|
||||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
|
||||||
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
|
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
|
||||||
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
|
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
|
||||||
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
|
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
|
||||||
|
@ -3554,7 +3553,7 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
}
|
}
|
||||||
|
|
||||||
protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
|
protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
|
||||||
if (MobUtils.isMobFamily(family)) {
|
if (family.isMobEnabled()) {
|
||||||
if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
|
if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
|
||||||
throw new IOException("A minimum HFile version of "
|
throw new IOException("A minimum HFile version of "
|
||||||
+ HFile.MIN_FORMAT_VERSION_WITH_TAGS
|
+ HFile.MIN_FORMAT_VERSION_WITH_TAGS
|
||||||
|
|
|
@ -25,6 +25,8 @@ import org.apache.hadoop.hbase.io.compress.Compression;
|
||||||
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
|
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
|
||||||
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
|
||||||
import org.apache.hadoop.hbase.regionserver.BloomType;
|
import org.apache.hadoop.hbase.regionserver.BloomType;
|
||||||
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.hadoop.hbase.util.PrettyPrinter;
|
||||||
import org.junit.experimental.categories.Category;
|
import org.junit.experimental.categories.Category;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -52,6 +54,8 @@ public class TestHColumnDescriptor {
|
||||||
hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
|
hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
|
||||||
hcd.setBloomFilterType(BloomType.ROW);
|
hcd.setBloomFilterType(BloomType.ROW);
|
||||||
hcd.setCompressionType(Algorithm.SNAPPY);
|
hcd.setCompressionType(Algorithm.SNAPPY);
|
||||||
|
hcd.setMobEnabled(true);
|
||||||
|
hcd.setMobThreshold(1000L);
|
||||||
|
|
||||||
|
|
||||||
byte [] bytes = hcd.toByteArray();
|
byte [] bytes = hcd.toByteArray();
|
||||||
|
@ -68,6 +72,8 @@ public class TestHColumnDescriptor {
|
||||||
assertTrue(deserializedHcd.getCompressionType().equals(Compression.Algorithm.SNAPPY));
|
assertTrue(deserializedHcd.getCompressionType().equals(Compression.Algorithm.SNAPPY));
|
||||||
assertTrue(deserializedHcd.getDataBlockEncoding().equals(DataBlockEncoding.FAST_DIFF));
|
assertTrue(deserializedHcd.getDataBlockEncoding().equals(DataBlockEncoding.FAST_DIFF));
|
||||||
assertTrue(deserializedHcd.getBloomFilterType().equals(BloomType.ROW));
|
assertTrue(deserializedHcd.getBloomFilterType().equals(BloomType.ROW));
|
||||||
|
assertEquals(hcd.isMobEnabled(), deserializedHcd.isMobEnabled());
|
||||||
|
assertEquals(hcd.getMobThreshold(), deserializedHcd.getMobThreshold());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -94,4 +100,16 @@ public class TestHColumnDescriptor {
|
||||||
desc.removeConfiguration(key);
|
desc.removeConfiguration(key);
|
||||||
assertEquals(null, desc.getConfigurationValue(key));
|
assertEquals(null, desc.getConfigurationValue(key));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMobValuesInHColumnDescriptorShouldReadable() {
|
||||||
|
boolean isMob = true;
|
||||||
|
long threshold = 1000;
|
||||||
|
String isMobString = PrettyPrinter.format(Bytes.toStringBinary(Bytes.toBytes(isMob)),
|
||||||
|
HColumnDescriptor.getUnit(HColumnDescriptor.IS_MOB));
|
||||||
|
String thresholdString = PrettyPrinter.format(Bytes.toStringBinary(Bytes.toBytes(threshold)),
|
||||||
|
HColumnDescriptor.getUnit(HColumnDescriptor.MOB_THRESHOLD));
|
||||||
|
assertEquals(String.valueOf(isMob), isMobString);
|
||||||
|
assertEquals(String.valueOf(threshold), thresholdString);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -137,8 +137,8 @@ public class TestDefaultMobStoreFlusher {
|
||||||
try {
|
try {
|
||||||
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TN));
|
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TN));
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
|
hcd.setMobThreshold(3L);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
|
|
||||||
|
|
|
@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.client.Admin;
|
||||||
import org.apache.hadoop.hbase.client.HTable;
|
import org.apache.hadoop.hbase.client.HTable;
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.mob.ExpiredMobFileCleaner;
|
import org.apache.hadoop.hbase.mob.ExpiredMobFileCleaner;
|
||||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
|
||||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
import org.apache.hadoop.hbase.mob.MobUtils;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
@ -87,8 +86,8 @@ public class TestExpiredMobFileCleaner {
|
||||||
private void init() throws Exception {
|
private void init() throws Exception {
|
||||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
|
hcd.setMobThreshold(3L);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
|
|
||||||
|
@ -100,8 +99,8 @@ public class TestExpiredMobFileCleaner {
|
||||||
|
|
||||||
private void modifyColumnExpiryDays(int expireDays) throws Exception {
|
private void modifyColumnExpiryDays(int expireDays) throws Exception {
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
|
hcd.setMobThreshold(3L);
|
||||||
// change ttl as expire days to make some row expired
|
// change ttl as expire days to make some row expired
|
||||||
int timeToLive = expireDays * secondsOfDay();
|
int timeToLive = expireDays * secondsOfDay();
|
||||||
hcd.setTimeToLive(timeToLive);
|
hcd.setTimeToLive(timeToLive);
|
||||||
|
|
|
@ -75,8 +75,8 @@ public class TestMobDataBlockEncoding {
|
||||||
throws Exception {
|
throws Exception {
|
||||||
desc = new HTableDescriptor(TableName.valueOf(TN));
|
desc = new HTableDescriptor(TableName.valueOf(TN));
|
||||||
hcd = new HColumnDescriptor(family);
|
hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(threshold));
|
hcd.setMobThreshold(threshold);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
hcd.setDataBlockEncoding(encoding);
|
hcd.setDataBlockEncoding(encoding);
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
|
|
|
@ -79,14 +79,14 @@ public class TestMobFileCache extends TestCase {
|
||||||
conf = UTIL.getConfiguration();
|
conf = UTIL.getConfiguration();
|
||||||
HTableDescriptor htd = UTIL.createTableDescriptor("testMobFileCache");
|
HTableDescriptor htd = UTIL.createTableDescriptor("testMobFileCache");
|
||||||
HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILY1);
|
HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILY1);
|
||||||
hcd1.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd1.setMobEnabled(true);
|
||||||
hcd1.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
|
hcd1.setMobThreshold(0);
|
||||||
HColumnDescriptor hcd2 = new HColumnDescriptor(FAMILY2);
|
HColumnDescriptor hcd2 = new HColumnDescriptor(FAMILY2);
|
||||||
hcd2.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd2.setMobEnabled(true);
|
||||||
hcd2.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
|
hcd2.setMobThreshold(0);
|
||||||
HColumnDescriptor hcd3 = new HColumnDescriptor(FAMILY3);
|
HColumnDescriptor hcd3 = new HColumnDescriptor(FAMILY3);
|
||||||
hcd3.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd3.setMobEnabled(true);
|
||||||
hcd3.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
|
hcd3.setMobThreshold(0);
|
||||||
htd.addFamily(hcd1);
|
htd.addFamily(hcd1);
|
||||||
htd.addFamily(hcd2);
|
htd.addFamily(hcd2);
|
||||||
htd.addFamily(hcd3);
|
htd.addFamily(hcd3);
|
||||||
|
@ -115,7 +115,7 @@ public class TestMobFileCache extends TestCase {
|
||||||
private Path createMobStoreFile(Configuration conf, String family) throws IOException {
|
private Path createMobStoreFile(Configuration conf, String family) throws IOException {
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
mobCacheConf = new MobCacheConfig(conf, hcd);
|
mobCacheConf = new MobCacheConfig(conf, hcd);
|
||||||
return createMobStoreFile(conf, hcd);
|
return createMobStoreFile(conf, hcd);
|
||||||
}
|
}
|
||||||
|
|
|
@ -92,8 +92,8 @@ public class TestMobSweepReducer {
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
|
hcd.setMobThreshold(3L);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
|
|
||||||
|
|
|
@ -86,8 +86,8 @@ public class TestMobSweeper {
|
||||||
tableName = "testSweeper" + tid;
|
tableName = "testSweeper" + tid;
|
||||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
|
hcd.setMobThreshold(3L);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
|
|
||||||
|
|
|
@ -84,8 +84,8 @@ public class TestDeleteMobTable {
|
||||||
TableName tn = TableName.valueOf(tableName);
|
TableName tn = TableName.valueOf(tableName);
|
||||||
HTableDescriptor htd = new HTableDescriptor(tn);
|
HTableDescriptor htd = new HTableDescriptor(tn);
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
|
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
|
hcd.setMobThreshold(0);
|
||||||
htd.addFamily(hcd);
|
htd.addFamily(hcd);
|
||||||
HBaseAdmin admin = null;
|
HBaseAdmin admin = null;
|
||||||
HTable table = null;
|
HTable table = null;
|
||||||
|
|
|
@ -115,8 +115,8 @@ public class TestHMobStore {
|
||||||
private void init(String methodName, Configuration conf, boolean testStore)
|
private void init(String methodName, Configuration conf, boolean testStore)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
hcd = new HColumnDescriptor(family);
|
hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
|
hcd.setMobThreshold(3L);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
init(methodName, conf, hcd, testStore);
|
init(methodName, conf, hcd, testStore);
|
||||||
}
|
}
|
||||||
|
@ -365,8 +365,8 @@ public class TestHMobStore {
|
||||||
|
|
||||||
HColumnDescriptor hcd;
|
HColumnDescriptor hcd;
|
||||||
hcd = new HColumnDescriptor(family);
|
hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(100L));
|
hcd.setMobThreshold(100);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
init(name.getMethodName(), conf, hcd, false);
|
init(name.getMethodName(), conf, hcd, false);
|
||||||
|
|
||||||
|
@ -406,7 +406,7 @@ public class TestHMobStore {
|
||||||
//this is not mob reference cell.
|
//this is not mob reference cell.
|
||||||
Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
|
Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
|
||||||
Assert.assertEquals(expected.get(i), results.get(i));
|
Assert.assertEquals(expected.get(i), results.get(i));
|
||||||
Assert.assertEquals(100, MobUtils.getMobThreshold(store.getFamily()));
|
Assert.assertEquals(100, store.getFamily().getMobThreshold());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -109,8 +109,8 @@ public class TestMobCompaction {
|
||||||
compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
|
compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
|
||||||
htd = UTIL.createTableDescriptor(name.getMethodName());
|
htd = UTIL.createTableDescriptor(name.getMethodName());
|
||||||
hcd = new HColumnDescriptor(COLUMN_FAMILY);
|
hcd = new HColumnDescriptor(COLUMN_FAMILY);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(mobThreshold));
|
hcd.setMobThreshold(mobThreshold);
|
||||||
hcd.setMaxVersions(1);
|
hcd.setMaxVersions(1);
|
||||||
htd.addFamily(hcd);
|
htd.addFamily(hcd);
|
||||||
|
|
||||||
|
@ -170,8 +170,7 @@ public class TestMobCompaction {
|
||||||
assertEquals("Before compaction: rows", compactionThreshold, countRows());
|
assertEquals("Before compaction: rows", compactionThreshold, countRows());
|
||||||
assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
|
assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
|
||||||
// Change the threshold larger than the data size
|
// Change the threshold larger than the data size
|
||||||
region.getTableDesc().getFamily(COLUMN_FAMILY).setValue(
|
region.getTableDesc().getFamily(COLUMN_FAMILY).setMobThreshold(500);
|
||||||
MobConstants.MOB_THRESHOLD, Bytes.toBytes(500L));
|
|
||||||
region.initialize();
|
region.initialize();
|
||||||
region.compactStores();
|
region.compactStores();
|
||||||
|
|
||||||
|
|
|
@ -82,8 +82,8 @@ public class TestMobStoreScanner {
|
||||||
public void setUp(long threshold, String TN) throws Exception {
|
public void setUp(long threshold, String TN) throws Exception {
|
||||||
desc = new HTableDescriptor(TableName.valueOf(TN));
|
desc = new HTableDescriptor(TableName.valueOf(TN));
|
||||||
hcd = new HColumnDescriptor(family);
|
hcd = new HColumnDescriptor(family);
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(threshold));
|
hcd.setMobThreshold(threshold);
|
||||||
hcd.setMaxVersions(4);
|
hcd.setMaxVersions(4);
|
||||||
desc.addFamily(hcd);
|
desc.addFamily(hcd);
|
||||||
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
|
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
|
||||||
|
|
|
@ -64,19 +64,18 @@
|
||||||
<example>
|
<example>
|
||||||
<title>Configure a Column for MOB Using HBase Shell</title>
|
<title>Configure a Column for MOB Using HBase Shell</title>
|
||||||
<screen>
|
<screen>
|
||||||
hbase> create 't1', 'f1', {IS_MOB => true, MOB_THRESHOLD => 102400}
|
hbase> create 't1', {NAME => 'f1', IS_MOB => true, MOB_THRESHOLD => 102400}
|
||||||
hbase> alter ‘t1′, {NAME => ‘f1', IS_MOB => true, MOB_THRESHOLD => 102400}
|
hbase> alter 't1', {NAME => 'f1', IS_MOB => true, MOB_THRESHOLD => 102400}
|
||||||
</screen>
|
</screen>
|
||||||
</example>
|
</example>
|
||||||
<example>
|
<example>
|
||||||
<title>Configure a Column for MOB Using the API</title>
|
<title>Configure a Column for MOB Using the API</title>
|
||||||
<programlisting language="java">
|
<programlisting language="java">
|
||||||
...
|
...
|
||||||
HColumnDescriptor hcd = new HColumnDescriptor(“f”);
|
HColumnDescriptor hcd = new HColumnDescriptor("f");
|
||||||
hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
hcd.setMobEnabled(true);
|
||||||
...
|
...
|
||||||
HColumnDescriptor hcd;
|
hcd.setMobThreshold(102400L);
|
||||||
hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(102400L);
|
|
||||||
...
|
...
|
||||||
</programlisting>
|
</programlisting>
|
||||||
</example>
|
</example>
|
||||||
|
@ -93,15 +92,15 @@ hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(102400L);
|
||||||
<itemizedlist>
|
<itemizedlist>
|
||||||
<listitem>
|
<listitem>
|
||||||
<para><literal>threshold</literal> is the threshold at which cells are considered to
|
<para><literal>threshold</literal> is the threshold at which cells are considered to
|
||||||
be MOBs. The default is 100 kb.</para>
|
be MOBs. The default is 1 kB.</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
<listitem>
|
<listitem>
|
||||||
<para><literal>minMobDataSize</literal> is the minimum value for the size of MOB
|
<para><literal>minMobDataSize</literal> is the minimum value for the size of MOB
|
||||||
data. The default is 80 kb.</para>
|
data. The default is 512 B.</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
<listitem>
|
<listitem>
|
||||||
<para><literal>maxMobDataSize</literal> is the maximum value for the size of MOB
|
<para><literal>maxMobDataSize</literal> is the maximum value for the size of MOB
|
||||||
data. The default is 5 MB.</para>
|
data. The default is 5 kB.</para>
|
||||||
</listitem>
|
</listitem>
|
||||||
</itemizedlist>
|
</itemizedlist>
|
||||||
</section>
|
</section>
|
||||||
|
|
Loading…
Reference in New Issue