HBASE-11647 MOB integration testing. (Jingcheng Du)
This commit is contained in:
anoopsjohn
parent
e8938c40d6
commit
3e563c5cc7
|
|
@ -0,0 +1,148 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.LoadTestDataGeneratorWithMOB;
|
||||
import org.apache.hadoop.hbase.util.LoadTestTool;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
/**
|
||||
* Integration Test for MOB ingest.
|
||||
*/
|
||||
@Category(IntegrationTests.class)
|
||||
public class IntegrationTestIngestWithMOB extends IntegrationTestIngest {
|
||||
private static final char COLON = ':';
|
||||
|
||||
private byte[] mobColumnFamily = LoadTestTool.COLUMN_FAMILY;
|
||||
public static final String THRESHOLD = "threshold";
|
||||
public static final String MIN_MOB_DATA_SIZE = "minMobDataSize";
|
||||
public static final String MAX_MOB_DATA_SIZE = "maxMobDataSize";
|
||||
private int threshold = 100 * 1024; //100KB
|
||||
private int minMobDataSize = threshold * 4 / 5; //80KB
|
||||
private int maxMobDataSize = threshold * 50; // 5MB
|
||||
|
||||
//similar to LOAD_TEST_TOOL_INIT_ARGS except OPT_IN_MEMORY is removed
|
||||
protected String[] LOAD_TEST_TOOL_MOB_INIT_ARGS = {
|
||||
LoadTestTool.OPT_COMPRESSION,
|
||||
LoadTestTool.OPT_DATA_BLOCK_ENCODING,
|
||||
LoadTestTool.OPT_ENCRYPTION,
|
||||
LoadTestTool.OPT_NUM_REGIONS_PER_SERVER,
|
||||
LoadTestTool.OPT_REGION_REPLICATION,
|
||||
};
|
||||
|
||||
@Override
|
||||
protected String[] getArgsForLoadTestToolInitTable() {
|
||||
List<String> args = new ArrayList<String>();
|
||||
args.add("-tn");
|
||||
args.add(getTablename());
|
||||
// pass all remaining args from conf with keys <test class name>.<load test tool arg>
|
||||
String clazz = this.getClass().getSimpleName();
|
||||
for (String arg : LOAD_TEST_TOOL_MOB_INIT_ARGS) {
|
||||
String val = conf.get(String.format("%s.%s", clazz, arg));
|
||||
if (val != null) {
|
||||
args.add("-" + arg);
|
||||
args.add(val);
|
||||
}
|
||||
}
|
||||
args.add("-init_only");
|
||||
return args.toArray(new String[args.size()]);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void addOptions() {
|
||||
super.addOptions();
|
||||
super.addOptWithArg(THRESHOLD, "The threshold to classify cells to mob data");
|
||||
super.addOptWithArg(MIN_MOB_DATA_SIZE, "Minimum value size for mob data");
|
||||
super.addOptWithArg(MAX_MOB_DATA_SIZE, "Maximum value size for mob data");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processOptions(CommandLine cmd) {
|
||||
super.processOptions(cmd);
|
||||
if (cmd.hasOption(THRESHOLD)) {
|
||||
threshold = Integer.parseInt(cmd.getOptionValue(THRESHOLD));
|
||||
}
|
||||
if (cmd.hasOption(MIN_MOB_DATA_SIZE)) {
|
||||
minMobDataSize = Integer.parseInt(cmd.getOptionValue(MIN_MOB_DATA_SIZE));
|
||||
}
|
||||
if (cmd.hasOption(MAX_MOB_DATA_SIZE)) {
|
||||
maxMobDataSize = Integer.parseInt(cmd.getOptionValue(MAX_MOB_DATA_SIZE));
|
||||
}
|
||||
if (minMobDataSize > maxMobDataSize) {
|
||||
throw new IllegalArgumentException(
|
||||
"The minMobDataSize should not be larger than minMobDataSize");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initTable() throws IOException {
|
||||
super.initTable();
|
||||
|
||||
byte[] tableName = getTablename().getBytes();
|
||||
HBaseAdmin admin = new HBaseAdmin(conf);
|
||||
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
|
||||
LOG.info("Disabling table " + getTablename());
|
||||
admin.disableTable(tableName);
|
||||
for (HColumnDescriptor columnDescriptor : tableDesc.getFamilies()) {
|
||||
if(Arrays.equals(columnDescriptor.getName(), mobColumnFamily)) {
|
||||
columnDescriptor.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
|
||||
columnDescriptor.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(threshold));
|
||||
admin.modifyColumn(tableName, columnDescriptor);
|
||||
}
|
||||
}
|
||||
LOG.info("Enabling table " + getTablename());
|
||||
admin.enableTable(tableName);
|
||||
admin.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String[] getArgsForLoadTestTool(String mode, String modeSpecificArg, long startKey,
|
||||
long numKeys) {
|
||||
String[] args = super.getArgsForLoadTestTool(mode, modeSpecificArg, startKey, numKeys);
|
||||
List<String> tmp = new ArrayList<String>(Arrays.asList(args));
|
||||
// LoadTestDataGeneratorMOB:mobColumnFamily:minMobDataSize:maxMobDataSize
|
||||
tmp.add(HIPHEN + LoadTestTool.OPT_GENERATOR);
|
||||
StringBuilder sb = new StringBuilder(LoadTestDataGeneratorWithMOB.class.getName());
|
||||
sb.append(COLON);
|
||||
sb.append(mobColumnFamily);
|
||||
sb.append(COLON);
|
||||
sb.append(minMobDataSize);
|
||||
sb.append(COLON);
|
||||
sb.append(maxMobDataSize);
|
||||
tmp.add(sb.toString());
|
||||
return tmp.toArray(new String[tmp.size()]);
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
IntegrationTestingUtility.setUseDistributedCluster(conf);
|
||||
int ret = ToolRunner.run(conf, new IntegrationTestIngestWithMOB(), args);
|
||||
System.exit(ret);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
|
||||
|
||||
/**
|
||||
* A load test data generator for MOB
|
||||
*/
|
||||
public class LoadTestDataGeneratorWithMOB
|
||||
extends MultiThreadedAction.DefaultDataGenerator {
|
||||
|
||||
private byte[] mobColumnFamily;
|
||||
private LoadTestKVGenerator mobKvGenerator;
|
||||
|
||||
public LoadTestDataGeneratorWithMOB(int minValueSize, int maxValueSize,
|
||||
int minColumnsPerKey, int maxColumnsPerKey, byte[]... columnFamilies) {
|
||||
super(minValueSize, maxValueSize, minColumnsPerKey, maxColumnsPerKey,
|
||||
columnFamilies);
|
||||
}
|
||||
|
||||
public LoadTestDataGeneratorWithMOB(byte[]... columnFamilies) {
|
||||
super(columnFamilies);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(String[] args) {
|
||||
super.initialize(args);
|
||||
if (args.length != 3) {
|
||||
throw new IllegalArgumentException(
|
||||
"LoadTestDataGeneratorWithMOB can have 3 arguments."
|
||||
+ "1st arguement is a column family, the 2nd argument "
|
||||
+ "is the minimum mob data size and the 3rd argument "
|
||||
+ "is the maximum mob data size.");
|
||||
}
|
||||
String mobColumnFamily = args[0];
|
||||
int minMobDataSize = Integer.parseInt(args[1]);
|
||||
int maxMobDataSize = Integer.parseInt(args[2]);
|
||||
configureMob(Bytes.toBytes(mobColumnFamily), minMobDataSize, maxMobDataSize);
|
||||
}
|
||||
|
||||
private void configureMob(byte[] mobColumnFamily, int minMobDataSize,
|
||||
int maxMobDataSize) {
|
||||
this.mobColumnFamily = mobColumnFamily;
|
||||
mobKvGenerator = new LoadTestKVGenerator(minMobDataSize, maxMobDataSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] generateValue(byte[] rowKey, byte[] cf,
|
||||
byte[] column) {
|
||||
if(Arrays.equals(cf, mobColumnFamily))
|
||||
return mobKvGenerator.generateRandomSizeValue(rowKey, cf, column);
|
||||
|
||||
return super.generateValue(rowKey, cf, column);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue