Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14688 Cleanup MOB tests

This commit is contained in:
Matteo Bertozzi 2015-10-27 10:28:07 -07:00
parent e04e7402cd
commit c91bfff586
13 changed files with 270 additions and 472 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
View File

@ -1845,7 +1845,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
//
// ==========================================================================
/**
* Provide an existing table name to truncate.
* Scans the table and issues a delete for each row read.
@ -2141,7 +2141,10 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
* Return the number of rows in the given table.
*/
public int countRows(final Table table) throws IOException {
Scan scan = new Scan();
return countRows(table, new Scan());
}
public int countRows(final Table table, final Scan scan) throws IOException {
ResultScanner results = table.getScanner(scan);
int count = 0;
for (@SuppressWarnings("unused") Result res : results) {

42
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
View File

@ -19,11 +19,16 @@
package org.apache.hadoop.hbase.mob;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
@ -73,15 +78,30 @@ public class MobTestUtil {
/**
* Compare two Cells only for their row family qualifier value
*/
public static void assertCellEquals(Cell firstKeyValue,
Cell secondKeyValue) {
Assert.assertEquals(Bytes.toString(CellUtil.cloneRow(firstKeyValue)),
Bytes.toString(CellUtil.cloneRow(secondKeyValue)));
Assert.assertEquals(Bytes.toString(CellUtil.cloneFamily(firstKeyValue)),
Bytes.toString(CellUtil.cloneFamily(secondKeyValue)));
Assert.assertEquals(Bytes.toString(CellUtil.cloneQualifier(firstKeyValue)),
Bytes.toString(CellUtil.cloneQualifier(secondKeyValue)));
Assert.assertEquals(Bytes.toString(CellUtil.cloneValue(firstKeyValue)),
Bytes.toString(CellUtil.cloneValue(secondKeyValue)));
}
public static void assertCellEquals(Cell firstKeyValue, Cell secondKeyValue) {
Assert.assertArrayEquals(CellUtil.cloneRow(firstKeyValue),
CellUtil.cloneRow(secondKeyValue));
Assert.assertArrayEquals(CellUtil.cloneFamily(firstKeyValue),
CellUtil.cloneFamily(secondKeyValue));
Assert.assertArrayEquals(CellUtil.cloneQualifier(firstKeyValue),
CellUtil.cloneQualifier(secondKeyValue));
Assert.assertArrayEquals(CellUtil.cloneValue(firstKeyValue),
CellUtil.cloneValue(secondKeyValue));
}
public static void assertCellsValue(Table table, Scan scan,
byte[] expectedValue, int expectedCount) throws IOException {
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
List<Cell> cells = res.listCells();
for(Cell cell : cells) {
// Verify the value
Assert.assertArrayEquals(expectedValue, CellUtil.cloneValue(cell));
count++;
}
}
results.close();
Assert.assertEquals(expectedCount, count);
}
}

31
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
View File

@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@ -43,10 +44,9 @@ import org.junit.experimental.categories.Category;
@Category(SmallTests.class)
public class TestCachedMobFile extends TestCase{
static final Log LOG = LogFactory.getLog(TestCachedMobFile.class);
private Configuration conf = HBaseConfiguration.create();
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Configuration conf = TEST_UTIL.getConfiguration();
private CacheConfig cacheConf = new CacheConfig(conf);
private static final String TABLE = "tableName";
private static final String FAMILY = "familyName";
private static final String FAMILY1 = "familyName1";
private static final String FAMILY2 = "familyName2";
private static final long EXPECTED_REFERENCE_ZERO = 0;
@ -56,13 +56,11 @@ public class TestCachedMobFile extends TestCase{
@Test
public void testOpenClose() throws Exception {
String caseName = getName();
FileSystem fs = FileSystem.get(conf);
Path testDir = FSUtils.getRootDir(conf);
Path outputDir = new Path(new Path(testDir, TABLE),
FAMILY);
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir).withFileContext(meta).build();
.withOutputDir(testDir).withFileContext(meta).build();
MobTestUtil.writeStoreFile(writer, caseName);
CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
@ -79,17 +77,15 @@ public class TestCachedMobFile extends TestCase{
@Test
public void testCompare() throws Exception {
String caseName = getName();
FileSystem fs = FileSystem.get(conf);
Path testDir = FSUtils.getRootDir(conf);
Path outputDir1 = new Path(new Path(testDir, TABLE),
FAMILY1);
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
Path outputDir1 = new Path(testDir, FAMILY1);
HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
StoreFile.Writer writer1 = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir1).withFileContext(meta).build();
MobTestUtil.writeStoreFile(writer1, caseName);
CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf);
Path outputDir2 = new Path(new Path(testDir, TABLE),
FAMILY2);
Path outputDir2 = new Path(testDir, FAMILY2);
StoreFile.Writer writer2 = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir2)
.withFileContext(meta)
@ -105,12 +101,11 @@ public class TestCachedMobFile extends TestCase{
@Test
public void testReadKeyValue() throws Exception {
FileSystem fs = FileSystem.get(conf);
Path testDir = FSUtils.getRootDir(conf);
Path outputDir = new Path(new Path(testDir, TABLE), "familyname");
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir).withFileContext(meta).build();
.withOutputDir(testDir).withFileContext(meta).build();
String caseName = getName();
MobTestUtil.writeStoreFile(writer, caseName);
CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);

177
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
View File

@ -18,7 +18,6 @@
*/
package org.apache.hadoop.hbase.mob;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
@ -26,9 +25,7 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@ -64,127 +61,67 @@ public class TestDefaultMobStoreFlusher {
}
@Test
public void testFlushNonMobFile() throws InterruptedException {
String TN = "testFlushNonMobFile";
TableName tn = TableName.valueOf(TN);
Table table = null;
HBaseAdmin admin = null;
public void testFlushNonMobFile() throws Exception {
TableName tn = TableName.valueOf("testFlushNonMobFile");
HTableDescriptor desc = new HTableDescriptor(tn);
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMaxVersions(4);
desc.addFamily(hcd);
try {
HTableDescriptor desc = new HTableDescriptor(tn);
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMaxVersions(4);
desc.addFamily(hcd);
admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(desc);
table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
.getTable(TableName.valueOf(TN));
//Put data
Put put0 = new Put(row1);
put0.addColumn(family, qf1, 1, value1);
table.put(put0);
//Put more data
Put put1 = new Put(row2);
put1.addColumn(family, qf2, 1, value2);
table.put(put1);
//Flush
admin.flush(tn);
Scan scan = new Scan();
scan.addColumn(family, qf1);
scan.setMaxVersions(4);
ResultScanner scanner = table.getScanner(scan);
//Compare
Result result = scanner.next();
int size = 0;
while (result != null) {
size++;
List<Cell> cells = result.getColumnCells(family, qf1);
// Verify the cell size
Assert.assertEquals(1, cells.size());
// Verify the value
Assert.assertEquals(Bytes.toString(value1),
Bytes.toString(CellUtil.cloneValue(cells.get(0))));
result = scanner.next();
}
scanner.close();
Assert.assertEquals(1, size);
admin.close();
} catch (MasterNotRunningException e1) {
e1.printStackTrace();
} catch (ZooKeeperConnectionException e2) {
e2.printStackTrace();
} catch (IOException e3) {
e3.printStackTrace();
}
testFlushFile(desc);
}
@Test
public void testFlushMobFile() throws InterruptedException {
String TN = "testFlushMobFile";
TableName tn = TableName.valueOf(TN);
Table table = null;
Admin admin = null;
public void testFlushMobFile() throws Exception {
TableName tn = TableName.valueOf("testFlushMobFile");
HTableDescriptor desc = new HTableDescriptor(tn);
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
hcd.setMobThreshold(3L);
hcd.setMaxVersions(4);
desc.addFamily(hcd);
try {
HTableDescriptor desc = new HTableDescriptor(tn);
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
hcd.setMobThreshold(3L);
hcd.setMaxVersions(4);
desc.addFamily(hcd);
Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
admin = c.getAdmin();
admin.createTable(desc);
table = c.getTable(TableName.valueOf(TN));
//put data
Put put0 = new Put(row1);
put0.addColumn(family, qf1, 1, value1);
table.put(put0);
//put more data
Put put1 = new Put(row2);
put1.addColumn(family, qf2, 1, value2);
table.put(put1);
//flush
admin.flush(tn);
//Scan
Scan scan = new Scan();
scan.addColumn(family, qf1);
scan.setMaxVersions(4);
ResultScanner scanner = table.getScanner(scan);
//Compare
Result result = scanner.next();
int size = 0;
while (result != null) {
size++;
List<Cell> cells = result.getColumnCells(family, qf1);
// Verify the the cell size
Assert.assertEquals(1, cells.size());
// Verify the value
Assert.assertEquals(Bytes.toString(value1),
Bytes.toString(CellUtil.cloneValue(cells.get(0))));
result = scanner.next();
}
scanner.close();
Assert.assertEquals(1, size);
admin.close();
} catch (MasterNotRunningException e1) {
e1.printStackTrace();
} catch (ZooKeeperConnectionException e2) {
e2.printStackTrace();
} catch (IOException e3) {
e3.printStackTrace();
}
testFlushFile(desc);
}
private void testFlushFile(HTableDescriptor htd) throws Exception {
Table table = null;
try {
table = TEST_UTIL.createTable(htd, null);
//put data
Put put0 = new Put(row1);
put0.addColumn(family, qf1, 1, value1);
table.put(put0);
//put more data
Put put1 = new Put(row2);
put1.addColumn(family, qf2, 1, value2);
table.put(put1);
//flush
TEST_UTIL.flush(htd.getTableName());
//Scan
Scan scan = new Scan();
scan.addColumn(family, qf1);
scan.setMaxVersions(4);
ResultScanner scanner = table.getScanner(scan);
//Compare
int size = 0;
for (Result result: scanner) {
size++;
List<Cell> cells = result.getColumnCells(family, qf1);
// Verify the cell size
Assert.assertEquals(1, cells.size());
// Verify the value
Assert.assertArrayEquals(value1, CellUtil.cloneValue(cells.get(0)));
}
scanner.close();
Assert.assertEquals(1, size);
} finally {
table.close();
}
}
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java
View File

@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@ -124,7 +123,7 @@ public class TestExpiredMobFileCleaner {
public void testCleaner() throws Exception {
init();
Path mobDirPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
byte[] dummyData = makeDummyData(600);
long ts = System.currentTimeMillis() - 3 * secondsOfDay() * 1000; // 3 days before
@ -158,11 +157,6 @@ public class TestExpiredMobFileCleaner {
assertEquals("After cleanup without delay 2", secondFile, lastFile);
}
private Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
Path p = new Path(MobUtils.getMobRegionPath(conf, tableName), familyName);
return p;
}
private int secondsOfDay() {
return 24 * 3600;
}

19
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
View File

@ -18,11 +18,8 @@
*/
package org.apache.hadoop.hbase.mob;
import java.util.List;
import java.util.Random;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
@ -32,7 +29,6 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -117,19 +113,6 @@ public class TestMobDataBlockEncoding {
Scan scan = new Scan();
scan.setMaxVersions(4);
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
List<Cell> cells = res.listCells();
for(Cell cell : cells) {
// Verify the value
Assert.assertEquals(Bytes.toString(value),
Bytes.toString(CellUtil.cloneValue(cell)));
count++;
}
}
results.close();
Assert.assertEquals(3, count);
MobTestUtil.assertCellsValue(table, scan, value, 3);
}
}

16
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
View File

@ -47,17 +47,14 @@ public class TestMobFile extends TestCase {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Configuration conf = TEST_UTIL.getConfiguration();
private CacheConfig cacheConf = new CacheConfig(conf);
private final String TABLE = "tableName";
private final String FAMILY = "familyName";
@Test
public void testReadKeyValue() throws Exception {
FileSystem fs = FileSystem.get(conf);
Path testDir = FSUtils.getRootDir(conf);
Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir)
.withOutputDir(testDir)
.withFileContext(meta)
.build();
String caseName = getName();
@ -106,12 +103,11 @@ public class TestMobFile extends TestCase {
@Test
public void testGetScanner() throws Exception {
FileSystem fs = FileSystem.get(conf);
Path testDir = FSUtils.getRootDir(conf);
Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
.withOutputDir(outputDir)
.withOutputDir(testDir)
.withFileContext(meta)
.build();
MobTestUtil.writeStoreFile(writer, getName());

28
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
View File

@ -413,8 +413,8 @@ public class TestMobCompactor {
result = table.get(get);
cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
// the ref name is the new file
Path mobFamilyPath = new Path(
MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString());
Path mobFamilyPath =
MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, hcd1.getNameAsString());
List<Path> paths = new ArrayList<Path>();
if (fs.exists(mobFamilyPath)) {
FileStatus[] files = fs.listStatus(mobFamilyPath);
@ -495,13 +495,7 @@ public class TestMobCompactor {
Scan scan = new Scan();
// Do not retrieve the mob data when scanning
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
count++;
}
results.close();
return count;
return TEST_UTIL.countRows(table, scan);
}
/**
@ -532,8 +526,7 @@ public class TestMobCompactor {
*/
private int countFiles(TableName tableName, boolean isMobFile, String familyName)
throws IOException {
Path mobDirPath = MobUtils.getMobFamilyPath(
MobUtils.getMobRegionPath(conf, tableName), familyName);
Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
int count = 0;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@ -553,8 +546,7 @@ public class TestMobCompactor {
}
private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
familyName);
Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
boolean hasFiles = false;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@ -579,8 +571,7 @@ public class TestMobCompactor {
* @return the number of the HFileLink
*/
private int countHFileLinks(String familyName) throws IOException {
Path mobDirPath = MobUtils.getMobFamilyPath(
MobUtils.getMobRegionPath(conf, tableName), familyName);
Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
int count = 0;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@ -601,8 +592,7 @@ public class TestMobCompactor {
* @return the number of files large than the size
*/
private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException {
Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
familyName);
Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName);
int count = 0;
if (fs.exists(mobDirPath)) {
FileStatus[] files = fs.listStatus(mobDirPath);
@ -729,8 +719,8 @@ public class TestMobCompactor {
// Do not retrieve the mob data when scanning
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
ResultScanner results = table.getScanner(scan);
Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
tableName), familyName);
Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(),
tableName, familyName);
List<Path> actualFilePaths = new ArrayList<>();
List<Path> expectFilePaths = new ArrayList<>();
for (Result res : results) {

105
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
View File

@ -104,8 +104,27 @@ public class TestPartitionedMobCompactor {
@Test
public void testCompactionSelectWithAllFiles() throws Exception {
resetConf();
String tableName = "testCompactionSelectWithAllFiles";
testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD,
CompactionType.ALL_FILES, false);
}
@Test
public void testCompactionSelectWithPartFiles() throws Exception {
String tableName = "testCompactionSelectWithPartFiles";
testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false);
}
@Test
public void testCompactionSelectWithForceAllFiles() throws Exception {
String tableName = "testCompactionSelectWithForceAllFiles";
testCompactionAtMergeSize(tableName, Long.MAX_VALUE, CompactionType.ALL_FILES, true);
}
private void testCompactionAtMergeSize(final String tableName,
final long mergeSize, final CompactionType type, final boolean isForceAllFiles)
throws Exception {
resetConf();
init(tableName);
int count = 10;
// create 10 mob files.
@ -113,7 +132,6 @@ public class TestPartitionedMobCompactor {
// create 10 del files
createStoreFiles(basePath, family, qf, count, Type.Delete);
listFiles();
long mergeSize = MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD;
List<String> expectedStartKeys = new ArrayList<>();
for(FileStatus file : mobFiles) {
if(file.getLen() < mergeSize) {
@ -122,90 +140,33 @@ public class TestPartitionedMobCompactor {
expectedStartKeys.add(startKey);
}
}
testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
}
@Test
public void testCompactionSelectWithPartFiles() throws Exception {
resetConf();
String tableName = "testCompactionSelectWithPartFiles";
init(tableName);
int count = 10;
// create 10 mob files.
createStoreFiles(basePath, family, qf, count, Type.Put);
// create 10 del files
createStoreFiles(basePath, family, qf, count, Type.Delete);
listFiles();
long mergeSize = 4000;
List<String> expectedStartKeys = new ArrayList<>();
for(FileStatus file : mobFiles) {
if(file.getLen() < 4000) {
String fileName = file.getPath().getName();
String startKey = fileName.substring(0, 32);
expectedStartKeys.add(startKey);
}
}
// set the mob compaction mergeable threshold
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
}
@Test
public void testCompactionSelectWithForceAllFiles() throws Exception {
resetConf();
String tableName = "testCompactionSelectWithForceAllFiles";
init(tableName);
int count = 10;
// create 10 mob files.
createStoreFiles(basePath, family, qf, count, Type.Put);
// create 10 del files
createStoreFiles(basePath, family, qf, count, Type.Delete);
listFiles();
long mergeSize = 4000;
List<String> expectedStartKeys = new ArrayList<>();
for(FileStatus file : mobFiles) {
String fileName = file.getPath().getName();
String startKey = fileName.substring(0, 32);
expectedStartKeys.add(startKey);
}
// set the mob compaction mergeable threshold
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
testSelectFiles(tableName, type, isForceAllFiles, expectedStartKeys);
}
@Test
public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
resetConf();
String tableName = "testCompactDelFilesWithDefaultBatchSize";
init(tableName);
// create 20 mob files.
createStoreFiles(basePath, family, qf, 20, Type.Put);
// create 13 del files
createStoreFiles(basePath, family, qf, 13, Type.Delete);
listFiles();
testCompactDelFiles(tableName, 1, 13, false);
testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE,
MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
}
@Test
public void testCompactDelFilesWithSmallBatchSize() throws Exception {
resetConf();
String tableName = "testCompactDelFilesWithSmallBatchSize";
init(tableName);
// create 20 mob files.
createStoreFiles(basePath, family, qf, 20, Type.Put);
// create 13 del files
createStoreFiles(basePath, family, qf, 13, Type.Delete);
listFiles();
// set the mob compaction batch size
conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 4);
testCompactDelFiles(tableName, 1, 13, false);
testCompactDelFilesAtBatchSize(tableName, 4, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
}
@Test
public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
resetConf();
String tableName = "testCompactDelFilesWithSmallBatchSize";
testCompactDelFilesAtBatchSize(tableName, 4, 2);
}
private void testCompactDelFilesAtBatchSize(String tableName, int batchSize,
int delfileMaxCount) throws Exception {
resetConf();
init(tableName);
// create 20 mob files.
createStoreFiles(basePath, family, qf, 20, Type.Put);
@ -214,10 +175,10 @@ public class TestPartitionedMobCompactor {
listFiles();
// set the max del file count
conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, delfileMaxCount);
// set the mob compaction batch size
conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 2);
testCompactDelFiles(tableName, 4, 13, false);
conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
testCompactDelFiles(tableName, 1, 13, false);
}
/**

157
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
View File

@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
@ -74,132 +73,110 @@ public class TestDeleteMobTable {
return mobVal;
}
@Test
public void testDeleteMobTable() throws Exception {
byte[] tableName = Bytes.toBytes("testDeleteMobTable");
TableName tn = TableName.valueOf(tableName);
HTableDescriptor htd = new HTableDescriptor(tn);
private HTableDescriptor createTableDescriptor(TableName tableName, boolean hasMob) {
HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
hcd.setMobEnabled(true);
hcd.setMobThreshold(0);
if (hasMob) {
hcd.setMobEnabled(true);
hcd.setMobThreshold(0);
}
htd.addFamily(hcd);
HBaseAdmin admin = null;
Table table = null;
try {
admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(htd);
table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
byte[] value = generateMobValue(10);
return htd;
}
private Table createTableWithOneFile(HTableDescriptor htd) throws IOException {
Table table = TEST_UTIL.createTable(htd, null);
try {
// insert data
byte[] value = generateMobValue(10);
byte[] row = Bytes.toBytes("row");
Put put = new Put(row);
put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
table.put(put);
admin.flush(tn);
// create an hfile
TEST_UTIL.getHBaseAdmin().flush(htd.getTableName());
} catch (IOException e) {
table.close();
throw e;
}
return table;
}
@Test
public void testDeleteMobTable() throws Exception {
TableName tn = TableName.valueOf("testDeleteMobTable");
HTableDescriptor htd = createTableDescriptor(tn, true);
HColumnDescriptor hcd = htd.getFamily(FAMILY);
String fileName = null;
Table table = createTableWithOneFile(htd);
try {
// the mob file exists
Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertTrue(mobTableDirExist(tn));
table.close();
admin.disableTable(tn);
admin.deleteTable(tn);
Assert.assertFalse(admin.tableExists(tn));
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertFalse(mobTableDirExist(tn));
} finally {
if (admin != null) {
admin.close();
}
table.close();
TEST_UTIL.deleteTable(tn);
}
Assert.assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(tn));
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertFalse(mobTableDirExist(tn));
}
@Test
public void testDeleteNonMobTable() throws Exception {
byte[] tableName = Bytes.toBytes("testDeleteNonMobTable");
TableName tn = TableName.valueOf(tableName);
HTableDescriptor htd = new HTableDescriptor(tn);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
htd.addFamily(hcd);
HBaseAdmin admin = null;
Table table = null;
TableName tn = TableName.valueOf("testDeleteNonMobTable");
HTableDescriptor htd = createTableDescriptor(tn, false);
HColumnDescriptor hcd = htd.getFamily(FAMILY);
Table table = createTableWithOneFile(htd);
try {
admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(htd);
table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
byte[] value = generateMobValue(10);
byte[] row = Bytes.toBytes("row");
Put put = new Put(row);
put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
table.put(put);
admin.flush(tn);
table.close();
// the mob file doesn't exist
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertFalse(mobTableDirExist(tn));
admin.disableTable(tn);
admin.deleteTable(tn);
Assert.assertFalse(admin.tableExists(tn));
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertFalse(mobTableDirExist(tn));
} finally {
if (admin != null) {
admin.close();
}
table.close();
TEST_UTIL.deleteTable(tn);
}
Assert.assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(tn));
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertFalse(mobTableDirExist(tn));
}
@Test
public void testMobFamilyDelete() throws Exception {
TableName tn = TableName.valueOf("testMobFamilyDelete");
HTableDescriptor htd = new HTableDescriptor(tn);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
hcd.setMobEnabled(true);
hcd.setMobThreshold(0);
htd.addFamily(hcd);
HTableDescriptor htd = createTableDescriptor(tn, true);
HColumnDescriptor hcd = htd.getFamily(FAMILY);
htd.addFamily(new HColumnDescriptor(Bytes.toBytes("family2")));
HBaseAdmin admin = null;
Table table = null;
Table table = createTableWithOneFile(htd);
try {
admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(htd);
table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn);
byte[] value = generateMobValue(10);
byte[] row = Bytes.toBytes("row");
Put put = new Put(row);
put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
table.put(put);
admin.flush(tn);
// the mob file exists
Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertTrue(mobTableDirExist(tn));
admin.deleteColumnFamily(tn, FAMILY);
TEST_UTIL.getHBaseAdmin().deleteColumnFamily(tn, FAMILY);
Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
Assert.assertFalse(mobColumnFamilyDirExist(tn));
Assert.assertFalse(mobColumnFamilyDirExist(tn, hcd.getNameAsString()));
} finally {
table.close();
if (admin != null) {
admin.close();
}
TEST_UTIL.deleteTable(tn);
}
}
@ -209,9 +186,8 @@ public class TestDeleteMobTable {
Path mobFileDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
if (fs.exists(mobFileDir)) {
return fs.listStatus(mobFileDir).length;
} else {
return 0;
}
return 0;
}
private int countArchiveMobFiles(TableName tn, String familyName)
@ -221,9 +197,8 @@ public class TestDeleteMobTable {
MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName);
if (fs.exists(storePath)) {
return fs.listStatus(storePath).length;
} else {
return 0;
}
return 0;
}
private boolean mobTableDirExist(TableName tn) throws IOException {
@ -232,12 +207,9 @@ public class TestDeleteMobTable {
return fs.exists(tableDir);
}
private boolean mobColumnFamilyDirExist(TableName tn) throws IOException {
private boolean mobColumnFamilyDirExist(TableName tn, String familyName) throws IOException {
FileSystem fs = TEST_UTIL.getTestFileSystem();
Path tableDir = FSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn);
HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tn);
Path mobFamilyDir = new Path(tableDir, new Path(mobRegionInfo.getEncodedName(),
Bytes.toString(FAMILY)));
Path mobFamilyDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
return fs.exists(mobFamilyDir);
}
@ -256,8 +228,7 @@ public class TestDeleteMobTable {
ResultScanner rs = table.getScanner(scan);
Result r = rs.next();
Assert.assertNotNull(r);
byte[] value = r.getValue(FAMILY, QF);
String fileName = Bytes.toString(value, Bytes.SIZEOF_INT, value.length - Bytes.SIZEOF_INT);
String fileName = MobUtils.getMobFileName(r.getColumnLatestCell(FAMILY, QF));
Path filePath = new Path(
MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName), fileName);
FileSystem fs = TEST_UTIL.getTestFileSystem();

31
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
View File

@ -278,8 +278,7 @@ public class TestMobStoreCompaction {
}
private int countMobFiles() throws IOException {
Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
hcd.getNameAsString());
Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString());
if (fs.exists(mobDirPath)) {
FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
return files.length;
@ -289,8 +288,7 @@ public class TestMobStoreCompaction {
private long countMobCellsInMetadata() throws IOException {
long mobCellsCount = 0;
Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
hcd.getNameAsString());
Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString());
Configuration copyOfConf = new Configuration(conf);
copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
CacheConfig cacheConfig = new CacheConfig(copyOfConf);
@ -357,9 +355,16 @@ public class TestMobStoreCompaction {
private int countRows() throws IOException {
Scan scan = new Scan();
// Do not retrieve the mob data when scanning
InternalScanner scanner = region.getScanner(scan);
try {
return countRows(scanner);
} finally {
scanner.close();
}
}
private int countRows(InternalScanner scanner) throws IOException {
// Do not retrieve the mob data when scanning
int scannedCount = 0;
List<Cell> results = new ArrayList<Cell>();
boolean hasMore = true;
@ -368,8 +373,6 @@ public class TestMobStoreCompaction {
scannedCount += results.size();
results.clear();
}
scanner.close();
return scannedCount;
}
@ -423,8 +426,7 @@ public class TestMobStoreCompaction {
Configuration copyOfConf = new Configuration(conf);
copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
CacheConfig cacheConfig = new CacheConfig(copyOfConf);
Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
hcd.getNameAsString());
Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString());
List<StoreFile> sfs = new ArrayList<>();
int numDelfiles = 0;
int size = 0;
@ -436,6 +438,7 @@ public class TestMobStoreCompaction {
numDelfiles++;
}
}
List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, false,
HConstants.LATEST_TIMESTAMP);
Scan scan = new Scan();
@ -446,12 +449,10 @@ public class TestMobStoreCompaction {
CellComparator.COMPARATOR);
StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
scanners, 0L, HConstants.LATEST_TIMESTAMP);
List<Cell> results = new ArrayList<>();
boolean hasMore = true;
while (hasMore) {
hasMore = scanner.next(results);
size += results.size();
results.clear();
try {
size += countRows(scanner);
} finally {
scanner.close();
}
}
// assert the number of the existing del files

116
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
View File

@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobTestUtil;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@ -196,12 +197,12 @@ public class TestMobStoreScanner {
table.put(put4);
Result result = rs.next();
Cell cell = result.getColumnLatestCell(family, qf1);
Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(cell)));
Assert.assertArrayEquals(value1, CellUtil.cloneValue(cell));
admin.flush(tn);
result = rs.next();
cell = result.getColumnLatestCell(family, qf1);
Assert.assertEquals("value2", Bytes.toString(CellUtil.cloneValue(cell)));
Assert.assertArrayEquals(value2, CellUtil.cloneValue(cell));
}
@Test
@ -213,7 +214,7 @@ public class TestMobStoreScanner {
get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true));
Result result = table.get(get);
Cell cell = result.getColumnLatestCell(family, qf1);
Assert.assertEquals(0, CellUtil.cloneValue(cell).length);
Assert.assertEquals(0, cell.getValueLength());
}
@Test
@ -249,8 +250,7 @@ public class TestMobStoreScanner {
private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family)
throws IOException {
Path regionDir = MobUtils.getMobRegionPath(conf, table);
Path famDir = new Path(regionDir, family);
Path famDir = MobUtils.getMobFamilyPath(conf, table, family);
FileStatus[] hfFss = fs.listStatus(famDir);
for (FileStatus hfs : hfFss) {
if (!hfs.isDirectory()) {
@ -262,7 +262,17 @@ public class TestMobStoreScanner {
private void testGetFromFiles(boolean reversed) throws Exception {
TableName tn = TableName.valueOf("testGetFromFiles" + reversed);
setUp(defaultThreshold, tn);
testGet(tn, reversed, true);
}
private void testGetFromMemStore(boolean reversed) throws Exception {
TableName tn = TableName.valueOf("testGetFromMemStore" + reversed);
testGet(tn, reversed, false);
}
private void testGet(TableName tableName, boolean reversed, boolean doFlush)
throws Exception {
setUp(defaultThreshold, tableName);
long ts1 = System.currentTimeMillis();
long ts2 = ts1 + 1;
long ts3 = ts1 + 2;
@ -274,55 +284,13 @@ public class TestMobStoreScanner {
put1.addColumn(family, qf3, ts1, value);
table.put(put1);
admin.flush(tn);
if (doFlush) {
admin.flush(tableName);
}
Scan scan = new Scan();
setScan(scan, reversed, false);
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
List<Cell> cells = res.listCells();
for(Cell cell : cells) {
// Verify the value
Assert.assertEquals(Bytes.toString(value),
Bytes.toString(CellUtil.cloneValue(cell)));
count++;
}
}
results.close();
Assert.assertEquals(3, count);
}
private void testGetFromMemStore(boolean reversed) throws Exception {
setUp(defaultThreshold, TableName.valueOf("testGetFromMemStore" + reversed));
long ts1 = System.currentTimeMillis();
long ts2 = ts1 + 1;
long ts3 = ts1 + 2;
byte [] value = generateMobValue((int)defaultThreshold+1);;
Put put1 = new Put(row1);
put1.addColumn(family, qf1, ts3, value);
put1.addColumn(family, qf2, ts2, value);
put1.addColumn(family, qf3, ts1, value);
table.put(put1);
Scan scan = new Scan();
setScan(scan, reversed, false);
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
List<Cell> cells = res.listCells();
for(Cell cell : cells) {
// Verify the value
Assert.assertEquals(Bytes.toString(value),
Bytes.toString(CellUtil.cloneValue(cell)));
count++;
}
}
results.close();
Assert.assertEquals(3, count);
MobTestUtil.assertCellsValue(table, scan, value, 3);
}
private void testGetReferences(boolean reversed) throws Exception {
@ -426,8 +394,8 @@ public class TestMobStoreScanner {
// Get the files in the mob path
Path mobFamilyPath;
mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tn),
hcd.getNameAsString());
mobFamilyPath = MobUtils.getMobFamilyPath(
TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString());
FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
FileStatus[] files = fs.listStatus(mobFamilyPath);
@ -458,19 +426,7 @@ public class TestMobStoreScanner {
// Scan from archive
Scan scan = new Scan();
setScan(scan, reversed, false);
ResultScanner results = table.getScanner(scan);
int count = 0;
for (Result res : results) {
List<Cell> cells = res.listCells();
for(Cell cell : cells) {
// Verify the value
Assert.assertEquals(Bytes.toString(value),
Bytes.toString(CellUtil.cloneValue(cell)));
count++;
}
}
results.close();
Assert.assertEquals(3, count);
MobTestUtil.assertCellsValue(table, scan, value, 3);
}
/**
@ -478,12 +434,9 @@ public class TestMobStoreScanner {
*/
private static void assertNotMobReference(Cell cell, byte[] row, byte[] family,
byte[] value) throws IOException {
Assert.assertEquals(Bytes.toString(row),
Bytes.toString(CellUtil.cloneRow(cell)));
Assert.assertEquals(Bytes.toString(family),
Bytes.toString(CellUtil.cloneFamily(cell)));
Assert.assertTrue(Bytes.toString(value).equals(
Bytes.toString(CellUtil.cloneValue(cell))));
Assert.assertArrayEquals(row, CellUtil.cloneRow(cell));
Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell));
Assert.assertArrayEquals(value, CellUtil.cloneValue(cell));
}
/**
@ -491,20 +444,15 @@ public class TestMobStoreScanner {
*/
private static void assertIsMobReference(Cell cell, byte[] row, byte[] family,
byte[] value, TableName tn) throws IOException {
Assert.assertEquals(Bytes.toString(row),
Bytes.toString(CellUtil.cloneRow(cell)));
Assert.assertEquals(Bytes.toString(family),
Bytes.toString(CellUtil.cloneFamily(cell)));
Assert.assertFalse(Bytes.toString(value).equals(
Bytes.toString(CellUtil.cloneValue(cell))));
Assert.assertArrayEquals(row, CellUtil.cloneRow(cell));
Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell));
Assert.assertFalse(Bytes.equals(value, CellUtil.cloneValue(cell)));
byte[] referenceValue = CellUtil.cloneValue(cell);
String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
referenceValue.length - Bytes.SIZEOF_INT);
String fileName = MobUtils.getMobFileName(cell);
int valLen = Bytes.toInt(referenceValue, 0, Bytes.SIZEOF_INT);
Assert.assertEquals(value.length, valLen);
Path mobFamilyPath;
mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
tn), hcd.getNameAsString());
Path mobFamilyPath = MobUtils.getMobFamilyPath(
TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString());
Path targetPath = new Path(mobFamilyPath, fileName);
FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
Assert.assertTrue(fs.exists(targetPath));

5
hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
View File

@ -131,7 +131,7 @@ public class BaseTestHBaseFsck {
Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
/**
* Create a new region in META.
*/
@ -633,8 +633,7 @@ public class BaseTestHBaseFsck {
* @throws IOException
*/
Path getFlushedMobFile(FileSystem fs, TableName table) throws IOException {
Path regionDir = MobUtils.getMobRegionPath(conf, table);
Path famDir = new Path(regionDir, FAM_STR);
Path famDir = MobUtils.getMobFamilyPath(conf, table, FAM_STR);
// keep doing this until we get a legit hfile
while (true) {