Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13886 Return empty value when the mob file is corrupt instead of throwing exceptions. (Jingcheng)

This commit is contained in:
anoopsjohn 2015-06-16 17:17:03 +05:30
parent 2e2742183f
commit b889339006
6 changed files with 123 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
View File

@ -35,6 +35,7 @@ public class MobConstants {
public static final String MOB_SCAN_RAW = "hbase.mob.scan.raw";
public static final String MOB_CACHE_BLOCKS = "hbase.mob.cache.blocks";
public static final String MOB_SCAN_REF_ONLY = "hbase.mob.scan.ref.only";
public static final String EMPTY_VALUE_ON_MOBCELL_MISS = "empty.value.on.mobcell.miss";
public static final String MOB_FILE_CACHE_SIZE_KEY = "hbase.mob.file.cache.size";
public static final int DEFAULT_MOB_FILE_CACHE_SIZE = 1000;

15
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
View File

@ -855,4 +855,19 @@ public class MobUtils {
}
return false;
}
/**
* Indicates whether return null value when the mob file is missing or corrupt.
* The information is set in the attribute "empty.value.on.mobcell.miss" of scan.
* @param scan The current scan.
* @return True if the readEmptyValueOnMobCellMiss is enabled.
*/
public static boolean isReadEmptyValueOnMobCellMiss(Scan scan) {
byte[] readEmptyValueOnMobCellMiss = scan.getAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS);
try {
return readEmptyValueOnMobCellMiss != null && Bytes.toBoolean(readEmptyValueOnMobCellMiss);
} catch (IllegalArgumentException e) {
return false;
}
}
}

39
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
View File

@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.master.TableLockManager;
@ -310,13 +311,14 @@ public class HMobStore extends HStore {
/**
* Reads the cell from the mob file, and the read point does not count.
* This is used for DefaultMobStoreCompactor where we can read empty value for the missing cell.
* @param reference The cell found in the HBase, its value is a path to a mob file.
* @param cacheBlocks Whether the scanner should cache blocks.
* @return The cell found in the mob file.
* @throws IOException
*/
public Cell resolve(Cell reference, boolean cacheBlocks) throws IOException {
return resolve(reference, cacheBlocks, -1);
return resolve(reference, cacheBlocks, -1, true);
}
/**
@ -324,10 +326,13 @@ public class HMobStore extends HStore {
* @param reference The cell found in the HBase, its value is a path to a mob file.
* @param cacheBlocks Whether the scanner should cache blocks.
* @param readPt the read point.
* @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
* missing or corrupt.
* @return The cell found in the mob file.
* @throws IOException
*/
public Cell resolve(Cell reference, boolean cacheBlocks, long readPt) throws IOException {
public Cell resolve(Cell reference, boolean cacheBlocks, long readPt,
boolean readEmptyValueOnMobCellMiss) throws IOException {
Cell result = null;
if (MobUtils.hasValidMobRefCellValue(reference)) {
String fileName = MobUtils.getMobFileName(reference);
@ -352,7 +357,8 @@ public class HMobStore extends HStore {
keyLock.releaseLockEntry(lockEntry);
}
}
result = readCell(locations, fileName, reference, cacheBlocks, readPt);
result = readCell(locations, fileName, reference, cacheBlocks, readPt,
readEmptyValueOnMobCellMiss);
}
}
if (result == null) {
@ -380,12 +386,15 @@ public class HMobStore extends HStore {
* @param search The cell to be searched.
* @param cacheMobBlocks Whether the scanner should cache blocks.
* @param readPt the read point.
* @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
* missing or corrupt.
* @return The found cell. Null if there's no such a cell.
* @throws IOException
*/
private Cell readCell(List<Path> locations, String fileName, Cell search, boolean cacheMobBlocks,
long readPt) throws IOException {
long readPt, boolean readEmptyValueOnMobCellMiss) throws IOException {
FileSystem fs = getFileSystem();
Throwable throwable = null;
for (Path location : locations) {
MobFile file = null;
Path path = new Path(location, fileName);
@ -395,27 +404,39 @@ public class HMobStore extends HStore {
cacheMobBlocks);
} catch (IOException e) {
mobCacheConfig.getMobFileCache().evictFile(fileName);
throwable = e;
if ((e instanceof FileNotFoundException) ||
(e.getCause() instanceof FileNotFoundException)) {
LOG.warn("Fail to read the cell, the mob file " + path + " doesn't exist", e);
} else if (e instanceof CorruptHFileException) {
LOG.error("The mob file " + path + " is corrupt", e);
break;
} else {
throw e;
}
} catch (NullPointerException e) {
} catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt()
mobCacheConfig.getMobFileCache().evictFile(fileName);
LOG.warn("Fail to read the cell", e);
} catch (AssertionError e) {
throwable = e;
} catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt()
mobCacheConfig.getMobFileCache().evictFile(fileName);
LOG.warn("Fail to read the cell", e);
throwable = e;
} finally {
if (file != null) {
mobCacheConfig.getMobFileCache().closeFile(file);
}
}
}
LOG.error("The mob file " + fileName + " could not be found in the locations "
+ locations);
return null;
LOG.error("The mob file " + fileName + " could not be found in the locations " + locations
+ " or it is corrupt");
if (readEmptyValueOnMobCellMiss) {
return null;
} else if (throwable instanceof IOException) {
throw (IOException) throwable;
} else {
throw new IOException(throwable);
}
}
/**

9
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MobStoreScanner.java
View File

@ -36,12 +36,16 @@ import org.apache.hadoop.hbase.mob.MobUtils;
public class MobStoreScanner extends StoreScanner {
private boolean cacheMobBlocks = false;
private boolean rawMobScan = false;
private boolean readEmptyValueOnMobCellMiss = false;
private final HMobStore mobStore;
public MobStoreScanner(Store store, ScanInfo scanInfo, Scan scan,
final NavigableSet<byte[]> columns, long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
cacheMobBlocks = MobUtils.isCacheMobBlocks(scan);
rawMobScan = MobUtils.isRawMobScan(scan);
readEmptyValueOnMobCellMiss = MobUtils.isReadEmptyValueOnMobCellMiss(scan);
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
@ -56,7 +60,7 @@ public class MobStoreScanner extends StoreScanner {
@Override
public boolean next(List<Cell> outResult, ScannerContext ctx) throws IOException {
boolean result = super.next(outResult, ctx);
if (!MobUtils.isRawMobScan(scan)) {
if (!rawMobScan) {
// retrieve the mob data
if (outResult.isEmpty()) {
return result;
@ -66,7 +70,8 @@ public class MobStoreScanner extends StoreScanner {
for (int i = 0; i < outResult.size(); i++) {
Cell cell = outResult.get(i);
if (MobUtils.isMobReferenceCell(cell)) {
Cell mobCell = mobStore.resolve(cell, cacheMobBlocks, readPt);
Cell mobCell = mobStore
.resolve(cell, cacheMobBlocks, readPt, readEmptyValueOnMobCellMiss);
mobKVCount++;
mobKVSize += mobCell.getValueLength();
outResult.set(i, mobCell);

9
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
View File

@ -36,12 +36,16 @@ import org.apache.hadoop.hbase.mob.MobUtils;
public class ReversedMobStoreScanner extends ReversedStoreScanner {
private boolean cacheMobBlocks = false;
private boolean rawMobScan = false;
private boolean readEmptyValueOnMobCellMiss = false;
protected final HMobStore mobStore;
ReversedMobStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet<byte[]> columns,
long readPt) throws IOException {
super(store, scanInfo, scan, columns, readPt);
cacheMobBlocks = MobUtils.isCacheMobBlocks(scan);
rawMobScan = MobUtils.isRawMobScan(scan);
readEmptyValueOnMobCellMiss = MobUtils.isReadEmptyValueOnMobCellMiss(scan);
if (!(store instanceof HMobStore)) {
throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
}
@ -56,7 +60,7 @@ public class ReversedMobStoreScanner extends ReversedStoreScanner {
@Override
public boolean next(List<Cell> outResult, ScannerContext ctx) throws IOException {
boolean result = super.next(outResult, ctx);
if (!MobUtils.isRawMobScan(scan)) {
if (!rawMobScan) {
// retrieve the mob data
if (outResult.isEmpty()) {
return result;
@ -66,7 +70,8 @@ public class ReversedMobStoreScanner extends ReversedStoreScanner {
for (int i = 0; i < outResult.size(); i++) {
Cell cell = outResult.get(i);
if (MobUtils.isMobReferenceCell(cell)) {
Cell mobCell = mobStore.resolve(cell, cacheMobBlocks, readPt);
Cell mobCell = mobStore
.resolve(cell, cacheMobBlocks, readPt, readEmptyValueOnMobCellMiss);
mobKVCount++;
mobKVSize += mobCell.getValueLength();
outResult.set(i, mobCell);

63
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -40,6 +41,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
import org.apache.hadoop.hbase.io.hfile.TestHFile;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@ -68,6 +71,8 @@ public class TestMobStoreScanner {
private static HTableDescriptor desc;
private static Random random = new Random();
private static long defaultThreshold = 10;
private FileSystem fs;
private Configuration conf;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@ -84,6 +89,8 @@ public class TestMobStoreScanner {
}
public void setUp(long threshold, TableName tn) throws Exception {
conf = TEST_UTIL.getConfiguration();
fs = FileSystem.get(conf);
desc = new HTableDescriptor(tn);
hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
@ -196,6 +203,62 @@ public class TestMobStoreScanner {
Assert.assertEquals("value2", Bytes.toString(cell.getValue()));
}
@Test
public void testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss() throws Exception {
TableName tn = TableName.valueOf("testReadFromCorruptMobFilesWithReadEmptyValueOnMobCellMiss");
setUp(0, tn);
createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
Get get = new Get(row1);
get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true));
Result result = table.get(get);
Cell cell = result.getColumnLatestCell(family, qf1);
Assert.assertEquals(0, CellUtil.cloneValue(cell).length);
}
@Test
public void testReadFromCorruptMobFiles() throws Exception {
TableName tn = TableName.valueOf("testReadFromCorruptMobFiles");
setUp(0, tn);
createRecordAndCorruptMobFile(tn, row1, family, qf1, Bytes.toBytes("value1"));
Get get = new Get(row1);
IOException ioe = null;
try {
table.get(get);
} catch (IOException e) {
ioe = e;
}
Assert.assertNotNull(ioe);
Assert.assertEquals(CorruptHFileException.class.getName(), ioe.getClass().getName());
}
private void createRecordAndCorruptMobFile(TableName tn, byte[] row, byte[] family, byte[] qf,
byte[] value) throws IOException {
Put put1 = new Put(row);
put1.addColumn(family, qf, value);
table.put(put1);
admin.flush(tn);
Path mobFile = getFlushedMobFile(conf, fs, tn, Bytes.toString(family));
Assert.assertNotNull(mobFile);
// create new corrupt mob file.
Path corruptFile = new Path(mobFile.getParent(), "dummy");
TestHFile.truncateFile(fs, mobFile, corruptFile);
fs.delete(mobFile, true);
fs.rename(corruptFile, mobFile);
}
private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family)
throws IOException {
Path regionDir = MobUtils.getMobRegionPath(conf, table);
Path famDir = new Path(regionDir, family);
FileStatus[] hfFss = fs.listStatus(famDir);
for (FileStatus hfs : hfFss) {
if (!hfs.isDirectory()) {
return hfs.getPath();
}
}
return null;
}
private void testGetFromFiles(boolean reversed) throws Exception {
TableName tn = TableName.valueOf("testGetFromFiles" + reversed);
setUp(defaultThreshold, tn);