Amend HBASE-18786 FileNotFoundException should not be silently handled for primary region replicas
Remove now invalid unit test TestCorruptedRegionStoreFile
This commit is contained in:
Andrew Purtell
parent
bb81e9f3ca
commit
fa0b0934db
|
|
@ -1,266 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.hadoop.hbase.regionserver;
|
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.fs.FSDataInputStream;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
|
||||||
import org.apache.hadoop.hbase.TableName;
|
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
|
||||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
|
||||||
import org.apache.hadoop.hbase.client.Durability;
|
|
||||||
import org.apache.hadoop.hbase.client.Put;
|
|
||||||
import org.apache.hadoop.hbase.client.Table;
|
|
||||||
import org.apache.hadoop.hbase.io.HFileLink;
|
|
||||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
|
||||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
|
||||||
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
|
||||||
import org.apache.hadoop.hbase.util.FSUtils;
|
|
||||||
import org.apache.hadoop.hbase.util.FSVisitor;
|
|
||||||
import org.apache.hadoop.hbase.TestTableName;
|
|
||||||
|
|
||||||
import org.junit.After;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Ignore;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.experimental.categories.Category;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* HBASE-13651 Handle StoreFileScanner FileNotFoundException
|
|
||||||
*/
|
|
||||||
@Ignore
|
|
||||||
@Category({MasterTests.class, LargeTests.class})
|
|
||||||
public class TestCorruptedRegionStoreFile {
|
|
||||||
private static final Log LOG = LogFactory.getLog(TestCorruptedRegionStoreFile.class);
|
|
||||||
|
|
||||||
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
|
||||||
|
|
||||||
private static final String FAMILY_NAME_STR = "f";
|
|
||||||
private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
|
|
||||||
|
|
||||||
private static final int NUM_FILES = 10;
|
|
||||||
private static final int ROW_PER_FILE = 2000;
|
|
||||||
private static final int NUM_ROWS = NUM_FILES * ROW_PER_FILE;
|
|
||||||
|
|
||||||
@Rule public TestTableName TEST_TABLE = new TestTableName();
|
|
||||||
|
|
||||||
private final ArrayList<Path> storeFiles = new ArrayList<>();
|
|
||||||
private Path tableDir;
|
|
||||||
private int rowCount;
|
|
||||||
|
|
||||||
private static void setupConf(Configuration conf) {
|
|
||||||
// Disable compaction so the store file count stays constant
|
|
||||||
conf.setLong("hbase.hstore.compactionThreshold", NUM_FILES + 1);
|
|
||||||
conf.setLong("hbase.hstore.blockingStoreFiles", NUM_FILES * 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setupTable(final TableName tableName) throws IOException {
|
|
||||||
// load the table
|
|
||||||
Table table = UTIL.createTable(tableName, FAMILY_NAME);
|
|
||||||
try {
|
|
||||||
rowCount = 0;
|
|
||||||
byte[] value = new byte[1024];
|
|
||||||
byte[] q = Bytes.toBytes("q");
|
|
||||||
while (rowCount < NUM_ROWS) {
|
|
||||||
Put put = new Put(Bytes.toBytes(String.format("%010d", rowCount)));
|
|
||||||
put.setDurability(Durability.SKIP_WAL);
|
|
||||||
put.addColumn(FAMILY_NAME, q, value);
|
|
||||||
table.put(put);
|
|
||||||
|
|
||||||
if ((rowCount++ % ROW_PER_FILE) == 0) {
|
|
||||||
UTIL.getAdmin().flush(tableName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
UTIL.getAdmin().flush(tableName);
|
|
||||||
table.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
assertEquals(NUM_ROWS, rowCount);
|
|
||||||
|
|
||||||
// get the store file paths
|
|
||||||
storeFiles.clear();
|
|
||||||
tableDir = FSUtils.getTableDir(getRootDir(), tableName);
|
|
||||||
FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {
|
|
||||||
@Override
|
|
||||||
public void storeFile(final String region, final String family, final String hfile)
|
|
||||||
throws IOException {
|
|
||||||
HFileLink link = HFileLink.build(UTIL.getConfiguration(), tableName, region, family, hfile);
|
|
||||||
storeFiles.add(link.getOriginPath());
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertTrue("Expected at least " + NUM_FILES + " store files", storeFiles.size() >= NUM_FILES);
|
|
||||||
LOG.info("Store files: " + storeFiles);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void setup() throws Exception {
|
|
||||||
setupConf(UTIL.getConfiguration());
|
|
||||||
UTIL.startMiniCluster(2, 3);
|
|
||||||
|
|
||||||
setupTable(TEST_TABLE.getTableName());
|
|
||||||
}
|
|
||||||
|
|
||||||
@After
|
|
||||||
public void tearDown() throws Exception {
|
|
||||||
try {
|
|
||||||
UTIL.shutdownMiniCluster();
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOG.warn("failure shutting down cluster", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void removeStoreFile(FileSystem fs, Path tmpStoreFilePath) throws Exception {
|
|
||||||
try (FSDataInputStream input = fs.open(storeFiles.get(0))) {
|
|
||||||
fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
|
|
||||||
LOG.info("Move file to local");
|
|
||||||
evictHFileCache(storeFiles.get(0));
|
|
||||||
// make sure that all the replicas have been deleted on DNs.
|
|
||||||
for (;;) {
|
|
||||||
try {
|
|
||||||
input.read(0, new byte[1], 0, 1);
|
|
||||||
} catch (FileNotFoundException e) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Thread.sleep(1000);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(timeout=180000)
|
|
||||||
public void testLosingFileDuringScan() throws Exception {
|
|
||||||
assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
|
|
||||||
|
|
||||||
final FileSystem fs = getFileSystem();
|
|
||||||
final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
|
|
||||||
|
|
||||||
// try to query with the missing file
|
|
||||||
int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
|
|
||||||
private boolean hasFile = true;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void beforeScanNext(Table table) throws Exception {
|
|
||||||
// move the path away (now the region is corrupted)
|
|
||||||
if (hasFile) {
|
|
||||||
removeStoreFile(fs, tmpStoreFilePath);
|
|
||||||
hasFile = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
|
|
||||||
count >= (NUM_ROWS - ROW_PER_FILE));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test(timeout=180000)
|
|
||||||
public void testLosingFileAfterScannerInit() throws Exception {
|
|
||||||
assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
|
|
||||||
|
|
||||||
final FileSystem fs = getFileSystem();
|
|
||||||
final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
|
|
||||||
|
|
||||||
// try to query with the missing file
|
|
||||||
int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
|
|
||||||
private boolean hasFile = true;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void beforeScan(Table table, Scan scan) throws Exception {
|
|
||||||
// move the path away (now the region is corrupted)
|
|
||||||
if (hasFile) {
|
|
||||||
removeStoreFile(fs, tmpStoreFilePath);
|
|
||||||
hasFile = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
|
|
||||||
count >= (NUM_ROWS - ROW_PER_FILE));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==========================================================================
|
|
||||||
// Helpers
|
|
||||||
// ==========================================================================
|
|
||||||
private FileSystem getFileSystem() {
|
|
||||||
return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path getRootDir() {
|
|
||||||
return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void evictHFileCache(final Path hfile) throws Exception {
|
|
||||||
for (RegionServerThread rst: UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
|
|
||||||
HRegionServer rs = rst.getRegionServer();
|
|
||||||
rs.getCacheConfig().getBlockCache().evictBlocksByHfileName(hfile.getName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private int fullScanAndCount(final TableName tableName) throws Exception {
|
|
||||||
return fullScanAndCount(tableName, new ScanInjector());
|
|
||||||
}
|
|
||||||
|
|
||||||
private int fullScanAndCount(final TableName tableName, final ScanInjector injector)
|
|
||||||
throws Exception {
|
|
||||||
Table table = UTIL.getConnection().getTable(tableName);
|
|
||||||
int count = 0;
|
|
||||||
try {
|
|
||||||
Scan scan = new Scan();
|
|
||||||
scan.setCaching(1);
|
|
||||||
scan.setCacheBlocks(false);
|
|
||||||
injector.beforeScan(table, scan);
|
|
||||||
ResultScanner scanner = table.getScanner(scan);
|
|
||||||
try {
|
|
||||||
while (true) {
|
|
||||||
injector.beforeScanNext(table);
|
|
||||||
Result result = scanner.next();
|
|
||||||
injector.afterScanNext(table, result);
|
|
||||||
if (result == null) break;
|
|
||||||
if ((count++ % (ROW_PER_FILE / 2)) == 0) {
|
|
||||||
LOG.debug("scan next " + count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
scanner.close();
|
|
||||||
injector.afterScan(table);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
table.close();
|
|
||||||
}
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
private class ScanInjector {
|
|
||||||
protected void beforeScan(Table table, Scan scan) throws Exception {}
|
|
||||||
protected void beforeScanNext(Table table) throws Exception {}
|
|
||||||
protected void afterScanNext(Table table, Result result) throws Exception {}
|
|
||||||
protected void afterScan(Table table) throws Exception {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue