Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24021 Fail fast when bulkLoadHFiles method catch some IOException (#1343) 

Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
niuyulin 2020-04-02 23:15:14 +08:00 committed by GitHub
parent 4ddf55d20a
commit 0b3918fe56
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -179,7 +179,6 @@ import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.hadoop.hbase.wal.WALKeyImpl;
import org.apache.hadoop.hbase.wal.WALSplitUtil;
import org.apache.hadoop.hbase.wal.WALSplitUtil.MutationReplay;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.TraceScope;
import org.apache.yetus.audience.InterfaceAudience;
@ -6315,8 +6314,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* @throws IOException if failed unrecoverably.
*/
public Map<byte[], List<Path>> bulkLoadHFiles(Collection<Pair<byte[], String>> familyPaths,
boolean assignSeqId, BulkLoadListener bulkLoadListener,
boolean copyFile, List<String> clusterIds, boolean replicate) throws IOException {
boolean assignSeqId, BulkLoadListener bulkLoadListener, boolean copyFile,
List<String> clusterIds, boolean replicate) throws IOException {
long seqId = -1;
Map<byte[], List<Path>> storeFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);
Map<String, Long> storeFilesSizes = new HashMap<>();
@ -6328,9 +6327,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
this.writeRequestsCount.increment();
// There possibly was a split that happened between when the split keys
// were gathered and before the HRegion's write lock was taken. We need
// were gathered and before the HRegion's write lock was taken. We need
// to validate the HFile region before attempting to bulk load all of them
List<IOException> ioes = new ArrayList<>();
IOException ioException = null;
List<Pair<byte[], String>> failures = new ArrayList<>();
for (Pair<byte[], String> p : familyPaths) {
byte[] familyName = p.getFirst();
@ -6338,9 +6337,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
HStore store = getStore(familyName);
if (store == null) {
IOException ioe = new org.apache.hadoop.hbase.DoNotRetryIOException(
ioException = new org.apache.hadoop.hbase.DoNotRetryIOException(
"No such column family " + Bytes.toStringBinary(familyName));
ioes.add(ioe);
} else {
try {
store.assertBulkLoadHFileOk(new Path(path));
@ -6349,18 +6347,16 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
failures.add(p);
} catch (IOException ioe) {
// unrecoverable (hdfs problem)
ioes.add(ioe);
ioException = ioe;
}
}
}
// validation failed because of some sort of IO problem.
if (ioes.size() != 0) {
IOException e = MultipleIOException.createIOException(ioes);
LOG.error("There were one or more IO errors when checking if the bulk load is ok.", e);
throw e;
// validation failed because of some sort of IO problem.
if (ioException != null) {
LOG.error("There was IO error when checking if the bulk load is ok.", ioException);
throw ioException;
}
}
// validation failed, bail out before doing anything permanent.
if (failures.size() != 0) {
StringBuilder list = new StringBuilder();

17
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
View File

@ -95,6 +95,7 @@ public class TestBulkLoad {
private final byte[] randomBytes = new byte[100];
private final byte[] family1 = Bytes.toBytes("family1");
private final byte[] family2 = Bytes.toBytes("family2");
private final byte[] family3 = Bytes.toBytes("family3");
@Rule
public TestName name = new TestName();
@ -204,6 +205,13 @@ public class TestBulkLoad {
null);
}
// after HBASE-24021 will throw DoNotRetryIOException, not MultipleIOException
@Test(expected = DoNotRetryIOException.class)
public void shouldCrashIfBulkLoadMultiFamiliesNotInTable() throws IOException {
testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2, family3),
false, null);
}
@Test(expected = DoNotRetryIOException.class)
public void bulkHLogShouldThrowErrorWhenFamilySpecifiedAndHFileExistsButNotInTableDescriptor()
throws IOException {
@ -223,6 +231,15 @@ public class TestBulkLoad {
testRegionWithFamilies(family1).bulkLoadHFiles(list, false, null);
}
// after HBASE-24021 will throw FileNotFoundException, not MultipleIOException
@Test(expected = FileNotFoundException.class)
public void shouldThrowErrorIfMultiHFileDoesNotExist() throws IOException {
List<Pair<byte[], String>> list = new ArrayList<>();
list.addAll(asList(withMissingHFileForFamily(family1)));
list.addAll(asList(withMissingHFileForFamily(family2)));
testRegionWithFamilies(family1, family2).bulkLoadHFiles(list, false, null);
}
private Pair<byte[], String> withMissingHFileForFamily(byte[] family) {
return new Pair<>(family, getNotExistFilePath());
}