Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir

This commit is contained in:
Matteo Bertozzi 2015-03-13 17:11:27 +00:00
parent 0f69ea409f
commit d2d4ef14f4
2 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
View File

@ -209,6 +209,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
byte[] family = familyDir.getName().getBytes();
FileStatus[] hfileStatuses = fs.listStatus(familyDir);
for (FileStatus hfileStatus : hfileStatuses) {
if (!hfileStatus.isFile()) {
LOG.warn("Skipping non-file " + hfileStatus);
continue;
}
long length = hfileStatus.getLen();
Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
@ -293,10 +297,18 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LoadQueueItem lqi = queueIter.next();
String familyNameInHFile = Bytes.toString(lqi.family);
if (!familyNames.contains(familyNameInHFile)) {
if (HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath)) {
boolean isValid = false;
try {
isValid = HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath);
if (!isValid) {
LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
}
} catch (FileNotFoundException e) {
LOG.warn("the file " + lqi + " was removed");
}
if (isValid) {
unmatchedFamilies.add(familyNameInHFile);
} else {
LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
queueIter.remove();
}
}

6
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
View File

@ -308,8 +308,9 @@ public class TestLoadIncrementalHFiles {
}
/**
* Write a random data file in a dir with a valid family name but not part of the table families
* we should we able to bulkload without getting the unmatched family exception. HBASE-13037
* Write a random data file and a non-file in a dir with a valid family name
* but not part of the table families. we should we able to bulkload without
* getting the unmatched family exception. HBASE-13037/HBASE-13227
*/
@Test(timeout = 60000)
public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
@ -324,6 +325,7 @@ public class TestLoadIncrementalHFiles {
final String NON_FAMILY_FOLDER = "_logs";
Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
fs.mkdirs(nonFamilyDir);
fs.mkdirs(new Path(nonFamilyDir, "non-file"));
createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
Table table = null;