Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)

This commit is contained in:
Matteo Bertozzi 2015-01-15 23:13:47 +00:00
parent eafc07a06d
commit 621b33f2f9
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
View File

@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
} }
Path familyDir = stat.getPath(); Path familyDir = stat.getPath();
byte[] family = familyDir.getName().getBytes(); byte[] family = familyDir.getName().getBytes();
Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); FileStatus[] hfileStatuses = fs.listStatus(familyDir);
for (Path hfile : hfiles) { for (FileStatus hfileStatus : hfileStatuses) {
long length = hfileStatus.getLen();
Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink // Skip "_", reference, HFileLink
String fileName = hfile.getName(); String fileName = hfile.getName();
if (fileName.startsWith("_")) continue; if (fileName.startsWith("_")) continue;
@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LOG.warn("Skipping HFileLink " + fileName); LOG.warn("Skipping HFileLink " + fileName);
continue; continue;
} }
if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
HConstants.DEFAULT_MAX_FILE_SIZE)) {
LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
length + " bytes can be problematic as it may lead to oversplitting.");
}
ret.add(new LoadQueueItem(family, hfile)); ret.add(new LoadQueueItem(family, hfile));
} }
} }

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -709,6 +709,12 @@ public class HStore implements Store {
+ this.getRegionInfo().getRegionNameAsString()); + this.getRegionInfo().getRegionNameAsString());
} }
if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
HConstants.DEFAULT_MAX_FILE_SIZE)) {
LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
reader.length() + " bytes can be problematic as it may lead to oversplitting.");
}
if (verifyBulkLoads) { if (verifyBulkLoads) {
long verificationStartTime = EnvironmentEdgeManager.currentTime(); long verificationStartTime = EnvironmentEdgeManager.currentTime();
LOG.info("Full verification started for bulk load hfile: " + srcPath.toString()); LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());