HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
This commit is contained in:
parent
eafc07a06d
commit
621b33f2f9
|
@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
|
|||
}
|
||||
Path familyDir = stat.getPath();
|
||||
byte[] family = familyDir.getName().getBytes();
|
||||
Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
|
||||
for (Path hfile : hfiles) {
|
||||
FileStatus[] hfileStatuses = fs.listStatus(familyDir);
|
||||
for (FileStatus hfileStatus : hfileStatuses) {
|
||||
long length = hfileStatus.getLen();
|
||||
Path hfile = hfileStatus.getPath();
|
||||
// Skip "_", reference, HFileLink
|
||||
String fileName = hfile.getName();
|
||||
if (fileName.startsWith("_")) continue;
|
||||
|
@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
|
|||
LOG.warn("Skipping HFileLink " + fileName);
|
||||
continue;
|
||||
}
|
||||
if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
|
||||
HConstants.DEFAULT_MAX_FILE_SIZE)) {
|
||||
LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
|
||||
length + " bytes can be problematic as it may lead to oversplitting.");
|
||||
}
|
||||
ret.add(new LoadQueueItem(family, hfile));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -709,6 +709,12 @@ public class HStore implements Store {
|
|||
+ this.getRegionInfo().getRegionNameAsString());
|
||||
}
|
||||
|
||||
if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
|
||||
HConstants.DEFAULT_MAX_FILE_SIZE)) {
|
||||
LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
|
||||
reader.length() + " bytes can be problematic as it may lead to oversplitting.");
|
||||
}
|
||||
|
||||
if (verifyBulkLoads) {
|
||||
long verificationStartTime = EnvironmentEdgeManager.currentTime();
|
||||
LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
|
||||
|
|
Loading…
Reference in New Issue