From 52b4db860780fea6cf5ec0dabb34a581090a535d Mon Sep 17 00:00:00 2001 From: Matteo Bertozzi Date: Thu, 15 Jan 2015 23:13:47 +0000 Subject: [PATCH] HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu) --- .../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 11 +++++++++-- .../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java index b6e8d9c9db5..9903fe9ef2c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java @@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool { } Path familyDir = stat.getPath(); byte[] family = familyDir.getName().getBytes(); - Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); - for (Path hfile : hfiles) { + FileStatus[] hfileStatuses = fs.listStatus(familyDir); + for (FileStatus hfileStatus : hfileStatuses) { + long length = hfileStatus.getLen(); + Path hfile = hfileStatus.getPath(); // Skip "_", reference, HFileLink String fileName = hfile.getName(); if (fileName.startsWith("_")) continue; @@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool { LOG.warn("Skipping HFileLink " + fileName); continue; } + if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE)) { + LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " + + length + " bytes can be problematic as it may lead to oversplitting."); + } ret.add(new LoadQueueItem(family, hfile)); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index ad701b7e880..6a65038b813 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -709,6 +709,12 @@ public class HStore implements Store { + this.getRegionInfo().getRegionNameAsString()); } + if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE)) { + LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " + + reader.length() + " bytes can be problematic as it may lead to oversplitting."); + } + if (verifyBulkLoads) { long verificationStartTime = EnvironmentEdgeManager.currentTime(); LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());