diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java index 30b5be3dcd6..cc0d0aaf28e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master; +import java.io.IOException; import java.util.Map; import org.apache.commons.logging.Log; @@ -27,8 +28,11 @@ import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; +import org.apache.hadoop.hbase.exceptions.LockTimeoutException; +import org.apache.hadoop.hbase.master.TableLockManager.TableLock; import org.apache.hadoop.hbase.mob.ExpiredMobFileCleaner; import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.mob.MobUtils; /** * The Class ExpiredMobFileCleanerChore for running cleaner regularly to remove the expired @@ -39,12 +43,14 @@ public class ExpiredMobFileCleanerChore extends Chore { private static final Log LOG = LogFactory.getLog(ExpiredMobFileCleanerChore.class); private final HMaster master; + private TableLockManager tableLockManager; private ExpiredMobFileCleaner cleaner; public ExpiredMobFileCleanerChore(HMaster master) { super(master.getServerName() + "-ExpiredMobFileCleanerChore", master.getConfiguration().getInt( MobConstants.MOB_CLEANER_PERIOD, MobConstants.DEFAULT_MOB_CLEANER_PERIOD), master); this.master = master; + this.tableLockManager = master.getTableLockManager(); cleaner = new ExpiredMobFileCleaner(); } @@ -56,7 +62,36 @@ public class ExpiredMobFileCleanerChore extends Chore { for (HTableDescriptor htd : map.values()) { for (HColumnDescriptor hcd : htd.getColumnFamilies()) { if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) { - cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd); + // clean only for mob-enabled column. + // obtain a read table lock before cleaning, synchronize with MobFileCompactionChore. + boolean tableLocked = false; + TableLock lock = null; + try { + // the tableLockManager might be null in testing. In that case, it is lock-free. + if (tableLockManager != null) { + lock = tableLockManager.readLock(MobUtils.getTableLockName(htd.getTableName()), + "Run ExpiredMobFileCleanerChore"); + lock.acquire(); + } + tableLocked = true; + cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd); + } catch (LockTimeoutException e) { + LOG.info("Fail to acquire the lock because of timeout, maybe a" + + " MobFileCompactor is running", e); + } catch (Exception e) { + LOG.error( + "Fail to clean the expired mob files for the column " + hcd.getNameAsString() + + " in the table " + htd.getNameAsString(), e); + } finally { + if (lock != null && tableLocked) { + try { + lock.release(); + } catch (IOException e) { + LOG.error( + "Fail to release the write lock for the table " + htd.getNameAsString(), e); + } + } + } } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java index 9973619f320..12491afee32 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.exceptions.LockTimeoutException; import org.apache.hadoop.hbase.master.TableLockManager.TableLock; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; @@ -103,6 +104,9 @@ public class MobFileCompactionChore extends Chore{ } tableLocked = true; compactor.compact(); + } catch (LockTimeoutException e) { + LOG.info("Fail to acquire the lock because of timeout, maybe a major compaction or an" + + " ExpiredMobFileCleanerChore is running", e); } catch (Exception e) { LOG.error("Fail to compact the mob files for the column " + hcd.getNameAsString() + " in the table " + htd.getNameAsString(), e);