From f62c8201b62eb440ac0eeb7f61229ba0a343af6d Mon Sep 17 00:00:00 2001 From: Vladimir Rodionov Date: Tue, 21 Aug 2018 12:35:00 -0700 Subject: [PATCH] HBASE-21077 MR job launched by hbase incremental backup command failed with FileNotFoundException Signed-off-by: tedyu --- .../hadoop/hbase/backup/impl/IncrementalBackupManager.java | 2 +- .../org/apache/hadoop/hbase/backup/TestBackupBase.java | 7 +++++++ .../apache/hadoop/hbase/backup/TestIncrementalBackup.java | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index 81652d8ae25..3eebf4299c3 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -178,7 +178,7 @@ public class IncrementalBackupManager extends BackupManager { Set set = new HashSet<>(); for (int i=0; i < logFromSystemTable.size(); i++) { WALItem item = logFromSystemTable.get(i); - set.add(item.walFile); + set.add((new Path(item.walFile)).getName()); } return set; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 08ecd632c62..94c2271a7d1 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -57,6 +57,8 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.cleaner.LogCleaner; +import org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner; import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.access.SecureTestUtil; @@ -288,6 +290,11 @@ public class TestBackupBase { BackupManager.decorateMasterConfiguration(conf1); BackupManager.decorateRegionServerConfiguration(conf1); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + // Set TTL for old WALs to 1 sec to enforce fast cleaning of an archived + // WAL files + conf1.setLong(TimeToLiveLogCleaner.TTL_CONF_KEY, 1000); + conf1.setLong(LogCleaner.OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC, 1000); + // Set MultiWAL (with 2 default WAL files per RS) conf1.set(WALFactory.WAL_PROVIDER, provider); TEST_UTIL.startMiniCluster(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java index b74f42fb839..48e2c5eb8b0 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java @@ -157,6 +157,8 @@ public class TestIncrementalBackup extends TestBackupBase { int NB_ROWS_FAM2 = 7; HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2); t3.close(); + // Wait for 5 sec to make sure that old WALs were deleted + Thread.sleep(5000); // #3 - incremental backup for multiple tables request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);