Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20729: B & R BackupLogCleaner must ignore ProcV2 WAL files 

Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Vladimir Rodionov 2018-08-01 11:16:08 -07:00 committed by tedyu
parent 1d0fca370b
commit d53a976e8d
3 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java
View File

@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
@ -1116,6 +1117,11 @@ public final class BackupSystemTable implements Closeable {
List<FileStatus> fileStatuses = new ArrayList<>(); List<FileStatus> fileStatuses = new ArrayList<>();
for (FileStatus file : files) { for (FileStatus file : files) {
String fn = file.getPath().getName();
if (fn.startsWith(WALProcedureStore.LOG_PREFIX)) {
ret.put(file, true);
continue;
}
String wal = file.getPath().toString(); String wal = file.getPath().toString();
Get get = createGetForCheckWALFile(wal); Get get = createGetForCheckWALFile(wal);
getBuffer.add(get); getBuffer.add(get);

10
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
View File

@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
@ -101,7 +102,7 @@ public class IncrementalBackupManager extends BackupManager {
List<WALItem> logFromSystemTable = List<WALItem> logFromSystemTable =
getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo() getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo()
.getBackupRootDir()); .getBackupRootDir());
logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable); logList = excludeAlreadyBackedUpAndProcV2WALs(logList, logFromSystemTable);
backupInfo.setIncrBackupFileList(logList); backupInfo.setIncrBackupFileList(logList);
return newTimestamps; return newTimestamps;
@ -144,14 +145,13 @@ public class IncrementalBackupManager extends BackupManager {
getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo() getLogFilesFromBackupSystem(previousTimestampMins, newTimestamps, getBackupInfo()
.getBackupRootDir()); .getBackupRootDir());
logList = excludeAlreadyBackedUpWALs(logList, logFromSystemTable); logList = excludeAlreadyBackedUpAndProcV2WALs(logList, logFromSystemTable);
backupInfo.setIncrBackupFileList(logList); backupInfo.setIncrBackupFileList(logList);
return logList; return logList;
} }
private List<String> excludeAlreadyBackedUpAndProcV2WALs(List<String> logList,
private List<String> excludeAlreadyBackedUpWALs(List<String> logList,
List<WALItem> logFromSystemTable) { List<WALItem> logFromSystemTable) {
Set<String> walFileNameSet = convertToSet(logFromSystemTable); Set<String> walFileNameSet = convertToSet(logFromSystemTable);
@ -160,7 +160,7 @@ public class IncrementalBackupManager extends BackupManager {
Path p = new Path(logList.get(i)); Path p = new Path(logList.get(i));
String name = p.getName(); String name = p.getName();
if (walFileNameSet.contains(name)) { if (walFileNameSet.contains(name) || name.startsWith(WALProcedureStore.LOG_PREFIX)) {
continue; continue;
} }

1
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
View File

@ -97,7 +97,6 @@ public class BackupLogCleaner extends BaseLogCleanerDelegate {
LOG.warn("Backup system table is not available: {}", tnfe.getMessage()); LOG.warn("Backup system table is not available: {}", tnfe.getMessage());
return files; return files;
} }
List<FileStatus> list = new ArrayList<>(); List<FileStatus> list = new ArrayList<>();
Map<FileStatus, Boolean> walFilesDeletableMap = table.areWALFilesDeletable(files); Map<FileStatus, Boolean> walFilesDeletableMap = table.areWALFilesDeletable(files);
for (Map.Entry<FileStatus, Boolean> entry: walFilesDeletableMap.entrySet()) { for (Map.Entry<FileStatus, Boolean> entry: walFilesDeletableMap.entrySet()) {