HBASE-3502 Remove the deletion limit in LogCleaner
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1067328 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
f0e8ab83fd
commit
7a2e1fdd4c
|
@ -44,6 +44,7 @@ Release 0.91.0 - Unreleased
|
||||||
(Ed Kohlwey via Gary Helmling)
|
(Ed Kohlwey via Gary Helmling)
|
||||||
HBASE-3502 Can't open region because can't open .regioninfo because
|
HBASE-3502 Can't open region because can't open .regioninfo because
|
||||||
AlreadyBeingCreatedException
|
AlreadyBeingCreatedException
|
||||||
|
HBASE-3501 Remove the deletion limit in LogCleaner
|
||||||
|
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
|
@ -37,16 +37,12 @@ import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
||||||
import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
|
import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This Chore, everytime it runs, will clear the wal logs in the old logs folder
|
* This Chore, everytime it runs, will clear the HLogs in the old logs folder
|
||||||
* that are deletable for each log cleaner in the chain, in order to limit the
|
* that are deletable for each log cleaner in the chain.
|
||||||
* number of deletes it sends, will only delete maximum 20 in a single run.
|
|
||||||
*/
|
*/
|
||||||
public class LogCleaner extends Chore {
|
public class LogCleaner extends Chore {
|
||||||
static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
|
static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
|
||||||
|
|
||||||
// Max number we can delete on every chore, this is to make sure we don't
|
|
||||||
// issue thousands of delete commands around the same time
|
|
||||||
private final int maxDeletedLogs;
|
|
||||||
private final FileSystem fs;
|
private final FileSystem fs;
|
||||||
private final Path oldLogDir;
|
private final Path oldLogDir;
|
||||||
private List<LogCleanerDelegate> logCleanersChain;
|
private List<LogCleanerDelegate> logCleanersChain;
|
||||||
|
@ -64,9 +60,6 @@ public class LogCleaner extends Chore {
|
||||||
Configuration conf, FileSystem fs,
|
Configuration conf, FileSystem fs,
|
||||||
Path oldLogDir) {
|
Path oldLogDir) {
|
||||||
super("LogsCleaner", p, s);
|
super("LogsCleaner", p, s);
|
||||||
|
|
||||||
this.maxDeletedLogs =
|
|
||||||
conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20);
|
|
||||||
this.fs = fs;
|
this.fs = fs;
|
||||||
this.oldLogDir = oldLogDir;
|
this.oldLogDir = oldLogDir;
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
|
@ -127,7 +120,6 @@ public class LogCleaner extends Chore {
|
||||||
try {
|
try {
|
||||||
FileStatus [] files = this.fs.listStatus(this.oldLogDir);
|
FileStatus [] files = this.fs.listStatus(this.oldLogDir);
|
||||||
if (files == null) return;
|
if (files == null) return;
|
||||||
int nbDeletedLog = 0;
|
|
||||||
FILE: for (FileStatus file : files) {
|
FILE: for (FileStatus file : files) {
|
||||||
Path filePath = file.getPath();
|
Path filePath = file.getPath();
|
||||||
if (HLog.validateHLogFilename(filePath.getName())) {
|
if (HLog.validateHLogFilename(filePath.getName())) {
|
||||||
|
@ -144,15 +136,10 @@ public class LogCleaner extends Chore {
|
||||||
}
|
}
|
||||||
// delete this log file if it passes all the log cleaners
|
// delete this log file if it passes all the log cleaners
|
||||||
this.fs.delete(filePath, true);
|
this.fs.delete(filePath, true);
|
||||||
nbDeletedLog++;
|
|
||||||
} else {
|
} else {
|
||||||
LOG.warn("Found a wrongly formated file: "
|
LOG.warn("Found a wrongly formated file: "
|
||||||
+ file.getPath().getName());
|
+ file.getPath().getName());
|
||||||
this.fs.delete(filePath, true);
|
this.fs.delete(filePath, true);
|
||||||
nbDeletedLog++;
|
|
||||||
}
|
|
||||||
if (nbDeletedLog >= maxDeletedLogs) {
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
|
|
@ -112,13 +112,6 @@ public class TestLogsCleaner {
|
||||||
|
|
||||||
assertEquals(34, fs.listStatus(oldLogDir).length);
|
assertEquals(34, fs.listStatus(oldLogDir).length);
|
||||||
|
|
||||||
// This will take care of 20 old log files (default max we can delete)
|
|
||||||
cleaner.chore();
|
|
||||||
|
|
||||||
assertEquals(14, fs.listStatus(oldLogDir).length);
|
|
||||||
|
|
||||||
// We will delete all remaining log files which are not scheduled for
|
|
||||||
// replication and those that are invalid
|
|
||||||
cleaner.chore();
|
cleaner.chore();
|
||||||
|
|
||||||
// We end up with the current log file, a newer one and the 3 old log
|
// We end up with the current log file, a newer one and the 3 old log
|
||||||
|
|
Loading…
Reference in New Issue