HBASE-3502 Remove the deletion limit in LogCleaner

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1067328 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2011-02-04 23:02:43 +00:00
parent f0e8ab83fd
commit 7a2e1fdd4c
3 changed files with 3 additions and 22 deletions

View File

@ -44,6 +44,7 @@ Release 0.91.0 - Unreleased
(Ed Kohlwey via Gary Helmling) (Ed Kohlwey via Gary Helmling)
HBASE-3502 Can't open region because can't open .regioninfo because HBASE-3502 Can't open region because can't open .regioninfo because
AlreadyBeingCreatedException AlreadyBeingCreatedException
HBASE-3501 Remove the deletion limit in LogCleaner
IMPROVEMENTS IMPROVEMENTS

View File

@ -37,16 +37,12 @@ import org.apache.hadoop.hbase.regionserver.wal.HLog;
import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
/** /**
* This Chore, everytime it runs, will clear the wal logs in the old logs folder * This Chore, everytime it runs, will clear the HLogs in the old logs folder
* that are deletable for each log cleaner in the chain, in order to limit the * that are deletable for each log cleaner in the chain.
* number of deletes it sends, will only delete maximum 20 in a single run.
*/ */
public class LogCleaner extends Chore { public class LogCleaner extends Chore {
static final Log LOG = LogFactory.getLog(LogCleaner.class.getName()); static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
// Max number we can delete on every chore, this is to make sure we don't
// issue thousands of delete commands around the same time
private final int maxDeletedLogs;
private final FileSystem fs; private final FileSystem fs;
private final Path oldLogDir; private final Path oldLogDir;
private List<LogCleanerDelegate> logCleanersChain; private List<LogCleanerDelegate> logCleanersChain;
@ -64,9 +60,6 @@ public class LogCleaner extends Chore {
Configuration conf, FileSystem fs, Configuration conf, FileSystem fs,
Path oldLogDir) { Path oldLogDir) {
super("LogsCleaner", p, s); super("LogsCleaner", p, s);
this.maxDeletedLogs =
conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20);
this.fs = fs; this.fs = fs;
this.oldLogDir = oldLogDir; this.oldLogDir = oldLogDir;
this.conf = conf; this.conf = conf;
@ -127,7 +120,6 @@ public class LogCleaner extends Chore {
try { try {
FileStatus [] files = this.fs.listStatus(this.oldLogDir); FileStatus [] files = this.fs.listStatus(this.oldLogDir);
if (files == null) return; if (files == null) return;
int nbDeletedLog = 0;
FILE: for (FileStatus file : files) { FILE: for (FileStatus file : files) {
Path filePath = file.getPath(); Path filePath = file.getPath();
if (HLog.validateHLogFilename(filePath.getName())) { if (HLog.validateHLogFilename(filePath.getName())) {
@ -144,15 +136,10 @@ public class LogCleaner extends Chore {
} }
// delete this log file if it passes all the log cleaners // delete this log file if it passes all the log cleaners
this.fs.delete(filePath, true); this.fs.delete(filePath, true);
nbDeletedLog++;
} else { } else {
LOG.warn("Found a wrongly formated file: " LOG.warn("Found a wrongly formated file: "
+ file.getPath().getName()); + file.getPath().getName());
this.fs.delete(filePath, true); this.fs.delete(filePath, true);
nbDeletedLog++;
}
if (nbDeletedLog >= maxDeletedLogs) {
break;
} }
} }
} catch (IOException e) { } catch (IOException e) {

View File

@ -112,13 +112,6 @@ public class TestLogsCleaner {
assertEquals(34, fs.listStatus(oldLogDir).length); assertEquals(34, fs.listStatus(oldLogDir).length);
// This will take care of 20 old log files (default max we can delete)
cleaner.chore();
assertEquals(14, fs.listStatus(oldLogDir).length);
// We will delete all remaining log files which are not scheduled for
// replication and those that are invalid
cleaner.chore(); cleaner.chore();
// We end up with the current log file, a newer one and the 3 old log // We end up with the current log file, a newer one and the 3 old log