Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25186 TestMasterRegionOnTwoFileSystems is failing after HBASE-25065 (#2544) 

Signed-off-by: Ramkrishna <ramkrishna@apache.org>
This commit is contained in:
Duo Zhang 2020-10-15 10:22:14 +08:00
parent 4e6c52c319
commit 2f566dc234
3 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionWALRoller.java
View File

@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.AbstractWALRoller; import org.apache.hadoop.hbase.wal.AbstractWALRoller;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -68,8 +67,11 @@ public final class MasterRegionWALRoller extends AbstractWALRoller<Abortable> {
} }
@Override @Override
protected void afterRoll(WAL wal) { protected void afterWALArchive(Path oldPath, Path newPath) {
// move the archived WAL files to the global archive path // move the archived WAL files to the global archive path
// here we do not use the newPath directly, so that even if we fail to move some of the
// newPaths, we are still safe because every time we will get all the files under the archive
// directory.
try { try {
MasterRegionUtils.moveFilesUnderDir(fs, walArchiveDir, globalWALArchiveDir, MasterRegionUtils.moveFilesUnderDir(fs, walArchiveDir, globalWALArchiveDir,
archivedWALSuffix); archivedWALSuffix);

12
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
View File

@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
@ -85,6 +86,11 @@ public abstract class AbstractWALRoller<T extends Abortable> extends Thread
AbstractWALRoller.this.notifyAll(); AbstractWALRoller.this.notifyAll();
} }
} }
@Override
public void postLogArchive(Path oldPath, Path newPath) throws IOException {
afterWALArchive(oldPath, newPath);
}
}); });
} }
} }
@ -184,7 +190,6 @@ public abstract class AbstractWALRoller<T extends Abortable> extends Thread
scheduleFlush(Bytes.toString(r.getKey()), r.getValue()); scheduleFlush(Bytes.toString(r.getKey()), r.getValue());
} }
} }
afterRoll(wal);
} }
} catch (FailedLogCloseException | ConnectException e) { } catch (FailedLogCloseException | ConnectException e) {
abort("Failed log close in log roller", e); abort("Failed log close in log roller", e);
@ -200,10 +205,7 @@ public abstract class AbstractWALRoller<T extends Abortable> extends Thread
LOG.info("LogRoller exiting."); LOG.info("LogRoller exiting.");
} }
/** protected void afterWALArchive(Path oldPath, Path newPath) {
* Called after we finish rolling the give {@code wal}.
*/
protected void afterRoll(WAL wal) {
} }
/** /**

8
hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionWALCleaner.java
View File

@ -86,8 +86,12 @@ public class TestMasterRegionWALCleaner extends MasterRegionTestBase {
region.requestRollAll(); region.requestRollAll();
region.waitUntilWalRollFinished(); region.waitUntilWalRollFinished();
// should have one // should have one
FileStatus[] files = fs.listStatus(globalWALArchiveDir);
Thread.sleep(9000); assertEquals(1, files.length); Thread.sleep(9000);
Thread.sleep(2000);
// should still be there
assertTrue(fs.exists(files[0].getPath()));
Thread.sleep(6000);
// should have been cleaned // should have been cleaned
assertEquals(0, fs.listStatus(globalWALArchiveDir).length); assertEquals(0, fs.listStatus(globalWALArchiveDir).length);
} }