Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22200 - WALSplitter.hasRecoveredEdits should use same FS instance from WAL region dir 

Signed-off-by: Zach York <zyork@apache.org>
This commit is contained in:
Wellington Chevreuil 2019-04-10 19:56:39 +01:00 committed by Zach York
parent e9db0b03f2
commit 02fd482630
3 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java
View File

@ -229,8 +229,7 @@ final class AssignmentManagerUtil {
} }
static void checkClosedRegion(MasterProcedureEnv env, RegionInfo regionInfo) throws IOException { static void checkClosedRegion(MasterProcedureEnv env, RegionInfo regionInfo) throws IOException {
if (WALSplitter.hasRecoveredEdits(env.getMasterServices().getFileSystem(), if (WALSplitter.hasRecoveredEdits(env.getMasterConfiguration(), regionInfo)) {
env.getMasterConfiguration(), regionInfo)) {
throw new IOException("Recovered.edits are found in Region: " + regionInfo + throw new IOException("Recovered.edits are found in Region: " + regionInfo +
", abort split/merge to prevent data loss"); ", abort split/merge to prevent data loss");
} }

7
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
View File

@ -538,14 +538,13 @@ public class WALSplitter {
/** /**
* Check whether there is recovered.edits in the region dir * Check whether there is recovered.edits in the region dir
* @param walFS FileSystem
* @param conf conf * @param conf conf
* @param regionInfo the region to check * @param regionInfo the region to check
* @throws IOException IOException * @throws IOException IOException
* @return true if recovered.edits exist in the region dir * @return true if recovered.edits exist in the region dir
*/ */
public static boolean hasRecoveredEdits(final FileSystem walFS, public static boolean hasRecoveredEdits(final Configuration conf,
final Configuration conf, final RegionInfo regionInfo) throws IOException { final RegionInfo regionInfo) throws IOException {
// No recovered.edits for non default replica regions // No recovered.edits for non default replica regions
if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
return false; return false;
@ -554,7 +553,7 @@ public class WALSplitter {
//directly without converting it to default replica's regioninfo. //directly without converting it to default replica's regioninfo.
Path regionDir = FSUtils.getWALRegionDir(conf, regionInfo.getTable(), Path regionDir = FSUtils.getWALRegionDir(conf, regionInfo.getTable(),
regionInfo.getEncodedName()); regionInfo.getEncodedName());
NavigableSet<Path> files = getSplitEditFilesSorted(walFS, regionDir); NavigableSet<Path> files = getSplitEditFilesSorted(FSUtils.getWALFileSystem(conf), regionDir);
return files != null && !files.isEmpty(); return files != null && !files.isEmpty();
} }

42
hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
View File

@ -383,18 +383,7 @@ public class TestWALSplit {
*/ */
@Test @Test
public void testRecoveredEditsPathForMeta() throws IOException { public void testRecoveredEditsPathForMeta() throws IOException {
byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); Path p = createRecoveredEditsPathForRegion();
Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);
Path regiondir = new Path(tdir,
RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
fs.mkdirs(regiondir);
long now = System.currentTimeMillis();
Entry entry =
new Entry(new WALKeyImpl(encoded,
TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
new WALEdit());
Path p = WALSplitter.getRegionSplitEditsPath(entry,
FILENAME_BEING_SPLIT, TMPDIRNAME, conf);
String parentOfParent = p.getParent().getParent().getName(); String parentOfParent = p.getParent().getParent().getName();
assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
} }
@ -405,25 +394,38 @@ public class TestWALSplit {
*/ */
@Test @Test
public void testOldRecoveredEditsFileSidelined() throws IOException { public void testOldRecoveredEditsFileSidelined() throws IOException {
byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); Path p = createRecoveredEditsPathForRegion();
Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);
Path regiondir = new Path(tdir, Path regiondir = new Path(tdir,
RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
fs.mkdirs(regiondir); fs.mkdirs(regiondir);
Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());
fs.createNewFile(parent); // create a recovered.edits file
String parentOfParent = p.getParent().getParent().getName();
assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
WALFactory.createRecoveredEditsWriter(fs, p, conf).close();
}
private Path createRecoveredEditsPathForRegion() throws IOException{
byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes();
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
Entry entry = Entry entry =
new Entry(new WALKeyImpl(encoded, new Entry(new WALKeyImpl(encoded,
TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
new WALEdit()); new WALEdit());
Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());
fs.createNewFile(parent); // create a recovered.edits file
Path p = WALSplitter.getRegionSplitEditsPath(entry, Path p = WALSplitter.getRegionSplitEditsPath(entry,
FILENAME_BEING_SPLIT, TMPDIRNAME, conf); FILENAME_BEING_SPLIT, TMPDIRNAME, conf);
String parentOfParent = p.getParent().getParent().getName(); return p;
assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); }
WALFactory.createRecoveredEditsWriter(fs, p, conf).close();
@Test
public void testHasRecoveredEdits() throws IOException {
Path p = createRecoveredEditsPathForRegion();
assertFalse(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO));
String renamedEdit = p.getName().split("-")[0];
fs.createNewFile(new Path(p.getParent(), renamedEdit));
assertTrue(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO));
} }
private void useDifferentDFSClient() throws IOException { private void useDifferentDFSClient() throws IOException {