Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24273 HBCK's "Orphan Regions on FileSystem" reports regions with referenced HFiles (#1613) (#1661) 

Signed-off-by: stack <stack@apache.org>
This commit is contained in:
huaxiangsun 2020-05-06 09:50:52 -07:00 committed by GitHub
parent 89ae3c55c8
commit 04ac08d49b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 39 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java
View File

@ -28,6 +28,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.ScheduledChore;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionInfo;
@ -134,7 +135,7 @@ public class HbckChore extends ScheduledChore {
loadRegionsFromInMemoryState();
loadRegionsFromRSReport();
try {
loadRegionsFromFS();
loadRegionsFromFS(scanForMergedParentRegions());
} catch (IOException e) {
LOG.warn("Failed to load the regions from filesystem", e);
}
@ -187,6 +188,31 @@ public class HbckChore extends ScheduledChore {
}
}
/**
* Scan hbase:meta to get set of merged parent regions, this is a very heavy scan.
*
* @return Return generated {@link HashSet}
*/
private HashSet<String> scanForMergedParentRegions() throws IOException {
HashSet<String> mergedParentRegions = new HashSet<>();
// Null tablename means scan all of meta.
MetaTableAccessor.scanMetaForTableRegions(this.master.getConnection(),
r -> {
List<RegionInfo> mergeParents = MetaTableAccessor.getMergeRegions(r.rawCells());
if (mergeParents != null) {
for (RegionInfo mergeRegion : mergeParents) {
if (mergeRegion != null) {
// This region is already being merged
mergedParentRegions.add(mergeRegion.getEncodedName());
}
}
}
return true;
},
null);
return mergedParentRegions;
}
private void loadRegionsFromInMemoryState() {
List<RegionState> regionStates =
master.getAssignmentManager().getRegionStates().getRegionStates();
@ -256,7 +282,7 @@ public class HbckChore extends ScheduledChore {
}
}
private void loadRegionsFromFS() throws IOException {
private void loadRegionsFromFS(final HashSet<String> mergedParentRegions) throws IOException {
Path rootDir = master.getMasterFileSystem().getRootDir();
FileSystem fs = master.getMasterFileSystem().getFileSystem();
@ -271,12 +297,12 @@ public class HbckChore extends ScheduledChore {
continue;
}
HbckRegionInfo hri = regionInfoMap.get(encodedRegionName);
if (hri == null) {
// If it is not in in-memory database and not a merged region,
// report it as an orphan region.
if (hri == null && !mergedParentRegions.contains(encodedRegionName)) {
orphanRegionsOnFS.put(encodedRegionName, regionDir);
continue;
}
HbckRegionInfo.HdfsEntry hdfsEntry = new HbckRegionInfo.HdfsEntry(regionDir);
hri.setHdfsEntry(hdfsEntry);
}
numRegions += regionDirs.size();
}

10
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.assignment.GCRegionProcedure;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@ -147,10 +148,12 @@ public class TestMetaFixer {
@Test
public void testOverlap() throws Exception {
TableName tn = TableName.valueOf(this.name.getMethodName());
TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
TEST_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
assertTrue(ris.size() > 5);
MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
HbckChore hbckChore = services.getHbckChore();
services.getCatalogJanitor().scan();
CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
assertTrue(report.isEmpty());
@ -174,6 +177,9 @@ public class TestMetaFixer {
throw new RuntimeException(e);
}
});
hbckChore.chore();
assertEquals(0, hbckChore.getOrphanRegionsOnFS().size());
}
/**