HBASE-9346 HBCK should provide an option to check if regions boundaries are the same in META and in stores. (Jean Marc)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1553075 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
anoopsamjohn 2013-12-23 06:45:37 +00:00
parent 8d7ed731a9
commit e6b2e7ff45
1 changed files with 123 additions and 2 deletions

View File

@ -93,8 +93,8 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
@ -215,6 +215,7 @@ public class HBaseFsck extends Configured {
private boolean rerun = false; // if we tried to fix something, rerun hbck private boolean rerun = false; // if we tried to fix something, rerun hbck
private static boolean summary = false; // if we want to print less output private static boolean summary = false; // if we want to print less output
private boolean checkMetaOnly = false; private boolean checkMetaOnly = false;
private boolean checkRegionBoundaries = false;
private boolean ignorePreCheckPermission = false; // if pre-check permission private boolean ignorePreCheckPermission = false; // if pre-check permission
/********* /*********
@ -479,6 +480,10 @@ public class HBaseFsck extends Configured {
admin.setBalancerRunning(oldBalancer, false); admin.setBalancerRunning(oldBalancer, false);
} }
if (checkRegionBoundaries) {
checkRegionBoundaries();
}
offlineReferenceFileRepair(); offlineReferenceFileRepair();
checkAndFixTableLocks(); checkAndFixTableLocks();
@ -488,6 +493,112 @@ public class HBaseFsck extends Configured {
return errors.summarize(); return errors.summarize();
} }
public static byte[] keyOnly (byte[] b) {
if (b == null)
return b;
int rowlength = Bytes.toShort(b, 0);
byte[] result = new byte[rowlength];
System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
return result;
}
private static class RegionBoundariesInformation {
public byte [] regionName;
public byte [] metaFirstKey;
public byte [] metaLastKey;
public byte [] storesFirstKey;
public byte [] storesLastKey;
public String toString () {
return "regionName=" + Bytes.toStringBinary(regionName) +
"\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
"\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
"\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
"\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
}
}
public void checkRegionBoundaries() {
try {
ByteArrayComparator comparator = new ByteArrayComparator();
List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
final RegionBoundariesInformation currentRegionBoundariesInformation =
new RegionBoundariesInformation();
for (HRegionInfo regionInfo : regions) {
currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
// For each region, get the start and stop key from the META and compare them to the
// same information from the Stores.
Path path = new Path(getConf().get(HConstants.HBASE_DIR) + "/"
+ Bytes.toString(regionInfo.getTable().getName()) + "/"
+ regionInfo.getEncodedName() + "/");
FileSystem fs = path.getFileSystem(getConf());
FileStatus[] files = fs.listStatus(path);
// For all the column families in this region...
byte[] storeFirstKey = null;
byte[] storeLastKey = null;
for (FileStatus file : files) {
String fileName = file.getPath().toString();
fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
FileStatus[] storeFiles = fs.listStatus(file.getPath());
// For all the stores in this column family.
for (FileStatus storeFile : storeFiles) {
HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
getConf()), getConf());
if ((reader.getFirstKey() != null)
&& ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
reader.getFirstKey()) > 0))) {
storeFirstKey = reader.getFirstKey();
}
if ((reader.getLastKey() != null)
&& ((storeLastKey == null) || (comparator.compare(storeLastKey,
reader.getLastKey())) < 0)) {
storeLastKey = reader.getLastKey();
}
reader.close();
}
}
}
currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
currentRegionBoundariesInformation.metaFirstKey = null;
if (currentRegionBoundariesInformation.metaLastKey.length == 0)
currentRegionBoundariesInformation.metaLastKey = null;
// For a region to be correct, we need the META start key to be smaller or equal to the
// smallest start key from all the stores, and the start key from the next META entry to
// be bigger than the last key from all the current stores. First region start key is null;
// Last region end key is null; some regions can be empty and not have any store.
boolean valid = true;
// Checking start key.
if ((currentRegionBoundariesInformation.storesFirstKey != null)
&& (currentRegionBoundariesInformation.metaFirstKey != null)) {
valid = valid
&& comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
currentRegionBoundariesInformation.metaFirstKey) >= 0;
}
// Checking stop key.
if ((currentRegionBoundariesInformation.storesLastKey != null)
&& (currentRegionBoundariesInformation.metaLastKey != null)) {
valid = valid
&& comparator.compare(currentRegionBoundariesInformation.storesLastKey,
currentRegionBoundariesInformation.metaLastKey) < 0;
}
if (!valid) {
errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
tablesInfo.get(regionInfo.getTable()));
LOG.warn("Region's boundaries not alligned between stores and META for:");
LOG.warn(currentRegionBoundariesInformation);
}
}
} catch (IOException e) {
LOG.error(e);
}
}
/** /**
* Iterates through the list of all orphan/invalid regiondirs. * Iterates through the list of all orphan/invalid regiondirs.
*/ */
@ -2992,7 +3103,7 @@ public class HBaseFsck extends Configured {
FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE, ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
} }
void clear(); void clear();
void report(String message); void report(String message);
@ -3343,6 +3454,13 @@ public class HBaseFsck extends Configured {
checkMetaOnly = true; checkMetaOnly = true;
} }
/**
* Set region boundaries check mode.
*/
void setRegionBoundariesCheck() {
checkRegionBoundaries = true;
}
/** /**
* Set table locks fix mode. * Set table locks fix mode.
* Delete table locks held for a long time * Delete table locks held for a long time
@ -3560,6 +3678,7 @@ public class HBaseFsck extends Configured {
out.println(" -summary Print only summary of the tables and status."); out.println(" -summary Print only summary of the tables and status.");
out.println(" -metaonly Only check the state of the hbase:meta table."); out.println(" -metaonly Only check the state of the hbase:meta table.");
out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta."); out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
out.println(""); out.println("");
out.println(" Metadata Repair options: (expert features, use with caution!)"); out.println(" Metadata Repair options: (expert features, use with caution!)");
@ -3774,6 +3893,8 @@ public class HBaseFsck extends Configured {
setSummary(); setSummary();
} else if (cmd.equals("-metaonly")) { } else if (cmd.equals("-metaonly")) {
setCheckMetaOnly(); setCheckMetaOnly();
} else if (cmd.equals("-boundaries")) {
setRegionBoundariesCheck();
} else if (cmd.equals("-fixTableLocks")) { } else if (cmd.equals("-fixTableLocks")) {
setFixTableLocks(true); setFixTableLocks(true);
} else if (cmd.startsWith("-")) { } else if (cmd.startsWith("-")) {