From e6b2e7ff45cdbe2058af4376c44df31cbdbb253e Mon Sep 17 00:00:00 2001 From: anoopsamjohn Date: Mon, 23 Dec 2013 06:45:37 +0000 Subject: [PATCH] HBASE-9346 HBCK should provide an option to check if regions boundaries are the same in META and in stores. (Jean Marc) git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1553075 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/hbase/util/HBaseFsck.java | 125 +++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 85e45128226..270009cd6ca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -93,8 +93,8 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; -import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; @@ -215,6 +215,7 @@ public class HBaseFsck extends Configured { private boolean rerun = false; // if we tried to fix something, rerun hbck private static boolean summary = false; // if we want to print less output private boolean checkMetaOnly = false; + private boolean checkRegionBoundaries = false; private boolean ignorePreCheckPermission = false; // if pre-check permission /********* @@ -479,6 +480,10 @@ public class HBaseFsck extends Configured { admin.setBalancerRunning(oldBalancer, false); } + if (checkRegionBoundaries) { + checkRegionBoundaries(); + } + offlineReferenceFileRepair(); checkAndFixTableLocks(); @@ -488,6 +493,112 @@ public class HBaseFsck extends Configured { return errors.summarize(); } + public static byte[] keyOnly (byte[] b) { + if (b == null) + return b; + int rowlength = Bytes.toShort(b, 0); + byte[] result = new byte[rowlength]; + System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength); + return result; + } + + private static class RegionBoundariesInformation { + public byte [] regionName; + public byte [] metaFirstKey; + public byte [] metaLastKey; + public byte [] storesFirstKey; + public byte [] storesLastKey; + public String toString () { + return "regionName=" + Bytes.toStringBinary(regionName) + + "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) + + "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) + + "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) + + "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey); + } + } + + public void checkRegionBoundaries() { + try { + ByteArrayComparator comparator = new ByteArrayComparator(); + List regions = MetaScanner.listAllRegions(getConf(), false); + final RegionBoundariesInformation currentRegionBoundariesInformation = + new RegionBoundariesInformation(); + for (HRegionInfo regionInfo : regions) { + currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); + // For each region, get the start and stop key from the META and compare them to the + // same information from the Stores. + Path path = new Path(getConf().get(HConstants.HBASE_DIR) + "/" + + Bytes.toString(regionInfo.getTable().getName()) + "/" + + regionInfo.getEncodedName() + "/"); + FileSystem fs = path.getFileSystem(getConf()); + FileStatus[] files = fs.listStatus(path); + // For all the column families in this region... + byte[] storeFirstKey = null; + byte[] storeLastKey = null; + for (FileStatus file : files) { + String fileName = file.getPath().toString(); + fileName = fileName.substring(fileName.lastIndexOf("/") + 1); + if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { + FileStatus[] storeFiles = fs.listStatus(file.getPath()); + // For all the stores in this column family. + for (FileStatus storeFile : storeFiles) { + HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( + getConf()), getConf()); + if ((reader.getFirstKey() != null) + && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, + reader.getFirstKey()) > 0))) { + storeFirstKey = reader.getFirstKey(); + } + if ((reader.getLastKey() != null) + && ((storeLastKey == null) || (comparator.compare(storeLastKey, + reader.getLastKey())) < 0)) { + storeLastKey = reader.getLastKey(); + } + reader.close(); + } + } + } + currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); + currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); + currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); + currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); + if (currentRegionBoundariesInformation.metaFirstKey.length == 0) + currentRegionBoundariesInformation.metaFirstKey = null; + if (currentRegionBoundariesInformation.metaLastKey.length == 0) + currentRegionBoundariesInformation.metaLastKey = null; + + // For a region to be correct, we need the META start key to be smaller or equal to the + // smallest start key from all the stores, and the start key from the next META entry to + // be bigger than the last key from all the current stores. First region start key is null; + // Last region end key is null; some regions can be empty and not have any store. + + boolean valid = true; + // Checking start key. + if ((currentRegionBoundariesInformation.storesFirstKey != null) + && (currentRegionBoundariesInformation.metaFirstKey != null)) { + valid = valid + && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, + currentRegionBoundariesInformation.metaFirstKey) >= 0; + } + // Checking stop key. + if ((currentRegionBoundariesInformation.storesLastKey != null) + && (currentRegionBoundariesInformation.metaLastKey != null)) { + valid = valid + && comparator.compare(currentRegionBoundariesInformation.storesLastKey, + currentRegionBoundariesInformation.metaLastKey) < 0; + } + if (!valid) { + errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", + tablesInfo.get(regionInfo.getTable())); + LOG.warn("Region's boundaries not alligned between stores and META for:"); + LOG.warn(currentRegionBoundariesInformation); + } + } + } catch (IOException e) { + LOG.error(e); + } + } + /** * Iterates through the list of all orphan/invalid regiondirs. */ @@ -2992,7 +3103,7 @@ public class HBaseFsck extends Configured { FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE, - WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK + WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR } void clear(); void report(String message); @@ -3343,6 +3454,13 @@ public class HBaseFsck extends Configured { checkMetaOnly = true; } + /** + * Set region boundaries check mode. + */ + void setRegionBoundariesCheck() { + checkRegionBoundaries = true; + } + /** * Set table locks fix mode. * Delete table locks held for a long time @@ -3560,6 +3678,7 @@ public class HBaseFsck extends Configured { out.println(" -summary Print only summary of the tables and status."); out.println(" -metaonly Only check the state of the hbase:meta table."); out.println(" -sidelineDir HDFS path to backup existing meta."); + out.println(" -boundaries Verify that regions boundaries are the same between META and store files."); out.println(""); out.println(" Metadata Repair options: (expert features, use with caution!)"); @@ -3774,6 +3893,8 @@ public class HBaseFsck extends Configured { setSummary(); } else if (cmd.equals("-metaonly")) { setCheckMetaOnly(); + } else if (cmd.equals("-boundaries")) { + setRegionBoundariesCheck(); } else if (cmd.equals("-fixTableLocks")) { setFixTableLocks(true); } else if (cmd.startsWith("-")) {