diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 3a5ea5ea4bc..6eab3ed1d82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -22,6 +22,7 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -195,6 +196,7 @@ public class HBaseFsck extends Configured implements Tool {
private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
private boolean fixSplitParents = false; // fix lingering split parents
private boolean fixReferenceFiles = false; // fix lingering reference store file
+ private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
// limit checking/fixes to listed tables, if empty attempt to check/fix all
// .META. are always checked
@@ -397,6 +399,14 @@ public class HBaseFsck extends Configured implements Tool {
return -2;
}
+ // Empty cells in .META.?
+ reportEmptyMetaCells();
+
+ // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from .META.
+ if (shouldFixEmptyMetaCells()) {
+ fixEmptyMetaCells();
+ }
+
// get a list of all tables that have not changed recently.
if (!checkMetaOnly) {
reportTablesInFlux();
@@ -411,9 +421,6 @@ public class HBaseFsck extends Configured implements Tool {
loadHdfsRegionInfos();
}
- // Empty cells in .META.?
- reportEmptyMetaCells();
-
// Get disabled tables from ZooKeeper
loadDisabledTables();
@@ -851,6 +858,21 @@ public class HBaseFsck extends Configured implements Tool {
return true;
}
+ /**
+ * To fix the empty REGIONINFO_QUALIFIER rows from .META.
+ * @throws IOException
+ */
+ public void fixEmptyMetaCells() throws IOException {
+ if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
+ LOG.info("Trying to fix empty REGIONINFO_QUALIFIER .META. rows.");
+ for (Result region : emptyRegionInfoQualifiers) {
+ deleteMetaRegion(region.getRow());
+ errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
+ }
+ emptyRegionInfoQualifiers.clear();
+ }
+ }
+
/**
* To fix orphan table by creating a .tableinfo file under tableDir
* 1. if TableInfo is cached, to recover the .tableinfo accordingly
@@ -1410,10 +1432,17 @@ public class HBaseFsck extends Configured implements Tool {
* Deletes region from meta table
*/
private void deleteMetaRegion(HbckInfo hi) throws IOException {
- Delete d = new Delete(hi.metaEntry.getRegionName());
+ deleteMetaRegion(hi.metaEntry.getRegionName());
+ }
+
+ /**
+ * Deletes region from meta table
+ */
+ private void deleteMetaRegion(byte[] metaKey) throws IOException {
+ Delete d = new Delete(metaKey);
meta.delete(d);
meta.flushCommits();
- LOG.info("Deleted " + hi.metaEntry.getRegionNameAsString() + " from META" );
+ LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
}
/**
@@ -2531,6 +2560,8 @@ public class HBaseFsck extends Configured implements Tool {
Pair pair = HRegionInfo.getHRegionInfoAndServerName(result);
if (pair == null || pair.getFirst() == null) {
emptyRegionInfoQualifiers.add(result);
+ errors.reportError(ERROR_CODE.EMPTY_META_CELL,
+ "Empty REGIONINFO_QUALIFIER found in .META.");
return true;
}
ServerName sn = null;
@@ -2604,6 +2635,21 @@ public class HBaseFsck extends Configured implements Tool {
}
return (modTime == me.modTime);
}
+
+ @Override
+ public int hashCode() {
+ int hash = Arrays.hashCode(getRegionName());
+ hash ^= getRegionId();
+ hash ^= Arrays.hashCode(getStartKey());
+ hash ^= Arrays.hashCode(getEndKey());
+ hash ^= Boolean.valueOf(isOffline()).hashCode();
+ hash ^= Arrays.hashCode(getTableName());
+ if (regionServer != null) {
+ hash ^= regionServer.hashCode();
+ }
+ hash ^= modTime;
+ return hash;
+ }
}
/**
@@ -2851,7 +2897,7 @@ public class HBaseFsck extends Configured implements Tool {
FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
- WRONG_USAGE
+ WRONG_USAGE, EMPTY_META_CELL
}
public void clear();
public void report(String message);
@@ -3225,6 +3271,14 @@ public class HBaseFsck extends Configured implements Tool {
return fixMeta;
}
+ public void setFixEmptyMetaCells(boolean shouldFix) {
+ fixEmptyMetaCells = shouldFix;
+ }
+
+ boolean shouldFixEmptyMetaCells() {
+ return fixEmptyMetaCells;
+ }
+
public void setCheckHdfs(boolean checking) {
checkHdfs = checking;
}
@@ -3411,6 +3465,8 @@ public class HBaseFsck extends Configured implements Tool {
out.println(" -fixSplitParents Try to force offline split parents to be online.");
out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
out.println(" -fixReferenceFiles Try to offline lingering reference store files");
+ out.println(" -fixEmptyMetaCells Try to fix .META. entries not referencing any region"
+ + " (empty REGIONINFO_QUALIFIER rows)");
out.println("");
out.println(" Datafile Repair options: (expert features, use with caution!)");
@@ -3532,6 +3588,8 @@ public class HBaseFsck extends Configured implements Tool {
sidelineCorruptHFiles = true;
} else if (cmd.equals("-fixReferenceFiles")) {
setFixReferenceFiles(true);
+ } else if (cmd.equals("-fixEmptyMetaCells")) {
+ setFixEmptyMetaCells(true);
} else if (cmd.equals("-repair")) {
// this attempts to merge overlapping hdfs regions, needs testing
// under load
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index ac9f3b3966c..5108bf2cf6a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1309,7 +1309,7 @@ public class TestHBaseFsck {
// TODO: fixHdfsHoles does not work against splits, since the parent dir lingers on
// for some time until children references are deleted. HBCK erroneously sees this as
// overlapping regions
- HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, null);
+ HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, false, null);
assertErrors(hbck, new ERROR_CODE[] {}); //no LINGERING_SPLIT_PARENT reported
// assert that the split META entry is still there.
@@ -1371,7 +1371,7 @@ public class TestHBaseFsck {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); //no LINGERING_SPLIT_PARENT
// now fix it. The fix should not revert the region split, but add daughters to META
- hbck = doFsck(conf, true, true, false, false, false, false, false, false, null);
+ hbck = doFsck(conf, true, true, false, false, false, false, false, false, false, null);
assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
@@ -1823,6 +1823,44 @@ public class TestHBaseFsck {
}
}
+ /**
+ * Test mission REGIONINFO_QUALIFIER in .META.
+ */
+ @Test
+ public void testMissingRegionInfoQualifier() throws Exception {
+ String table = "testMissingRegionInfoQualifier";
+ try {
+ setupTable(table);
+
+ // Mess it up by removing the RegionInfo for one region.
+ HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
+ ResultScanner scanner = meta.getScanner(new Scan());
+ Result result = scanner.next();
+ Delete delete = new Delete (result.getRow());
+ delete.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ meta.delete(delete);
+
+ // Mess it up by creating a fake META entry with no associated RegionInfo
+ meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
+ HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
+ meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
+ HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
+ meta.close();
+
+ HBaseFsck hbck = doFsck(conf, false);
+ assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
+
+ // fix reference file
+ hbck = doFsck(conf, true);
+
+ // check that reference file fixed
+ assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
+ } finally {
+ deleteTable(table);
+ }
+ }
+
+
/**
* Test pluggable error reporter. It can be plugged in
* from system property or configuration.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 99f4f9b8a1c..7bc77611350 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -38,13 +38,13 @@ public class HbckTestingUtil {
public static HBaseFsck doFsck(
Configuration conf, boolean fix, String table) throws Exception {
- return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, fix, table);
+ return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, fix, fix, table);
}
public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
boolean fixHdfsOrphans, boolean fixTableOrphans, boolean fixVersionFile,
- boolean fixReferenceFiles, String table) throws Exception {
+ boolean fixReferenceFiles, boolean fixEmptyMetaRegionInfo, String table) throws Exception {
HBaseFsck fsck = new HBaseFsck(conf, exec);
fsck.connect();
fsck.setDisplayFullReport(); // i.e. -details
@@ -57,6 +57,7 @@ public class HbckTestingUtil {
fsck.setFixTableOrphans(fixTableOrphans);
fsck.setFixVersionFile(fixVersionFile);
fsck.setFixReferenceFiles(fixReferenceFiles);
+ fsck.setFixEmptyMetaCells(fixEmptyMetaRegionInfo);
if (table != null) {
fsck.includeTable(table);
}