diff --git a/CHANGES.txt b/CHANGES.txt index 22ea8b2bfe0..1b7a1c80905 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -26,6 +26,7 @@ Hbase Change Log HBASE-641 Improve master split logging HBASE-642 Splitting log in a hostile environment -- bad hdfs -- we drop write-ahead-log edits + HBASE-646 EOFException opening HStoreFile info file (spin on HBASE-645and 550) IMPROVEMENTS HBASE-559 MR example job to count table rows diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 4e03313019b..153a31b777b 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1704,7 +1704,7 @@ public class HRegion implements HConstants { filter != null ? (RowFilterInterface)WritableUtils.clone(filter, conf) : filter); } - } catch(IOException e) { + } catch (IOException e) { for (int i = 0; i < this.scanners.length; i++) { if(scanners[i] != null) { closeScanner(i); @@ -1828,12 +1828,17 @@ public class HRegion implements HConstants { try { scanners[i].close(); } catch (IOException e) { - LOG.warn("Failed closeing scanner " + i, e); + LOG.warn("Failed closing scanner " + i, e); } } finally { scanners[i] = null; - resultSets[i] = null; - keys[i] = null; + // These data members can be null if exception in constructor + if (resultSets != null) { + resultSets[i] = null; + } + if (keys != null) { + keys[i] = null; + } } } diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HStore.java b/src/java/org/apache/hadoop/hbase/regionserver/HStore.java index 5f037ff1c76..79dc5005e06 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -392,6 +392,14 @@ public class HStore implements HConstants { ArrayList mapfiles = new ArrayList(infofiles.length); for (int i = 0; i < infofiles.length; i++) { Path p = infofiles[i].getPath(); + // Check for empty info file. Should never be the case but can happen + // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646 + if (this.fs.getFileStatus(p).getLen() <= 0) { + LOG.warn("Skipping " + p + " because its empty. DATA LOSS? Can " + + "this scenario be repaired? HBASE-646"); + continue; + } + Matcher m = REF_NAME_PARSER.matcher(p.getName()); /* * * * * * N O T E * * * * * @@ -434,6 +442,13 @@ public class HStore implements HConstants { "Cleaned up info file. Continuing...Probable DATA LOSS!!!"); continue; } + if (isEmptyDataFile(mapfile)) { + curfile.delete(); + // We can have empty data file if data loss in hdfs. + LOG.warn("Mapfile " + mapfile.toString() + " has empty data. " + + "Deleting. Continuing...Probable DATA LOSS!!! See HBASE-646."); + continue; + } // TODO: Confirm referent exists. @@ -460,7 +475,22 @@ public class HStore implements HConstants { } return results; } - + + /* + * @param mapfile + * @return True if the passed mapfile has a zero-length data component (its + * broken). + * @throws IOException + */ + private boolean isEmptyDataFile(final Path mapfile) + throws IOException { + // Mapfiles are made of 'data' and 'index' files. Confirm 'data' is + // non-null if it exists (may not have been written to yet). + Path dataFile = new Path(mapfile, "data"); + return this.fs.exists(dataFile) && + this.fs.getFileStatus(dataFile).getLen() == 0; + } + ////////////////////////////////////////////////////////////////////////////// // Bloom filters //////////////////////////////////////////////////////////////////////////////