HBASE-646 EOFException opening HStoreFile info file (spin on HBASE-645 and 550)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@661040 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -26,6 +26,7 @@ Hbase Change Log
    HBASE-641   Improve master split logging
    HBASE-642   Splitting log in a hostile environment -- bad hdfs -- we drop
                write-ahead-log edits
+   HBASE-646   EOFException opening HStoreFile info file (spin on HBASE-645and 550)
 
   IMPROVEMENTS
    HBASE-559   MR example job to count table rows

src/java/org/apache/hadoop/hbase/regionserver/HRegion.java

@@ -1704,7 +1704,7 @@ public class HRegion implements HConstants {
             filter != null ?
               (RowFilterInterface)WritableUtils.clone(filter, conf) : filter);
         }
-      } catch(IOException e) {
+      } catch (IOException e) {
         for (int i = 0; i < this.scanners.length; i++) {
           if(scanners[i] != null) {
             closeScanner(i);
@@ -1828,14 +1828,19 @@ public class HRegion implements HConstants {
         try {
           scanners[i].close();
         } catch (IOException e) {
-          LOG.warn("Failed closeing scanner " + i, e);
+          LOG.warn("Failed closing scanner " + i, e);
         }
       } finally {
         scanners[i] = null;
+        // These data members can be null if exception in constructor
+        if (resultSets != null) {
           resultSets[i] = null;
+        }
+        if (keys != null) {
           keys[i] = null;
         }
       }
+    }
 
     /**
      * {@inheritDoc}

src/java/org/apache/hadoop/hbase/regionserver/HStore.java

@@ -392,6 +392,14 @@ public class HStore implements HConstants {
     ArrayList<Path> mapfiles = new ArrayList<Path>(infofiles.length);
     for (int i = 0; i < infofiles.length; i++) {
       Path p = infofiles[i].getPath();
+      // Check for empty info file.  Should never be the case but can happen
+      // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
+      if (this.fs.getFileStatus(p).getLen() <= 0) {
+        LOG.warn("Skipping " + p + " because its empty.  DATA LOSS?  Can " +
+          "this scenario be repaired?  HBASE-646");
+        continue;
+      }
+
       Matcher m = REF_NAME_PARSER.matcher(p.getName());
       /*
        *  *  *  *  *  N O T E  *  *  *  *  *
@@ -434,6 +442,13 @@ public class HStore implements HConstants {
           "Cleaned up info file.  Continuing...Probable DATA LOSS!!!");
         continue;
       }
+      if (isEmptyDataFile(mapfile)) {
+        curfile.delete();
+        // We can have empty data file if data loss in hdfs.
+        LOG.warn("Mapfile " + mapfile.toString() + " has empty data. " +
+          "Deleting.  Continuing...Probable DATA LOSS!!!  See HBASE-646.");
+        continue;
+      }
       
       // TODO: Confirm referent exists.
       
@@ -461,6 +476,21 @@ public class HStore implements HConstants {
     return results;
   }
 
+  /* 
+   * @param mapfile
+   * @return True if the passed mapfile has a zero-length data component (its
+   * broken).
+   * @throws IOException
+   */
+  private boolean isEmptyDataFile(final Path mapfile)
+  throws IOException {
+    // Mapfiles are made of 'data' and 'index' files.  Confirm 'data' is
+    // non-null if it exists (may not have been written to yet).
+    Path dataFile = new Path(mapfile, "data");
+    return this.fs.exists(dataFile) &&
+      this.fs.getFileStatus(dataFile).getLen() == 0;
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   // Bloom filters
   //////////////////////////////////////////////////////////////////////////////