From 8d0665881e97734b395e6dda26fea2abbb85f7eb Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 28 May 2008 21:17:31 +0000 Subject: [PATCH] HBASE-648 If mapfile index is empty, run repair git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@661089 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../hadoop/hbase/regionserver/HStore.java | 81 ++++++++++++------- .../hadoop/hbase/regionserver/HStoreFile.java | 7 +- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1b7a1c80905..ff699f460bb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -27,6 +27,7 @@ Hbase Change Log HBASE-642 Splitting log in a hostile environment -- bad hdfs -- we drop write-ahead-log edits HBASE-646 EOFException opening HStoreFile info file (spin on HBASE-645and 550) + HBASE-648 If mapfile index is empty, run repair IMPROVEMENTS HBASE-559 MR example job to count table rows diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HStore.java b/src/java/org/apache/hadoop/hbase/regionserver/HStore.java index 79dc5005e06..6609cd4386c 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -263,29 +263,14 @@ public class HStore implements HConstants { boolean first = true; for(Map.Entry e: this.storefiles.entrySet()) { MapFile.Reader r = null; - try { - if (first) { - // Use a block cache (if configured) for the first reader only - // so as to control memory usage. - r = e.getValue().getReader(this.fs, this.bloomFilter, - family.isBlockCacheEnabled()); - first = false; - } else { - r = e.getValue().getReader(this.fs, this.bloomFilter); - } - } catch (EOFException eofe) { - LOG.warn("Failed open of reader " + e.toString() + "; attempting fix", - eofe); - try { - // Try fixing this file.. if we can. - MapFile.fix(this.fs, e.getValue().getMapFilePath(), - HStoreFile.HbaseMapFile.KEY_CLASS, - HStoreFile.HbaseMapFile.VALUE_CLASS, false, this.conf); - } catch (Exception fixe) { - LOG.warn("Failed fix of " + e.toString() + - "...continuing; Probable DATA LOSS!!!", fixe); - continue; - } + if (first) { + // Use a block cache (if configured) for the first reader only + // so as to control memory usage. + r = e.getValue().getReader(this.fs, this.bloomFilter, + family.isBlockCacheEnabled()); + first = false; + } else { + r = e.getValue().getReader(this.fs, this.bloomFilter); } this.readers.put(e.getKey(), r); } @@ -377,7 +362,8 @@ public class HStore implements HConstants { /* * Creates a series of HStoreFiles loaded from the given directory. * There must be a matching 'mapdir' and 'loginfo' pair of files. - * If only one exists, we'll delete it. + * If only one exists, we'll delete it. Does other consistency tests + * checking files are not zero, etc. * * @param infodir qualified path for info file directory * @param mapdir qualified path for map file directory @@ -434,7 +420,6 @@ public class HStore implements HConstants { LOG.info("HSTORE_LOGINFOFILE " + curfile + " does not contain a sequence number - ignoring"); } - Path mapfile = curfile.getMapFilePath(); if (!fs.exists(mapfile)) { fs.delete(curfile.getInfoFilePath(), false); @@ -449,6 +434,23 @@ public class HStore implements HConstants { "Deleting. Continuing...Probable DATA LOSS!!! See HBASE-646."); continue; } + if (isEmptyIndexFile(mapfile)) { + try { + // Try fixing this file.. if we can. Use the hbase version of fix. + // Need to remove the old index file first else fix won't go ahead. + this.fs.delete(new Path(mapfile, MapFile.INDEX_FILE_NAME)); + long count = MapFile.fix(this.fs, mapfile, HStoreFile.HbaseMapFile.KEY_CLASS, + HStoreFile.HbaseMapFile.VALUE_CLASS, false, this.conf); + if (LOG.isDebugEnabled()) { + LOG.debug("Fixed index on " + mapfile.toString() + "; had " + + count + " entries"); + } + } catch (Exception e) { + LOG.warn("Failed fix of " + mapfile.toString() + + "...continuing; Probable DATA LOSS!!!", e); + continue; + } + } // TODO: Confirm referent exists. @@ -486,9 +488,32 @@ public class HStore implements HConstants { throws IOException { // Mapfiles are made of 'data' and 'index' files. Confirm 'data' is // non-null if it exists (may not have been written to yet). - Path dataFile = new Path(mapfile, "data"); - return this.fs.exists(dataFile) && - this.fs.getFileStatus(dataFile).getLen() == 0; + return isEmptyFile(new Path(mapfile, MapFile.DATA_FILE_NAME)); + } + + /* + * @param mapfile + * @return True if the passed mapfile has a zero-length index component (its + * broken). + * @throws IOException + */ + private boolean isEmptyIndexFile(final Path mapfile) + throws IOException { + // Mapfiles are made of 'data' and 'index' files. Confirm 'data' is + // non-null if it exists (may not have been written to yet). + return isEmptyFile(new Path(mapfile, MapFile.INDEX_FILE_NAME)); + } + + /* + * @param mapfile + * @return True if the passed mapfile has a zero-length index component (its + * broken). + * @throws IOException + */ + private boolean isEmptyFile(final Path f) + throws IOException { + return this.fs.exists(f) && + this.fs.getFileStatus(f).getLen() == 0; } ////////////////////////////////////////////////////////////////////////////// diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java b/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java index 359999d27c8..6ab2a678783 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java @@ -622,8 +622,9 @@ public class HStoreFile implements HConstants { * Hbase customizations of MapFile. */ static class HbaseMapFile extends MapFile { - static final Class KEY_CLASS = HStoreKey.class; - static final Class VALUE_CLASS = ImmutableBytesWritable.class; + static final Class KEY_CLASS = HStoreKey.class; + static final Class VALUE_CLASS = + ImmutableBytesWritable.class; /** * A reader capable of reading and caching blocks of the data file. @@ -696,8 +697,6 @@ public class HStoreFile implements HConstants { * @param conf * @param fs * @param dirName - * @param keyClass - * @param valClass * @param compression * @throws IOException */