HBASE-11011 Avoid extra getFileStatus() calls on Region startup

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1588517 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java

@@ -196,6 +196,32 @@ public class HRegionFileSystem {
     return storeFiles;
   }
 
+  /**
+   * Return Qualified Path of the specified family/file
+   *
+   * @param familyName Column Family Name
+   * @param fileName File Name
+   * @return The qualified Path for the specified family/file
+   */
+  Path getStoreFilePath(final String familyName, final String fileName) {
+    Path familyDir = getStoreDir(familyName);
+    return new Path(familyDir, fileName).makeQualified(this.fs);
+  }
+
+  /**
+   * Return the store file information of the specified family/file.
+   *
+   * @param familyName Column Family Name
+   * @param fileName File Name
+   * @return The {@link StoreFileInfo} for the specified family/file
+   */
+  StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
+      throws IOException {
+    Path familyDir = getStoreDir(familyName);
+    FileStatus status = fs.getFileStatus(new Path(familyDir, fileName));
+    return new StoreFileInfo(this.conf, this.fs, status);
+  }
+
   /**
    * Returns true if the specified family has reference files
    * @param familyName Column Family Name

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.io.FileNotFoundException;
 import java.io.InterruptedIOException;
 import java.net.InetSocketAddress;
 import java.security.Key;
@@ -501,7 +502,7 @@ public class HStore implements Store {
       completionService.submit(new Callable<StoreFile>() {
         @Override
         public StoreFile call() throws IOException {
-          StoreFile storeFile = createStoreFileAndReader(storeFileInfo.getPath());
+          StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
           return storeFile;
         }
       });
@@ -537,7 +538,7 @@ public class HStore implements Store {
       for (StoreFile file : results) {
         try {
           if (file != null) file.closeReader(true);
-        } catch (IOException e) { 
+        } catch (IOException e) {
           LOG.warn(e.getMessage());
         }
       }
@@ -549,6 +550,11 @@ public class HStore implements Store {
 
   private StoreFile createStoreFileAndReader(final Path p) throws IOException {
     StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);
+    return createStoreFileAndReader(info);
+  }
+
+  private StoreFile createStoreFileAndReader(final StoreFileInfo info)
+      throws IOException {
     info.setRegionCoprocessorHost(this.region.getCoprocessorHost());
     StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf,
       this.family.getBloomFilterType());
@@ -1077,7 +1083,7 @@ public class HStore implements Store {
       // TODO: get rid of this!
       if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
         LOG.warn("hbase.hstore.compaction.complete is set to false");
-        sfs = new ArrayList<StoreFile>();
+        sfs = new ArrayList<StoreFile>(newFiles.size());
         for (Path newFile : newFiles) {
           // Create storefile around what we wrote with a reader on it.
           StoreFile sf = createStoreFileAndReader(newFile);
@@ -1101,7 +1107,7 @@ public class HStore implements Store {
 
   private List<StoreFile> moveCompatedFilesIntoPlace(
       CompactionRequest cr, List<Path> newFiles) throws IOException {
-    List<StoreFile> sfs = new ArrayList<StoreFile>();
+    List<StoreFile> sfs = new ArrayList<StoreFile>(newFiles.size());
     for (Path newFile : newFiles) {
       assert newFile != null;
       StoreFile sf = moveFileIntoPlace(newFile);
@@ -1130,7 +1136,7 @@ public class HStore implements Store {
   private void writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
       Collection<StoreFile> newFiles) throws IOException {
     if (region.getLog() == null) return;
-    List<Path> inputPaths = new ArrayList<Path>();
+    List<Path> inputPaths = new ArrayList<Path>(filesCompacted.size());
     for (StoreFile f : filesCompacted) {
       inputPaths.add(f.getPath());
     }
@@ -1202,7 +1208,7 @@ public class HStore implements Store {
   /**
    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
-   * See HBASE-2331.
+   * See HBASE-2231.
    * @param compaction
    */
   @Override
@@ -1210,43 +1216,37 @@ public class HStore implements Store {
       throws IOException {
     LOG.debug("Completing compaction from the WAL marker");
     List<String> compactionInputs = compaction.getCompactionInputList();
-    List<String> compactionOutputs = compaction.getCompactionOutputList();
 
-    List<StoreFile> outputStoreFiles = new ArrayList<StoreFile>(compactionOutputs.size());
-    for (String compactionOutput : compactionOutputs) {
-      //we should have this store file already
-      boolean found = false;
-      Path outputPath = new Path(fs.getStoreDir(family.getNameAsString()), compactionOutput);
-      outputPath = outputPath.makeQualified(fs.getFileSystem());
-      for (StoreFile sf : this.getStorefiles()) {
-        if (sf.getPath().makeQualified(sf.getPath().getFileSystem(conf)).equals(outputPath)) {
-          found = true;
-          break;
-        }
-      }
-      if (!found) {
-        if (getFileSystem().exists(outputPath)) {
-          outputStoreFiles.add(createStoreFileAndReader(outputPath));
-        }
-      }
-    }
+    // The Compaction Marker is written after the compaction is completed,
+    // and the files moved into the region/family folder.
+    //
+    // If we crash after the entry is written, we may not have removed the
+    // input files, but the output file is present.
+    // (The unremoved input files will be removed by this function)
+    //
+    // If we scan the directory and the file is not present, it can mean that:
+    //   - The file was manually removed by the user
+    //   - The file was removed as consequence of subsequent compaction
+    // so, we can't do anything with the "compaction output list" because those
+    // files have already been loaded when opening the region (by virtue of
+    // being in the store's folder) or they may be missing due to a compaction.
 
+    String familyName = this.getColumnFamilyName();
     List<Path> inputPaths = new ArrayList<Path>(compactionInputs.size());
     for (String compactionInput : compactionInputs) {
-      Path inputPath = new Path(fs.getStoreDir(family.getNameAsString()), compactionInput);
-      inputPath = inputPath.makeQualified(fs.getFileSystem());
+      Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
       inputPaths.add(inputPath);
     }
 
     //some of the input files might already be deleted
     List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size());
     for (StoreFile sf : this.getStorefiles()) {
-      if (inputPaths.contains(sf.getPath().makeQualified(fs.getFileSystem()))) {
+      if (inputPaths.contains(sf.getQualifiedPath())) {
         inputStoreFiles.add(sf);
       }
     }
 
-    this.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
+    this.replaceStoreFiles(inputStoreFiles, Collections.EMPTY_LIST);
     this.completeCompaction(inputStoreFiles);
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java

@@ -233,6 +233,13 @@ public class StoreFile {
     return this.fileInfo.getPath();
   }
 
+  /**
+   * @return Returns the qualified path of this StoreFile
+   */
+  public Path getQualifiedPath() {
+    return this.fileInfo.getPath().makeQualified(fs);
+  }
+
   /**
    * @return True if this is a StoreFile Reference; call after {@link #open()}
    * else may get wrong answer.