diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index d597650fc8a..449747549e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -196,6 +196,32 @@ public class HRegionFileSystem { return storeFiles; } + /** + * Return Qualified Path of the specified family/file + * + * @param familyName Column Family Name + * @param fileName File Name + * @return The qualified Path for the specified family/file + */ + Path getStoreFilePath(final String familyName, final String fileName) { + Path familyDir = getStoreDir(familyName); + return new Path(familyDir, fileName).makeQualified(this.fs); + } + + /** + * Return the store file information of the specified family/file. + * + * @param familyName Column Family Name + * @param fileName File Name + * @return The {@link StoreFileInfo} for the specified family/file + */ + StoreFileInfo getStoreFileInfo(final String familyName, final String fileName) + throws IOException { + Path familyDir = getStoreDir(familyName); + FileStatus status = fs.getFileStatus(new Path(familyDir, fileName)); + return new StoreFileInfo(this.conf, this.fs, status); + } + /** * Returns true if the specified family has reference files * @param familyName Column Family Name diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index cb5fbcb8588..7f9f48286aa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.io.FileNotFoundException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; import java.security.Key; @@ -501,7 +502,7 @@ public class HStore implements Store { completionService.submit(new Callable() { @Override public StoreFile call() throws IOException { - StoreFile storeFile = createStoreFileAndReader(storeFileInfo.getPath()); + StoreFile storeFile = createStoreFileAndReader(storeFileInfo); return storeFile; } }); @@ -537,7 +538,7 @@ public class HStore implements Store { for (StoreFile file : results) { try { if (file != null) file.closeReader(true); - } catch (IOException e) { + } catch (IOException e) { LOG.warn(e.getMessage()); } } @@ -549,6 +550,11 @@ public class HStore implements Store { private StoreFile createStoreFileAndReader(final Path p) throws IOException { StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p); + return createStoreFileAndReader(info); + } + + private StoreFile createStoreFileAndReader(final StoreFileInfo info) + throws IOException { info.setRegionCoprocessorHost(this.region.getCoprocessorHost()); StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf, this.family.getBloomFilterType()); @@ -1077,7 +1083,7 @@ public class HStore implements Store { // TODO: get rid of this! if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { LOG.warn("hbase.hstore.compaction.complete is set to false"); - sfs = new ArrayList(); + sfs = new ArrayList(newFiles.size()); for (Path newFile : newFiles) { // Create storefile around what we wrote with a reader on it. StoreFile sf = createStoreFileAndReader(newFile); @@ -1101,7 +1107,7 @@ public class HStore implements Store { private List moveCompatedFilesIntoPlace( CompactionRequest cr, List newFiles) throws IOException { - List sfs = new ArrayList(); + List sfs = new ArrayList(newFiles.size()); for (Path newFile : newFiles) { assert newFile != null; StoreFile sf = moveFileIntoPlace(newFile); @@ -1130,7 +1136,7 @@ public class HStore implements Store { private void writeCompactionWalRecord(Collection filesCompacted, Collection newFiles) throws IOException { if (region.getLog() == null) return; - List inputPaths = new ArrayList(); + List inputPaths = new ArrayList(filesCompacted.size()); for (StoreFile f : filesCompacted) { inputPaths.add(f.getPath()); } @@ -1202,7 +1208,7 @@ public class HStore implements Store { /** * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. - * See HBASE-2331. + * See HBASE-2231. * @param compaction */ @Override @@ -1210,43 +1216,37 @@ public class HStore implements Store { throws IOException { LOG.debug("Completing compaction from the WAL marker"); List compactionInputs = compaction.getCompactionInputList(); - List compactionOutputs = compaction.getCompactionOutputList(); - List outputStoreFiles = new ArrayList(compactionOutputs.size()); - for (String compactionOutput : compactionOutputs) { - //we should have this store file already - boolean found = false; - Path outputPath = new Path(fs.getStoreDir(family.getNameAsString()), compactionOutput); - outputPath = outputPath.makeQualified(fs.getFileSystem()); - for (StoreFile sf : this.getStorefiles()) { - if (sf.getPath().makeQualified(sf.getPath().getFileSystem(conf)).equals(outputPath)) { - found = true; - break; - } - } - if (!found) { - if (getFileSystem().exists(outputPath)) { - outputStoreFiles.add(createStoreFileAndReader(outputPath)); - } - } - } + // The Compaction Marker is written after the compaction is completed, + // and the files moved into the region/family folder. + // + // If we crash after the entry is written, we may not have removed the + // input files, but the output file is present. + // (The unremoved input files will be removed by this function) + // + // If we scan the directory and the file is not present, it can mean that: + // - The file was manually removed by the user + // - The file was removed as consequence of subsequent compaction + // so, we can't do anything with the "compaction output list" because those + // files have already been loaded when opening the region (by virtue of + // being in the store's folder) or they may be missing due to a compaction. + String familyName = this.getColumnFamilyName(); List inputPaths = new ArrayList(compactionInputs.size()); for (String compactionInput : compactionInputs) { - Path inputPath = new Path(fs.getStoreDir(family.getNameAsString()), compactionInput); - inputPath = inputPath.makeQualified(fs.getFileSystem()); + Path inputPath = fs.getStoreFilePath(familyName, compactionInput); inputPaths.add(inputPath); } //some of the input files might already be deleted List inputStoreFiles = new ArrayList(compactionInputs.size()); for (StoreFile sf : this.getStorefiles()) { - if (inputPaths.contains(sf.getPath().makeQualified(fs.getFileSystem()))) { + if (inputPaths.contains(sf.getQualifiedPath())) { inputStoreFiles.add(sf); } } - this.replaceStoreFiles(inputStoreFiles, outputStoreFiles); + this.replaceStoreFiles(inputStoreFiles, Collections.EMPTY_LIST); this.completeCompaction(inputStoreFiles); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index 31a26ab8eae..ebbc83667b5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -233,6 +233,13 @@ public class StoreFile { return this.fileInfo.getPath(); } + /** + * @return Returns the qualified path of this StoreFile + */ + public Path getQualifiedPath() { + return this.fileInfo.getPath().makeQualified(fs); + } + /** * @return True if this is a StoreFile Reference; call after {@link #open()} * else may get wrong answer.