LUCENE-4848: Use Java 7 NIO2-FileChannel instead of RandomAccessFile for NIOFSDirectory and MMapDirectory

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1459437 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -24,6 +24,13 @@ New Features
 * LUCENE-4747: Move to Java 7 as minimum Java version.
   (Robert Muir, Uwe Schindler)
 
+Optimizations
+
+* LUCENE-4848: Use Java 7 NIO2-FileChannel instead of RandomAccessFile
+  for NIOFSDirectory and MMapDirectory. This allows to delete open files
+  on Windows if NIOFSDirectory is used, mmapped files are still locked.
+  (Michael Poindexter, Robert Muir, Uwe Schindler)
+
 ======================= Lucene 4.3.0 =======================
 
 Changes in backwards compatibility policy

lucene/core/src/java/org/apache/lucene/store/FSDirectory.java

@@ -298,8 +298,12 @@ public abstract class FSDirectory extends Directory {
       throw new IOException("Cannot overwrite: " + file);
   }
 
-  protected void onIndexOutputClosed(FSIndexOutput io) {
+  /**
-    staleFiles.add(io.name);
+   * Sub classes should call this method on closing an open {@link IndexOutput}, reporting the name of the file
+   * that was closed. {@code FSDirectory} needs this information to take care of syncing stale files.
+   */
+  protected void onIndexOutputClosed(String name) {
+    staleFiles.add(name);
   }
 
   @Override
@@ -392,65 +396,6 @@ public abstract class FSDirectory extends Directory {
     return chunkSize;
   }
 
-  /** Base class for reading input from a RandomAccessFile */
-  protected abstract static class FSIndexInput extends BufferedIndexInput {
-    /** the underlying RandomAccessFile */
-    protected final RandomAccessFile file;
-    boolean isClone = false;
-    /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */ 
-    protected final int chunkSize;
-    /** start offset: non-zero in the slice case */
-    protected final long off;
-    /** end offset (start+length) */
-    protected final long end;
-    
-    /** Create a new FSIndexInput, reading the entire file from <code>path</code> */
-    protected FSIndexInput(String resourceDesc, File path, IOContext context, int chunkSize) throws IOException {
-      super(resourceDesc, context);
-      this.file = new RandomAccessFile(path, "r"); 
-      this.chunkSize = chunkSize;
-      this.off = 0L;
-      this.end = file.length();
-    }
-    
-    /** Create a new FSIndexInput, representing a slice of an existing open <code>file</code> */
-    protected FSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
-      super(resourceDesc, bufferSize);
-      this.file = file;
-      this.chunkSize = chunkSize;
-      this.off = off;
-      this.end = off + length;
-      this.isClone = true; // well, we are sorta?
-    }
-    
-    @Override
-    public void close() throws IOException {
-      // only close the file if this is not a clone
-      if (!isClone) {
-        file.close();
-      }
-    }
-    
-    @Override
-    public FSIndexInput clone() {
-      FSIndexInput clone = (FSIndexInput)super.clone();
-      clone.isClone = true;
-      return clone;
-    }
-    
-    @Override
-    public final long length() {
-      return end - off;
-    }
-    
-    /** Method used for testing. Returns true if the underlying
-     *  file descriptor is valid.
-     */
-    boolean isFDValid() throws IOException {
-      return file.getFD().valid();
-    }
-  }
-  
   /**
    * Writes output with {@link RandomAccessFile#write(byte[], int, int)}
    */
@@ -476,7 +421,7 @@ public abstract class FSDirectory extends Directory {
     
     @Override
     public void close() throws IOException {
-      parent.onIndexOutputClosed(this);
+      parent.onIndexOutputClosed(name);
       // only close the file if it has not been closed yet
       if (isOpen) {
         boolean success = false;

lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java

@@ -19,11 +19,11 @@ package org.apache.lucene.store;
  
 import java.io.IOException;
 import java.io.File;
-import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException; // javadoc @link
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileChannel.MapMode;
+import java.nio.file.StandardOpenOption;
 
 import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
@@ -189,12 +189,9 @@ public class MMapDirectory extends FSDirectory {
   @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
-    File f = new File(getDirectory(), name);
+    File file = new File(getDirectory(), name);
-    RandomAccessFile raf = new RandomAccessFile(f, "r");
+    try (FileChannel c = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
-    try {
+      return new MMapIndexInput("MMapIndexInput(path=\"" + file.toString() + "\")", c);
-      return new MMapIndexInput("MMapIndexInput(path=\"" + f + "\")", raf);
-    } finally {
-      raf.close();
     }
   }
   
@@ -218,8 +215,8 @@ public class MMapDirectory extends FSDirectory {
   private final class MMapIndexInput extends ByteBufferIndexInput {
     private final boolean useUnmapHack;
     
-    MMapIndexInput(String resourceDescription, RandomAccessFile raf) throws IOException {
+    MMapIndexInput(String resourceDescription, FileChannel fc) throws IOException {
-      super(resourceDescription, map(raf, 0, raf.length()), raf.length(), chunkSizePower, getUseUnmap());
+      super(resourceDescription, map(fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
       this.useUnmapHack = getUseUnmap();
     }
     
@@ -256,9 +253,9 @@ public class MMapDirectory extends FSDirectory {
   }
   
   /** Maps a file into a set of buffers */
-  ByteBuffer[] map(RandomAccessFile raf, long offset, long length) throws IOException {
+  ByteBuffer[] map(FileChannel fc, long offset, long length) throws IOException {
     if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
-      throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString());
+      throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + fc.toString());
     
     final long chunkSize = 1L << chunkSizePower;
     
@@ -268,13 +265,12 @@ public class MMapDirectory extends FSDirectory {
     ByteBuffer buffers[] = new ByteBuffer[nrBuffers];
     
     long bufferStart = 0L;
-    FileChannel rafc = raf.getChannel();
     for (int bufNr = 0; bufNr < nrBuffers; bufNr++) { 
       int bufSize = (int) ( (length > (bufferStart + chunkSize))
           ? chunkSize
               : (length - bufferStart)
           );
-      buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
+      buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
       bufferStart += bufSize;
     }
     

lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java

@@ -20,10 +20,10 @@ package org.apache.lucene.store;
 import java.io.File;
 import java.io.EOFException;
 import java.io.IOException;
-import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException; // javadoc @link
 import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
 import java.util.concurrent.Future; // javadoc
 
 /**
@@ -77,7 +77,9 @@ public class NIOFSDirectory extends FSDirectory {
   @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
-    return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize());
+    File path = new File(getDirectory(), name);
+    FileChannel fc = FileChannel.open(path.toPath(), StandardOpenOption.READ);
+    return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context, getReadChunkSize());
   }
   
   @Override
@@ -85,7 +87,7 @@ public class NIOFSDirectory extends FSDirectory {
       final IOContext context) throws IOException {
     ensureOpen();
     final File path = new File(getDirectory(), name);
-    final RandomAccessFile descriptor = new RandomAccessFile(path, "r");
+    final FileChannel descriptor = FileChannel.open(path.toPath(), StandardOpenOption.READ);
     return new Directory.IndexInputSlicer() {
 
       @Override
@@ -95,7 +97,7 @@ public class NIOFSDirectory extends FSDirectory {
 
       @Override
       public IndexInput openSlice(String sliceDescription, long offset, long length) {
-        return new NIOFSIndexInput(sliceDescription, path, descriptor, descriptor.getChannel(), offset,
+        return new NIOFSIndexInput("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
             length, BufferedIndexInput.bufferSize(context), getReadChunkSize());
       }
     };
@@ -104,21 +106,54 @@ public class NIOFSDirectory extends FSDirectory {
   /**
    * Reads bytes with {@link FileChannel#read(ByteBuffer, long)}
    */
-  protected static class NIOFSIndexInput extends FSIndexInput {
+  protected static class NIOFSIndexInput extends BufferedIndexInput {
+    /** the file channel we will read from */
+    protected final FileChannel channel;
+    /** is this instance a clone and hence does not own the file to close it */
+    boolean isClone = false;
+    /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */ 
+    protected final int chunkSize;
+    /** start offset: non-zero in the slice case */
+    protected final long off;
+    /** end offset (start+length) */
+    protected final long end;
     
     private ByteBuffer byteBuf; // wraps the buffer for NIO
 
-    final FileChannel channel;
+    public NIOFSIndexInput(String resourceDesc, FileChannel fc, IOContext context, int chunkSize) throws IOException {
-
+      super(resourceDesc, context);
-    public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
+      this.channel = fc; 
-      super("NIOFSIndexInput(path=\"" + path + "\")", path, context, chunkSize);
+      this.chunkSize = chunkSize;
-      channel = file.getChannel();
+      this.off = 0L;
+      this.end = fc.size();
     }
     
-    public NIOFSIndexInput(String sliceDescription, File path, RandomAccessFile file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
+    public NIOFSIndexInput(String resourceDesc, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
-      super("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + off + ":" + (off+length) + ")", file, off, length, bufferSize, chunkSize);
+      super(resourceDesc, bufferSize);
-      channel = fc;
+      this.channel = fc;
-      isClone = true;
+      this.chunkSize = chunkSize;
+      this.off = off;
+      this.end = off + length;
+      this.isClone = true;
+    }
+    
+    @Override
+    public void close() throws IOException {
+      if (!isClone) {
+        channel.close();
+      }
+    }
+    
+    @Override
+    public NIOFSIndexInput clone() {
+      NIOFSIndexInput clone = (NIOFSIndexInput)super.clone();
+      clone.isClone = true;
+      return clone;
+    }
+    
+    @Override
+    public final long length() {
+      return end - off;
     }
 
     @Override
@@ -186,5 +221,4 @@ public class NIOFSDirectory extends FSDirectory {
     @Override
     protected void seekInternal(long pos) throws IOException {}
   }
-
 }

lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java

@@ -55,7 +55,8 @@ public class SimpleFSDirectory extends FSDirectory {
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
     final File path = new File(directory, name);
-    return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", path, context, getReadChunkSize());
+    RandomAccessFile raf = new RandomAccessFile(path, "r");
+    return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context, getReadChunkSize());
   }
 
   @Override
@@ -83,14 +84,52 @@ public class SimpleFSDirectory extends FSDirectory {
    * Reads bytes with {@link RandomAccessFile#seek(long)} followed by
    * {@link RandomAccessFile#read(byte[], int, int)}.  
    */
-  protected static class SimpleFSIndexInput extends FSIndexInput {
+  protected static class SimpleFSIndexInput extends BufferedIndexInput {
+    /** the file channel we will read from */
+    protected final RandomAccessFile file;
+    /** is this instance a clone and hence does not own the file to close it */
+    boolean isClone = false;
+    /** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */ 
+    protected final int chunkSize;
+    /** start offset: non-zero in the slice case */
+    protected final long off;
+    /** end offset (start+length) */
+    protected final long end;
     
-    public SimpleFSIndexInput(String resourceDesc, File path, IOContext context, int chunkSize) throws IOException {
+    public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, IOContext context, int chunkSize) throws IOException {
-      super(resourceDesc, path, context, chunkSize);
+      super(resourceDesc, context);
+      this.file = file; 
+      this.chunkSize = chunkSize;
+      this.off = 0L;
+      this.end = file.length();
     }
     
     public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
-      super(resourceDesc, file, off, length, bufferSize, chunkSize);
+      super(resourceDesc, bufferSize);
+      this.file = file;
+      this.chunkSize = chunkSize;
+      this.off = off;
+      this.end = off + length;
+      this.isClone = true;
+    }
+    
+    @Override
+    public void close() throws IOException {
+      if (!isClone) {
+        file.close();
+      }
+    }
+    
+    @Override
+    public SimpleFSIndexInput clone() {
+      SimpleFSIndexInput clone = (SimpleFSIndexInput)super.clone();
+      clone.isClone = true;
+      return clone;
+    }
+    
+    @Override
+    public final long length() {
+      return end - off;
     }
   
     /** IndexInput methods */
@@ -136,5 +175,9 @@ public class SimpleFSDirectory extends FSDirectory {
     @Override
     protected void seekInternal(long position) {
     }
+    
+    boolean isFDValid() throws IOException {
+      return file.getFD().valid();
+    }
   }
 }

lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java

@@ -21,6 +21,9 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -93,16 +96,16 @@ public class TestBufferedIndexInput extends LuceneTestCase {
     writeBytes(tmpInputFile, TEST_FILE_LENGTH);
 
     // run test with chunk size of 10 bytes
-    runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")", tmpInputFile,
+    runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")", 
-        newIOContext(random()), 10), inputBufferSize, random());
+        new RandomAccessFile(tmpInputFile, "r"), newIOContext(random()), 10), inputBufferSize, random());
 
     // run test with chunk size of 10 bytes
-    runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
+    runReadBytesAndClose(new NIOFSIndexInput("NIOFSIndexInput(path=\"" + tmpInputFile + "\")", 
-        newIOContext(random()), 10), inputBufferSize, random());
+        FileChannel.open(tmpInputFile.toPath(), StandardOpenOption.READ), newIOContext(random()), 10), 
+        inputBufferSize, random());
   }
 
-  private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
+  private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r) throws IOException {
-      throws IOException {
     try {
       runReadBytes(input, bufferSize, r);
     } finally {