Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4848: Use Java 7 NIO2-FileChannel instead of RandomAccessFile for NIOFSDirectory and MMapDirectory 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1459437 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2013-03-21 18:19:19 +00:00
parent 27f5cd5fb8
commit 6385447660
6 changed files with 132 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/CHANGES.txt
View File

@ -24,6 +24,13 @@ New Features
* LUCENE-4747: Move to Java 7 as minimum Java version.
(Robert Muir, Uwe Schindler)
Optimizations
* LUCENE-4848: Use Java 7 NIO2-FileChannel instead of RandomAccessFile
for NIOFSDirectory and MMapDirectory. This allows to delete open files
on Windows if NIOFSDirectory is used, mmapped files are still locked.
(Michael Poindexter, Robert Muir, Uwe Schindler)
======================= Lucene 4.3.0 =======================
Changes in backwards compatibility policy

69
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -298,8 +298,12 @@ public abstract class FSDirectory extends Directory {
throw new IOException("Cannot overwrite: " + file);
}
protected void onIndexOutputClosed(FSIndexOutput io) {
staleFiles.add(io.name);
/**
* Sub classes should call this method on closing an open {@link IndexOutput}, reporting the name of the file
* that was closed. {@code FSDirectory} needs this information to take care of syncing stale files.
*/
protected void onIndexOutputClosed(String name) {
staleFiles.add(name);
}
@Override
@ -392,65 +396,6 @@ public abstract class FSDirectory extends Directory {
return chunkSize;
}
/** Base class for reading input from a RandomAccessFile */
protected abstract static class FSIndexInput extends BufferedIndexInput {
/** the underlying RandomAccessFile */
protected final RandomAccessFile file;
boolean isClone = false;
/** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
protected final int chunkSize;
/** start offset: non-zero in the slice case */
protected final long off;
/** end offset (start+length) */
protected final long end;
/** Create a new FSIndexInput, reading the entire file from <code>path</code> */
protected FSIndexInput(String resourceDesc, File path, IOContext context, int chunkSize) throws IOException {
super(resourceDesc, context);
this.file = new RandomAccessFile(path, "r");
this.chunkSize = chunkSize;
this.off = 0L;
this.end = file.length();
}
/** Create a new FSIndexInput, representing a slice of an existing open <code>file</code> */
protected FSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
super(resourceDesc, bufferSize);
this.file = file;
this.chunkSize = chunkSize;
this.off = off;
this.end = off + length;
this.isClone = true; // well, we are sorta?
}
@Override
public void close() throws IOException {
// only close the file if this is not a clone
if (!isClone) {
file.close();
}
}
@Override
public FSIndexInput clone() {
FSIndexInput clone = (FSIndexInput)super.clone();
clone.isClone = true;
return clone;
}
@Override
public final long length() {
return end - off;
}
/** Method used for testing. Returns true if the underlying
* file descriptor is valid.
*/
boolean isFDValid() throws IOException {
return file.getFD().valid();
}
}
/**
* Writes output with {@link RandomAccessFile#write(byte[], int, int)}
*/
@ -476,7 +421,7 @@ public abstract class FSDirectory extends Directory {
@Override
public void close() throws IOException {
parent.onIndexOutputClosed(this);
parent.onIndexOutputClosed(name);
// only close the file if it has not been closed yet
if (isOpen) {
boolean success = false;

22
lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
View File

@ -19,11 +19,11 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.io.File;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.StandardOpenOption;
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
@ -189,12 +189,9 @@ public class MMapDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
File f = new File(getDirectory(), name);
RandomAccessFile raf = new RandomAccessFile(f, "r");
try {
return new MMapIndexInput("MMapIndexInput(path=\"" + f + "\")", raf);
} finally {
raf.close();
File file = new File(getDirectory(), name);
try (FileChannel c = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
return new MMapIndexInput("MMapIndexInput(path=\"" + file.toString() + "\")", c);
}
}
@ -218,8 +215,8 @@ public class MMapDirectory extends FSDirectory {
private final class MMapIndexInput extends ByteBufferIndexInput {
private final boolean useUnmapHack;
MMapIndexInput(String resourceDescription, RandomAccessFile raf) throws IOException {
super(resourceDescription, map(raf, 0, raf.length()), raf.length(), chunkSizePower, getUseUnmap());
MMapIndexInput(String resourceDescription, FileChannel fc) throws IOException {
super(resourceDescription, map(fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
this.useUnmapHack = getUseUnmap();
}
@ -256,9 +253,9 @@ public class MMapDirectory extends FSDirectory {
}
/** Maps a file into a set of buffers */
ByteBuffer[] map(RandomAccessFile raf, long offset, long length) throws IOException {
ByteBuffer[] map(FileChannel fc, long offset, long length) throws IOException {
if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString());
throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + fc.toString());
final long chunkSize = 1L << chunkSizePower;
@ -268,13 +265,12 @@ public class MMapDirectory extends FSDirectory {
ByteBuffer buffers[] = new ByteBuffer[nrBuffers];
long bufferStart = 0L;
FileChannel rafc = raf.getChannel();
for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
int bufSize = (int) ( (length > (bufferStart + chunkSize))
? chunkSize
: (length - bufferStart)
);
buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
bufferStart += bufSize;
}

64
lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
View File

@ -20,10 +20,10 @@ package org.apache.lucene.store;
import java.io.File;
import java.io.EOFException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.Future; // javadoc
/**
@ -77,7 +77,9 @@ public class NIOFSDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize());
File path = new File(getDirectory(), name);
FileChannel fc = FileChannel.open(path.toPath(), StandardOpenOption.READ);
return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context, getReadChunkSize());
}
@Override
@ -85,7 +87,7 @@ public class NIOFSDirectory extends FSDirectory {
final IOContext context) throws IOException {
ensureOpen();
final File path = new File(getDirectory(), name);
final RandomAccessFile descriptor = new RandomAccessFile(path, "r");
final FileChannel descriptor = FileChannel.open(path.toPath(), StandardOpenOption.READ);
return new Directory.IndexInputSlicer() {
@Override
@ -95,7 +97,7 @@ public class NIOFSDirectory extends FSDirectory {
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new NIOFSIndexInput(sliceDescription, path, descriptor, descriptor.getChannel(), offset,
return new NIOFSIndexInput("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
length, BufferedIndexInput.bufferSize(context), getReadChunkSize());
}
};
@ -104,21 +106,54 @@ public class NIOFSDirectory extends FSDirectory {
/**
* Reads bytes with {@link FileChannel#read(ByteBuffer, long)}
*/
protected static class NIOFSIndexInput extends FSIndexInput {
protected static class NIOFSIndexInput extends BufferedIndexInput {
/** the file channel we will read from */
protected final FileChannel channel;
/** is this instance a clone and hence does not own the file to close it */
boolean isClone = false;
/** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
protected final int chunkSize;
/** start offset: non-zero in the slice case */
protected final long off;
/** end offset (start+length) */
protected final long end;
private ByteBuffer byteBuf; // wraps the buffer for NIO
final FileChannel channel;
public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
super("NIOFSIndexInput(path=\"" + path + "\")", path, context, chunkSize);
channel = file.getChannel();
public NIOFSIndexInput(String resourceDesc, FileChannel fc, IOContext context, int chunkSize) throws IOException {
super(resourceDesc, context);
this.channel = fc;
this.chunkSize = chunkSize;
this.off = 0L;
this.end = fc.size();
}
public NIOFSIndexInput(String sliceDescription, File path, RandomAccessFile file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
super("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + off + ":" + (off+length) + ")", file, off, length, bufferSize, chunkSize);
channel = fc;
isClone = true;
public NIOFSIndexInput(String resourceDesc, FileChannel fc, long off, long length, int bufferSize, int chunkSize) {
super(resourceDesc, bufferSize);
this.channel = fc;
this.chunkSize = chunkSize;
this.off = off;
this.end = off + length;
this.isClone = true;
}
@Override
public void close() throws IOException {
if (!isClone) {
channel.close();
}
}
@Override
public NIOFSIndexInput clone() {
NIOFSIndexInput clone = (NIOFSIndexInput)super.clone();
clone.isClone = true;
return clone;
}
@Override
public final long length() {
return end - off;
}
@Override
@ -186,5 +221,4 @@ public class NIOFSDirectory extends FSDirectory {
@Override
protected void seekInternal(long pos) throws IOException {}
}
}

53
lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
View File

@ -55,7 +55,8 @@ public class SimpleFSDirectory extends FSDirectory {
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
final File path = new File(directory, name);
return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", path, context, getReadChunkSize());
RandomAccessFile raf = new RandomAccessFile(path, "r");
return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context, getReadChunkSize());
}
@Override
@ -83,14 +84,52 @@ public class SimpleFSDirectory extends FSDirectory {
* Reads bytes with {@link RandomAccessFile#seek(long)} followed by
* {@link RandomAccessFile#read(byte[], int, int)}.
*/
protected static class SimpleFSIndexInput extends FSIndexInput {
protected static class SimpleFSIndexInput extends BufferedIndexInput {
/** the file channel we will read from */
protected final RandomAccessFile file;
/** is this instance a clone and hence does not own the file to close it */
boolean isClone = false;
/** maximum read length on a 32bit JVM to prevent incorrect OOM, see LUCENE-1566 */
protected final int chunkSize;
/** start offset: non-zero in the slice case */
protected final long off;
/** end offset (start+length) */
protected final long end;
public SimpleFSIndexInput(String resourceDesc, File path, IOContext context, int chunkSize) throws IOException {
super(resourceDesc, path, context, chunkSize);
public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, IOContext context, int chunkSize) throws IOException {
super(resourceDesc, context);
this.file = file;
this.chunkSize = chunkSize;
this.off = 0L;
this.end = file.length();
}
public SimpleFSIndexInput(String resourceDesc, RandomAccessFile file, long off, long length, int bufferSize, int chunkSize) {
super(resourceDesc, file, off, length, bufferSize, chunkSize);
super(resourceDesc, bufferSize);
this.file = file;
this.chunkSize = chunkSize;
this.off = off;
this.end = off + length;
this.isClone = true;
}
@Override
public void close() throws IOException {
if (!isClone) {
file.close();
}
}
@Override
public SimpleFSIndexInput clone() {
SimpleFSIndexInput clone = (SimpleFSIndexInput)super.clone();
clone.isClone = true;
return clone;
}
@Override
public final long length() {
return end - off;
}
/** IndexInput methods */
@ -136,5 +175,9 @@ public class SimpleFSDirectory extends FSDirectory {
@Override
protected void seekInternal(long position) {
}
boolean isFDValid() throws IOException {
return file.getFD().valid();
}
}
}

15
lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
View File

@ -21,6 +21,9 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -93,16 +96,16 @@ public class TestBufferedIndexInput extends LuceneTestCase {
writeBytes(tmpInputFile, TEST_FILE_LENGTH);
// run test with chunk size of 10 bytes
runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")", tmpInputFile,
newIOContext(random()), 10), inputBufferSize, random());
runReadBytesAndClose(new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + tmpInputFile + "\")",
new RandomAccessFile(tmpInputFile, "r"), newIOContext(random()), 10), inputBufferSize, random());
// run test with chunk size of 10 bytes
runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile,
newIOContext(random()), 10), inputBufferSize, random());
runReadBytesAndClose(new NIOFSIndexInput("NIOFSIndexInput(path=\"" + tmpInputFile + "\")",
FileChannel.open(tmpInputFile.toPath(), StandardOpenOption.READ), newIOContext(random()), 10),
inputBufferSize, random());
}
private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r)
throws IOException {
private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r) throws IOException {
try {
runReadBytes(input, bufferSize, r);
} finally {