LUCENE-4371: Replace IndexInputSlicer with IndexInput.slice

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1595480 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-05-17 13:44:09 +00:00
parent 5cdde67569
commit 7aeac4f07b
30 changed files with 182 additions and 386 deletions

View File

@ -77,6 +77,9 @@ API Changes
takes the same selectors. Add helper methods to DocValues.java that are better takes the same selectors. Add helper methods to DocValues.java that are better
suited for search code (never return null, etc). (Mike McCandless, Robert Muir) suited for search code (never return null, etc). (Mike McCandless, Robert Muir)
* LUCENE-4371: Removed IndexInputSlicer and Directory.createSlicer() and replaced
with IndexInput.slice(). (Robert Muir)
Documentation Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect * LUCENE-5392: Add/improve analysis package documentation to reflect

View File

@ -312,5 +312,9 @@ public abstract class MultiLevelSkipListReader implements Closeable {
this.pos = (int) (pos - pointer); this.pos = (int) (pos - pointer);
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException();
}
} }
} }

View File

@ -74,4 +74,9 @@ public class BufferedChecksumIndexInput extends ChecksumIndexInput {
public IndexInput clone() { public IndexInput clone() {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException();
}
} }

View File

@ -315,6 +315,11 @@ public abstract class BufferedIndexInput extends IndexInput {
return clone; return clone;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return wrap("SlicedIndexInput(" + sliceDescription + " in " + this + ")", this, offset, length);
}
/** /**
* Flushes the in-memory buffer to the given output, copying at most * Flushes the in-memory buffer to the given output, copying at most
@ -349,4 +354,62 @@ public abstract class BufferedIndexInput extends IndexInput {
} }
} }
/**
* Wraps a portion of a file with buffering.
*/
public static BufferedIndexInput wrap(String description, IndexInput other, long offset, long length) {
return new SlicedIndexInput(description, other, offset, length);
}
/**
* Implementation of an IndexInput that reads from a portion of a file.
*/
private static final class SlicedIndexInput extends BufferedIndexInput {
IndexInput base;
long fileOffset;
long length;
SlicedIndexInput(String sliceDescription, IndexInput base, long offset, long length) {
super("SlicedIndexInput(" + sliceDescription + " in " + base + " slice=" + offset + ":" + (offset+length) + ")", BufferedIndexInput.BUFFER_SIZE);
if (offset < 0 || length < 0) {
throw new IllegalArgumentException();
}
assert offset + length <= base.length();
this.base = base.clone();
this.fileOffset = offset;
this.length = length;
}
@Override
public SlicedIndexInput clone() {
SlicedIndexInput clone = (SlicedIndexInput)super.clone();
clone.base = base.clone();
clone.fileOffset = fileOffset;
clone.length = length;
return clone;
}
@Override
protected void readInternal(byte[] b, int offset, int len) throws IOException {
long start = getFilePointer();
if (start + len > length) {
throw new EOFException("read past EOF: " + this);
}
base.seek(fileOffset + start);
base.readBytes(b, offset, len, false);
}
@Override
protected void seekInternal(long pos) {}
@Override
public void close() throws IOException {
base.close();
}
@Override
public long length() {
return length;
}
}
} }

View File

@ -197,10 +197,8 @@ abstract class ByteBufferIndexInput extends IndexInput {
/** /**
* Creates a slice of this index input, with the given description, offset, and length. The slice is seeked to the beginning. * Creates a slice of this index input, with the given description, offset, and length. The slice is seeked to the beginning.
*/ */
@Override
public final ByteBufferIndexInput slice(String sliceDescription, long offset, long length) { public final ByteBufferIndexInput slice(String sliceDescription, long offset, long length) {
if (isClone) { // well we could, but this is stupid
throw new IllegalStateException("cannot slice() " + sliceDescription + " from a cloned IndexInput: " + this);
}
final ByteBufferIndexInput clone = buildSlice(offset, length); final ByteBufferIndexInput clone = buildSlice(offset, length);
clone.sliceDescription = sliceDescription; clone.sliceDescription = sliceDescription;
try { try {

View File

@ -87,7 +87,7 @@ public final class CompoundFileDirectory extends BaseDirectory {
private final boolean openForWrite; private final boolean openForWrite;
private static final Map<String,FileEntry> SENTINEL = Collections.emptyMap(); private static final Map<String,FileEntry> SENTINEL = Collections.emptyMap();
private final CompoundFileWriter writer; private final CompoundFileWriter writer;
private final IndexInputSlicer handle; private final IndexInput handle;
private int version; private int version;
/** /**
@ -101,7 +101,7 @@ public final class CompoundFileDirectory extends BaseDirectory {
this.openForWrite = openForWrite; this.openForWrite = openForWrite;
if (!openForWrite) { if (!openForWrite) {
boolean success = false; boolean success = false;
handle = directory.createSlicer(fileName, context); handle = directory.openInput(fileName, context);
try { try {
this.entries = readEntries(directory, fileName); this.entries = readEntries(directory, fileName);
success = true; success = true;
@ -192,7 +192,7 @@ public final class CompoundFileDirectory extends BaseDirectory {
if (entry == null) { if (entry == null) {
throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")"); throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
} }
return handle.openSlice(name, entry.offset, entry.length); return handle.slice(name, entry.offset, entry.length);
} }
/** Returns an array of strings, one for each file in the directory. */ /** Returns an array of strings, one for each file in the directory. */
@ -258,28 +258,6 @@ public final class CompoundFileDirectory extends BaseDirectory {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public IndexInputSlicer createSlicer(final String name, IOContext context)
throws IOException {
ensureOpen();
assert !openForWrite;
final String id = IndexFileNames.stripSegmentName(name);
final FileEntry entry = entries.get(id);
if (entry == null) {
throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
}
return new IndexInputSlicer() {
@Override
public void close() {
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException {
return handle.openSlice(sliceDescription, entry.offset + offset, length);
}
};
}
@Override @Override
public String toString() { public String toString() {
return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")"; return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")";

View File

@ -17,7 +17,6 @@ package org.apache.lucene.store;
* limitations under the License. * limitations under the License.
*/ */
import java.io.EOFException;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.Closeable; import java.io.Closeable;
@ -200,113 +199,8 @@ public abstract class Directory implements Closeable {
} }
} }
/**
* Creates an {@link IndexInputSlicer} for the given file name.
* IndexInputSlicer allows other {@link Directory} implementations to
* efficiently open one or more sliced {@link IndexInput} instances from a
* single file handle. The underlying file handle is kept open until the
* {@link IndexInputSlicer} is closed.
* <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException}
* if the file does not exist.
*
* @throws IOException
* if an {@link IOException} occurs
* @lucene.internal
* @lucene.experimental
*/
public IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
ensureOpen();
return new IndexInputSlicer() {
private final IndexInput base = Directory.this.openInput(name, context);
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new SlicedIndexInput("SlicedIndexInput(" + sliceDescription + " in " + base + ")", base, offset, length);
}
@Override
public void close() throws IOException {
base.close();
}
};
}
/** /**
* @throws AlreadyClosedException if this Directory is closed * @throws AlreadyClosedException if this Directory is closed
*/ */
protected void ensureOpen() throws AlreadyClosedException {} protected void ensureOpen() throws AlreadyClosedException {}
/**
* Allows to create one or more sliced {@link IndexInput} instances from a single
* file handle. Some {@link Directory} implementations may be able to efficiently map slices of a file
* into memory when only certain parts of a file are required.
* @lucene.internal
* @lucene.experimental
*/
public abstract class IndexInputSlicer implements Closeable {
/**
* Returns an {@link IndexInput} slice starting at the given offset with the given length.
*/
public abstract IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException;
}
/** Implementation of an IndexInput that reads from a portion of
* a file.
*/
private static final class SlicedIndexInput extends BufferedIndexInput {
IndexInput base;
long fileOffset;
long length;
SlicedIndexInput(final String sliceDescription, final IndexInput base, final long fileOffset, final long length) {
this(sliceDescription, base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE);
}
SlicedIndexInput(final String sliceDescription, final IndexInput base, final long fileOffset, final long length, int readBufferSize) {
super("SlicedIndexInput(" + sliceDescription + " in " + base + " slice=" + fileOffset + ":" + (fileOffset+length) + ")", readBufferSize);
this.base = base.clone();
this.fileOffset = fileOffset;
this.length = length;
}
@Override
public SlicedIndexInput clone() {
SlicedIndexInput clone = (SlicedIndexInput)super.clone();
clone.base = base.clone();
clone.fileOffset = fileOffset;
clone.length = length;
return clone;
}
/** Expert: implements buffer refill. Reads bytes from the current
* position in the input.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param len the number of bytes to read
*/
@Override
protected void readInternal(byte[] b, int offset, int len) throws IOException {
long start = getFilePointer();
if(start + len > length)
throw new EOFException("read past EOF: " + this);
base.seek(fileOffset + start);
base.readBytes(b, offset, len, false);
}
/** Expert: implements seek. Sets current position in this file, where
* the next {@link #readInternal(byte[],int,int)} will occur.
* @see #readInternal(byte[],int,int)
*/
@Override
protected void seekInternal(long pos) {}
/** Closes the stream to further operations. */
@Override
public void close() throws IOException {
base.close();
}
@Override
public long length() {
return length;
}
}
} }

View File

@ -166,10 +166,4 @@ public class FileSwitchDirectory extends BaseDirectory {
public IndexInput openInput(String name, IOContext context) throws IOException { public IndexInput openInput(String name, IOContext context) throws IOException {
return getDirectory(name).openInput(name, context); return getDirectory(name).openInput(name, context);
} }
@Override
public IndexInputSlicer createSlicer(String name, IOContext context)
throws IOException {
return getDirectory(name).createSlicer(name, context);
}
} }

View File

@ -82,4 +82,10 @@ public abstract class IndexInput extends DataInput implements Cloneable,Closeabl
public IndexInput clone() { public IndexInput clone() {
return (IndexInput) super.clone(); return (IndexInput) super.clone();
} }
/**
* Creates a slice of this index input, with the given description, offset, and length.
* The slice is seeked to the beginning.
*/
public abstract IndexInput slice(String sliceDescription, long offset, long length) throws IOException;
} }

View File

@ -196,23 +196,6 @@ public class MMapDirectory extends FSDirectory {
return new MMapIndexInput("MMapIndexInput(path=\"" + file.toString() + "\")", c); return new MMapIndexInput("MMapIndexInput(path=\"" + file.toString() + "\")", c);
} }
} }
@Override
public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException {
final MMapIndexInput full = (MMapIndexInput) openInput(name, context);
return new IndexInputSlicer() {
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException {
ensureOpen();
return full.slice(sliceDescription, offset, length);
}
@Override
public void close() throws IOException {
full.close();
}
};
}
private final class MMapIndexInput extends ByteBufferIndexInput { private final class MMapIndexInput extends ByteBufferIndexInput {
private final boolean useUnmapHack; private final boolean useUnmapHack;

View File

@ -20,7 +20,6 @@ package org.apache.lucene.store;
import java.io.File; import java.io.File;
import java.io.EOFException; import java.io.EOFException;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
@ -83,31 +82,10 @@ public class NIOFSDirectory extends FSDirectory {
return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context); return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context);
} }
@Override
public IndexInputSlicer createSlicer(final String name,
final IOContext context) throws IOException {
ensureOpen();
final File path = new File(getDirectory(), name);
final FileChannel descriptor = FileChannel.open(path.toPath(), StandardOpenOption.READ);
return new Directory.IndexInputSlicer() {
@Override
public void close() throws IOException {
descriptor.close();
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new NIOFSIndexInput("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
length, BufferedIndexInput.bufferSize(context));
}
};
}
/** /**
* Reads bytes with {@link FileChannel#read(ByteBuffer, long)} * Reads bytes with {@link FileChannel#read(ByteBuffer, long)}
*/ */
protected static class NIOFSIndexInput extends BufferedIndexInput { static final class NIOFSIndexInput extends BufferedIndexInput {
/** /**
* The maximum chunk size for reads of 16384 bytes. * The maximum chunk size for reads of 16384 bytes.
*/ */
@ -153,6 +131,11 @@ public class NIOFSDirectory extends FSDirectory {
return clone; return clone;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new NIOFSIndexInput(sliceDescription, channel, off + offset, length, getBufferSize());
}
@Override @Override
public final long length() { public final long length() {
return end - off; return end - off;

View File

@ -227,22 +227,6 @@ public class NRTCachingDirectory extends Directory {
return delegate.openInput(name, context); return delegate.openInput(name, context);
} }
} }
@Override
public synchronized IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
ensureOpen();
if (VERBOSE) {
System.out.println("nrtdir.openInput name=" + name);
}
if (cache.fileNameExists(name)) {
if (VERBOSE) {
System.out.println(" from cache");
}
return cache.createSlicer(name, context);
} else {
return delegate.createSlicer(name, context);
}
}
/** Close this directory, which flushes any cached files /** Close this directory, which flushes any cached files
* to the delegate and then closes the delegate. */ * to the delegate and then closes the delegate. */

View File

@ -118,4 +118,10 @@ public class RAMInputStream extends IndexInput implements Cloneable {
} }
bufferPosition = (int) (pos % BUFFER_SIZE); bufferPosition = (int) (pos % BUFFER_SIZE);
} }
// TODO: improve this, kinda stupid
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
} }

View File

@ -50,13 +50,6 @@ public final class RateLimitedDirectoryWrapper extends FilterDirectory {
return output; return output;
} }
@Override
public IndexInputSlicer createSlicer(String name, IOContext context)
throws IOException {
ensureOpen();
return in.createSlicer(name, context);
}
@Override @Override
public void copy(Directory to, String src, String dest, IOContext context) throws IOException { public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
ensureOpen(); ensureOpen();

View File

@ -59,32 +59,11 @@ public class SimpleFSDirectory extends FSDirectory {
return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context); return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);
} }
@Override
public IndexInputSlicer createSlicer(final String name,
final IOContext context) throws IOException {
ensureOpen();
final File file = new File(getDirectory(), name);
final RandomAccessFile descriptor = new RandomAccessFile(file, "r");
return new IndexInputSlicer() {
@Override
public void close() throws IOException {
descriptor.close();
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new SimpleFSIndexInput("SimpleFSIndexInput(" + sliceDescription + " in path=\"" + file.getPath() + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
length, BufferedIndexInput.bufferSize(context));
}
};
}
/** /**
* Reads bytes with {@link RandomAccessFile#seek(long)} followed by * Reads bytes with {@link RandomAccessFile#seek(long)} followed by
* {@link RandomAccessFile#read(byte[], int, int)}. * {@link RandomAccessFile#read(byte[], int, int)}.
*/ */
protected static class SimpleFSIndexInput extends BufferedIndexInput { static final class SimpleFSIndexInput extends BufferedIndexInput {
/** /**
* The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs * The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs
* a native buffer outside of stack if the read buffer size is larger. * a native buffer outside of stack if the read buffer size is larger.
@ -129,6 +108,11 @@ public class SimpleFSDirectory extends FSDirectory {
return clone; return clone;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new SimpleFSIndexInput(sliceDescription, file, off + offset, length, getBufferSize());
}
@Override @Override
public final long length() { public final long length() {
return end - off; return end - off;

View File

@ -50,11 +50,6 @@ public final class TrackingDirectoryWrapper extends FilterDirectory {
in.copy(to, src, dest, context); in.copy(to, src, dest, context);
} }
@Override
public Directory.IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
return in.createSlicer(name, context);
}
// maybe clone before returning.... all callers are // maybe clone before returning.... all callers are
// cloning anyway.... // cloning anyway....
public Set<String> getCreatedFiles() { public Set<String> getCreatedFiles() {

View File

@ -183,6 +183,12 @@ public class TestFieldsReader extends LuceneTestCase {
} }
return i; return i;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
IndexInput slice = delegate.slice(sliceDescription, offset, length);
return new FaultyIndexInput(slice);
}
} }
// LUCENE-1262 // LUCENE-1262

View File

@ -20,9 +20,11 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TestUtil;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.RAMDirectory;
@ -169,34 +171,26 @@ public class TestIndexInput extends LuceneTestCase {
} }
} }
// this test only checks BufferedIndexInput because MockIndexInput extends BufferedIndexInput // this test checks the IndexInput methods of any impl
public void testBufferedIndexInputRead() throws IOException {
IndexInput is = new MockIndexInput(READ_TEST_BYTES);
checkReads(is, IOException.class);
is.close();
is = new MockIndexInput(RANDOM_TEST_BYTES);
checkRandomReads(is);
is.close();
}
// this test checks the raw IndexInput methods as it uses RAMIndexInput which extends IndexInput directly
public void testRawIndexInputRead() throws IOException { public void testRawIndexInputRead() throws IOException {
Random random = random(); for (int i = 0; i < 10; i++) {
final RAMDirectory dir = new RAMDirectory(); Random random = random();
IndexOutput os = dir.createOutput("foo", newIOContext(random)); final Directory dir = newDirectory();
os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length); IndexOutput os = dir.createOutput("foo", newIOContext(random));
os.close(); os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length);
IndexInput is = dir.openInput("foo", newIOContext(random)); os.close();
checkReads(is, IOException.class); IndexInput is = dir.openInput("foo", newIOContext(random));
is.close(); checkReads(is, IOException.class);
is.close();
os = dir.createOutput("bar", newIOContext(random)); os = dir.createOutput("bar", newIOContext(random));
os.writeBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.length); os.writeBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.length);
os.close(); os.close();
is = dir.openInput("bar", newIOContext(random)); is = dir.openInput("bar", newIOContext(random));
checkRandomReads(is); checkRandomReads(is);
is.close(); is.close();
dir.close(); dir.close();
}
} }
public void testByteArrayDataInput() throws IOException { public void testByteArrayDataInput() throws IOException {

View File

@ -227,6 +227,10 @@ public class TestLazyProxSkipping extends LuceneTestCase {
public SeeksCountingStream clone() { public SeeksCountingStream clone() {
return new SeeksCountingStream(this.input.clone()); return new SeeksCountingStream(this.input.clone());
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new SeeksCountingStream(this.input.clone());
}
} }
} }

View File

@ -189,5 +189,9 @@ public class TestMultiLevelSkipList extends LuceneTestCase {
return new CountingStream(this.input.clone()); return new CountingStream(this.input.clone());
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new CountingStream(this.input.slice(sliceDescription, offset, length));
}
} }
} }

View File

@ -219,6 +219,11 @@ public class TestBufferedIndexInput extends LuceneTestCase {
public long length() { public long length() {
return len; return len;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException();
}
} }
public void testSetBufferSize() throws IOException { public void testSetBufferSize() throws IOException {

View File

@ -32,7 +32,6 @@ public class TestFilterDirectory extends LuceneTestCase {
// except those under the 'exclude' list // except those under the 'exclude' list
Set<Method> exclude = new HashSet<>(); Set<Method> exclude = new HashSet<>();
exclude.add(Directory.class.getMethod("copy", Directory.class, String.class, String.class, IOContext.class)); exclude.add(Directory.class.getMethod("copy", Directory.class, String.class, String.class, IOContext.class));
exclude.add(Directory.class.getMethod("createSlicer", String.class, IOContext.class));
exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class)); exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class));
for (Method m : FilterDirectory.class.getMethods()) { for (Method m : FilterDirectory.class.getMethods()) {
if (m.getDeclaringClass() == Directory.class) { if (m.getDeclaringClass() == Directory.class) {

View File

@ -26,7 +26,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory.IndexInputSlicer;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TestUtil;
@ -113,9 +112,9 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
io.writeInt(1); io.writeInt(1);
io.writeInt(2); io.writeInt(2);
io.close(); io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random())); IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput one = slicer.openSlice("first int", 0, 4); IndexInput one = slicer.slice("first int", 0, 4);
IndexInput two = slicer.openSlice("second int", 4, 4); IndexInput two = slicer.slice("second int", 4, 4);
IndexInput three = one.clone(); // clone of clone IndexInput three = one.clone(); // clone of clone
IndexInput four = two.clone(); // clone of clone IndexInput four = two.clone(); // clone of clone
slicer.close(); slicer.close();
@ -158,9 +157,9 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
io.writeInt(1); io.writeInt(1);
io.writeInt(2); io.writeInt(2);
io.close(); io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random())); IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput one = slicer.openSlice("first int", 0, 4); IndexInput one = slicer.slice("first int", 0, 4);
IndexInput two = slicer.openSlice("second int", 4, 4); IndexInput two = slicer.slice("second int", 4, 4);
one.close(); one.close();
try { try {
one.readInt(); one.readInt();
@ -170,7 +169,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
} }
assertEquals(2, two.readInt()); assertEquals(2, two.readInt());
// reopen a new slice "one": // reopen a new slice "one":
one = slicer.openSlice("first int", 0, 4); one = slicer.slice("first int", 0, 4);
assertEquals(1, one.readInt()); assertEquals(1, one.readInt());
one.close(); one.close();
two.close(); two.close();
@ -195,8 +194,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceZero"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceZero"), null, 1<<i);
IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random()));
io.close(); io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("zeroBytes", newIOContext(random())); IndexInput slicer = mmapDir.openInput("zeroBytes", newIOContext(random()));
IndexInput ii = slicer.openSlice("zero-length slice", 0, 0); IndexInput ii = slicer.slice("zero-length slice", 0, 0);
ii.seek(0L); ii.seek(0L);
ii.close(); ii.close();
slicer.close(); slicer.close();
@ -230,8 +229,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
random().nextBytes(bytes); random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length); io.writeBytes(bytes, bytes.length);
io.close(); io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random())); IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput ii = slicer.openSlice("full slice", 0, bytes.length); IndexInput ii = slicer.slice("full slice", 0, bytes.length);
byte actual[] = new byte[1<<i]; byte actual[] = new byte[1<<i];
ii.readBytes(actual, 0, actual.length); ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual)); assertEquals(new BytesRef(bytes), new BytesRef(actual));
@ -282,11 +281,11 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
ii.readBytes(actual, 0, actual.length); ii.readBytes(actual, 0, actual.length);
ii.close(); ii.close();
assertEquals(new BytesRef(bytes), new BytesRef(actual)); assertEquals(new BytesRef(bytes), new BytesRef(actual));
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random())); IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) { for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) {
for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) { for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) {
byte slice[] = new byte[sliceLength]; byte slice[] = new byte[sliceLength];
IndexInput input = slicer.openSlice("bytesSlice", sliceStart, slice.length); IndexInput input = slicer.slice("bytesSlice", sliceStart, slice.length);
input.readBytes(slice, 0, slice.length); input.readBytes(slice, 0, slice.length);
input.close(); input.close();
assertEquals(new BytesRef(bytes, sliceStart, sliceLength), new BytesRef(slice)); assertEquals(new BytesRef(bytes, sliceStart, sliceLength), new BytesRef(slice));

View File

@ -118,7 +118,11 @@ public class SlowRAMDirectory extends RAMDirectory {
ii.readBytes(b, offset, len); ii.readBytes(b, offset, len);
} }
// TODO: is it intentional that clone doesnt wrap?
@Override public IndexInput clone() { return ii.clone(); } @Override public IndexInput clone() { return ii.clone(); }
@Override public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return ii.slice(sliceDescription, offset, length);
}
@Override public void close() throws IOException { ii.close(); } @Override public void close() throws IOException { ii.close(); }
@Override public boolean equals(Object o) { return ii.equals(o); } @Override public boolean equals(Object o) { return ii.equals(o); }
@Override public long getFilePointer() { return ii.getFilePointer(); } @Override public long getFilePointer() { return ii.getFilePointer(); }

View File

@ -406,5 +406,11 @@ public class NativeUnixDirectory extends FSDirectory {
throw new RuntimeException("IOException during clone: " + this, ioe); throw new RuntimeException("IOException during clone: " + this, ioe);
} }
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
// TODO: is this the right thing to do?
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
} }
} }

View File

@ -1,70 +0,0 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.store.BufferedIndexInput;
// TODO: what is this used for? just testing BufferedIndexInput?
// if so it should be pkg-private. otherwise its a dup of ByteArrayIndexInput?
/**
* IndexInput backed by a byte[] for testing.
*/
public class MockIndexInput extends BufferedIndexInput {
private byte[] buffer;
private int pointer = 0;
private long length;
public MockIndexInput(byte[] bytes) {
super("MockIndexInput", BufferedIndexInput.BUFFER_SIZE);
buffer = bytes;
length = bytes.length;
}
@Override
protected void readInternal(byte[] dest, int destOffset, int len) {
int remainder = len;
int start = pointer;
while (remainder != 0) {
// int bufferNumber = start / buffer.length;
int bufferOffset = start % buffer.length;
int bytesInBuffer = buffer.length - bufferOffset;
int bytesToCopy = bytesInBuffer >= remainder ? remainder : bytesInBuffer;
System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy);
destOffset += bytesToCopy;
start += bytesToCopy;
remainder -= bytesToCopy;
}
pointer += len;
}
@Override
public void close() {
// ignore
}
@Override
protected void seekInternal(long pos) {
pointer = (int) pos;
}
@Override
public long length() {
return length;
}
}

View File

@ -75,9 +75,4 @@ public class BaseDirectoryWrapper extends FilterDirectory {
public void copy(Directory to, String src, String dest, IOContext context) throws IOException { public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
in.copy(to, src, dest, context); in.copy(to, src, dest, context);
} }
@Override
public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException {
return in.createSlicer(name, context);
}
} }

View File

@ -946,45 +946,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
// randomize the IOContext here? // randomize the IOContext here?
in.copy(to, src, dest, context); in.copy(to, src, dest, context);
} }
@Override
public IndexInputSlicer createSlicer(final String name, IOContext context)
throws IOException {
maybeYield();
if (!LuceneTestCase.slowFileExists(in, name)) {
throw randomState.nextBoolean() ? new FileNotFoundException(name) : new NoSuchFileException(name);
}
// cannot open a file for input if it's still open for
// output, except for segments.gen and segments_N
if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
}
final IndexInputSlicer delegateHandle = in.createSlicer(name, context);
final IndexInputSlicer handle = new IndexInputSlicer() {
private boolean isClosed;
@Override
public void close() throws IOException {
if (!isClosed) {
delegateHandle.close();
MockDirectoryWrapper.this.removeOpenFile(this, name);
isClosed = true;
}
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException {
maybeYield();
IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, delegateHandle.openSlice(sliceDescription, offset, length));
addFileHandle(ii, name, Handle.Input);
return ii;
}
};
addFileHandle(handle, name, Handle.Slice);
return handle;
}
final class BufferedIndexOutputWrapper extends BufferedIndexOutput { final class BufferedIndexOutputWrapper extends BufferedIndexOutput {
private final IndexOutput io; private final IndexOutput io;

View File

@ -88,6 +88,16 @@ public class MockIndexInputWrapper extends IndexInput {
return clone; return clone;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
ensureOpen();
dir.inputCloneCount.incrementAndGet();
IndexInput slice = delegate.slice(sliceDescription, offset, length);
MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, sliceDescription, slice);
clone.isClone = true;
return clone;
}
@Override @Override
public long getFilePointer() { public long getFilePointer() {
ensureOpen(); ensureOpen();

View File

@ -20,6 +20,7 @@ package org.apache.solr.store.blockcache;
import java.io.EOFException; import java.io.EOFException;
import java.io.IOException; import java.io.IOException;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexOutput;
@ -255,6 +256,11 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
return clone; return clone;
} }
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
/** /**
* Flushes the in-memory bufer to the given output, copying at most * Flushes the in-memory bufer to the given output, copying at most
* <code>numBytes</code>. * <code>numBytes</code>.