Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4371: Replace IndexInputSlicer with IndexInput.slice 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1595480 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-05-17 13:44:09 +00:00
parent 5cdde67569
commit 7aeac4f07b
30 changed files with 182 additions and 386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -77,6 +77,9 @@ API Changes
takes the same selectors. Add helper methods to DocValues.java that are better
suited for search code (never return null, etc). (Mike McCandless, Robert Muir)
* LUCENE-4371: Removed IndexInputSlicer and Directory.createSlicer() and replaced
with IndexInput.slice(). (Robert Muir)
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect

4
lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
View File

@ -312,5 +312,9 @@ public abstract class MultiLevelSkipListReader implements Closeable {
this.pos = (int) (pos - pointer);
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException();
}
}
}

5
lucene/core/src/java/org/apache/lucene/store/BufferedChecksumIndexInput.java
View File

@ -74,4 +74,9 @@ public class BufferedChecksumIndexInput extends ChecksumIndexInput {
public IndexInput clone() {
throw new UnsupportedOperationException();
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException();
}
}

63
lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
View File

@ -316,6 +316,11 @@ public abstract class BufferedIndexInput extends IndexInput {
return clone;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return wrap("SlicedIndexInput(" + sliceDescription + " in " + this + ")", this, offset, length);
}
/**
* Flushes the in-memory buffer to the given output, copying at most
* <code>numBytes</code>.
@ -349,4 +354,62 @@ public abstract class BufferedIndexInput extends IndexInput {
}
}
/**
* Wraps a portion of a file with buffering.
*/
public static BufferedIndexInput wrap(String description, IndexInput other, long offset, long length) {
return new SlicedIndexInput(description, other, offset, length);
}
/**
* Implementation of an IndexInput that reads from a portion of a file.
*/
private static final class SlicedIndexInput extends BufferedIndexInput {
IndexInput base;
long fileOffset;
long length;
SlicedIndexInput(String sliceDescription, IndexInput base, long offset, long length) {
super("SlicedIndexInput(" + sliceDescription + " in " + base + " slice=" + offset + ":" + (offset+length) + ")", BufferedIndexInput.BUFFER_SIZE);
if (offset < 0 || length < 0) {
throw new IllegalArgumentException();
}
assert offset + length <= base.length();
this.base = base.clone();
this.fileOffset = offset;
this.length = length;
}
@Override
public SlicedIndexInput clone() {
SlicedIndexInput clone = (SlicedIndexInput)super.clone();
clone.base = base.clone();
clone.fileOffset = fileOffset;
clone.length = length;
return clone;
}
@Override
protected void readInternal(byte[] b, int offset, int len) throws IOException {
long start = getFilePointer();
if (start + len > length) {
throw new EOFException("read past EOF: " + this);
}
base.seek(fileOffset + start);
base.readBytes(b, offset, len, false);
}
@Override
protected void seekInternal(long pos) {}
@Override
public void close() throws IOException {
base.close();
}
@Override
public long length() {
return length;
}
}
}

4
lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
View File

@ -197,10 +197,8 @@ abstract class ByteBufferIndexInput extends IndexInput {
/**
* Creates a slice of this index input, with the given description, offset, and length. The slice is seeked to the beginning.
*/
@Override
public final ByteBufferIndexInput slice(String sliceDescription, long offset, long length) {
if (isClone) { // well we could, but this is stupid
throw new IllegalStateException("cannot slice() " + sliceDescription + " from a cloned IndexInput: " + this);
}
final ByteBufferIndexInput clone = buildSlice(offset, length);
clone.sliceDescription = sliceDescription;
try {

28
lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
View File

@ -87,7 +87,7 @@ public final class CompoundFileDirectory extends BaseDirectory {
private final boolean openForWrite;
private static final Map<String,FileEntry> SENTINEL = Collections.emptyMap();
private final CompoundFileWriter writer;
private final IndexInputSlicer handle;
private final IndexInput handle;
private int version;
/**
@ -101,7 +101,7 @@ public final class CompoundFileDirectory extends BaseDirectory {
this.openForWrite = openForWrite;
if (!openForWrite) {
boolean success = false;
handle = directory.createSlicer(fileName, context);
handle = directory.openInput(fileName, context);
try {
this.entries = readEntries(directory, fileName);
success = true;
@ -192,7 +192,7 @@ public final class CompoundFileDirectory extends BaseDirectory {
if (entry == null) {
throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
}
return handle.openSlice(name, entry.offset, entry.length);
return handle.slice(name, entry.offset, entry.length);
}
/** Returns an array of strings, one for each file in the directory. */
@ -258,28 +258,6 @@ public final class CompoundFileDirectory extends BaseDirectory {
throw new UnsupportedOperationException();
}
@Override
public IndexInputSlicer createSlicer(final String name, IOContext context)
throws IOException {
ensureOpen();
assert !openForWrite;
final String id = IndexFileNames.stripSegmentName(name);
final FileEntry entry = entries.get(id);
if (entry == null) {
throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
}
return new IndexInputSlicer() {
@Override
public void close() {
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException {
return handle.openSlice(sliceDescription, entry.offset + offset, length);
}
};
}
@Override
public String toString() {
return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")";

106
lucene/core/src/java/org/apache/lucene/store/Directory.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.store;
* limitations under the License.
*/
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Closeable;
@ -200,113 +199,8 @@ public abstract class Directory implements Closeable {
}
}
/**
* Creates an {@link IndexInputSlicer} for the given file name.
* IndexInputSlicer allows other {@link Directory} implementations to
* efficiently open one or more sliced {@link IndexInput} instances from a
* single file handle. The underlying file handle is kept open until the
* {@link IndexInputSlicer} is closed.
* <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException}
* if the file does not exist.
*
* @throws IOException
* if an {@link IOException} occurs
* @lucene.internal
* @lucene.experimental
*/
public IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
ensureOpen();
return new IndexInputSlicer() {
private final IndexInput base = Directory.this.openInput(name, context);
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new SlicedIndexInput("SlicedIndexInput(" + sliceDescription + " in " + base + ")", base, offset, length);
}
@Override
public void close() throws IOException {
base.close();
}
};
}
/**
* @throws AlreadyClosedException if this Directory is closed
*/
protected void ensureOpen() throws AlreadyClosedException {}
/**
* Allows to create one or more sliced {@link IndexInput} instances from a single
* file handle. Some {@link Directory} implementations may be able to efficiently map slices of a file
* into memory when only certain parts of a file are required.
* @lucene.internal
* @lucene.experimental
*/
public abstract class IndexInputSlicer implements Closeable {
/**
* Returns an {@link IndexInput} slice starting at the given offset with the given length.
*/
public abstract IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException;
}
/** Implementation of an IndexInput that reads from a portion of
* a file.
*/
private static final class SlicedIndexInput extends BufferedIndexInput {
IndexInput base;
long fileOffset;
long length;
SlicedIndexInput(final String sliceDescription, final IndexInput base, final long fileOffset, final long length) {
this(sliceDescription, base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE);
}
SlicedIndexInput(final String sliceDescription, final IndexInput base, final long fileOffset, final long length, int readBufferSize) {
super("SlicedIndexInput(" + sliceDescription + " in " + base + " slice=" + fileOffset + ":" + (fileOffset+length) + ")", readBufferSize);
this.base = base.clone();
this.fileOffset = fileOffset;
this.length = length;
}
@Override
public SlicedIndexInput clone() {
SlicedIndexInput clone = (SlicedIndexInput)super.clone();
clone.base = base.clone();
clone.fileOffset = fileOffset;
clone.length = length;
return clone;
}
/** Expert: implements buffer refill. Reads bytes from the current
* position in the input.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param len the number of bytes to read
*/
@Override
protected void readInternal(byte[] b, int offset, int len) throws IOException {
long start = getFilePointer();
if(start + len > length)
throw new EOFException("read past EOF: " + this);
base.seek(fileOffset + start);
base.readBytes(b, offset, len, false);
}
/** Expert: implements seek. Sets current position in this file, where
* the next {@link #readInternal(byte[],int,int)} will occur.
* @see #readInternal(byte[],int,int)
*/
@Override
protected void seekInternal(long pos) {}
/** Closes the stream to further operations. */
@Override
public void close() throws IOException {
base.close();
}
@Override
public long length() {
return length;
}
}
}

6
lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
View File

@ -166,10 +166,4 @@ public class FileSwitchDirectory extends BaseDirectory {
public IndexInput openInput(String name, IOContext context) throws IOException {
return getDirectory(name).openInput(name, context);
}
@Override
public IndexInputSlicer createSlicer(String name, IOContext context)
throws IOException {
return getDirectory(name).createSlicer(name, context);
}
}

6
lucene/core/src/java/org/apache/lucene/store/IndexInput.java
View File

@ -82,4 +82,10 @@ public abstract class IndexInput extends DataInput implements Cloneable,Closeabl
public IndexInput clone() {
return (IndexInput) super.clone();
}
/**
* Creates a slice of this index input, with the given description, offset, and length.
* The slice is seeked to the beginning.
*/
public abstract IndexInput slice(String sliceDescription, long offset, long length) throws IOException;
}

17
lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
View File

@ -197,23 +197,6 @@ public class MMapDirectory extends FSDirectory {
}
}
@Override
public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException {
final MMapIndexInput full = (MMapIndexInput) openInput(name, context);
return new IndexInputSlicer() {
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException {
ensureOpen();
return full.slice(sliceDescription, offset, length);
}
@Override
public void close() throws IOException {
full.close();
}
};
}
private final class MMapIndexInput extends ByteBufferIndexInput {
private final boolean useUnmapHack;

29
lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.store;
import java.io.File;
import java.io.EOFException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
@ -83,31 +82,10 @@ public class NIOFSDirectory extends FSDirectory {
return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context);
}
@Override
public IndexInputSlicer createSlicer(final String name,
final IOContext context) throws IOException {
ensureOpen();
final File path = new File(getDirectory(), name);
final FileChannel descriptor = FileChannel.open(path.toPath(), StandardOpenOption.READ);
return new Directory.IndexInputSlicer() {
@Override
public void close() throws IOException {
descriptor.close();
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new NIOFSIndexInput("NIOFSIndexInput(" + sliceDescription + " in path=\"" + path + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
length, BufferedIndexInput.bufferSize(context));
}
};
}
/**
* Reads bytes with {@link FileChannel#read(ByteBuffer, long)}
*/
protected static class NIOFSIndexInput extends BufferedIndexInput {
static final class NIOFSIndexInput extends BufferedIndexInput {
/**
* The maximum chunk size for reads of 16384 bytes.
*/
@ -153,6 +131,11 @@ public class NIOFSDirectory extends FSDirectory {
return clone;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new NIOFSIndexInput(sliceDescription, channel, off + offset, length, getBufferSize());
}
@Override
public final long length() {
return end - off;

16
lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
View File

@ -228,22 +228,6 @@ public class NRTCachingDirectory extends Directory {
}
}
@Override
public synchronized IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
ensureOpen();
if (VERBOSE) {
System.out.println("nrtdir.openInput name=" + name);
}
if (cache.fileNameExists(name)) {
if (VERBOSE) {
System.out.println(" from cache");
}
return cache.createSlicer(name, context);
} else {
return delegate.createSlicer(name, context);
}
}
/** Close this directory, which flushes any cached files
* to the delegate and then closes the delegate. */
@Override

6
lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
View File

@ -118,4 +118,10 @@ public class RAMInputStream extends IndexInput implements Cloneable {
}
bufferPosition = (int) (pos % BUFFER_SIZE);
}
// TODO: improve this, kinda stupid
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
}

7
lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java
View File

@ -50,13 +50,6 @@ public final class RateLimitedDirectoryWrapper extends FilterDirectory {
return output;
}
@Override
public IndexInputSlicer createSlicer(String name, IOContext context)
throws IOException {
ensureOpen();
return in.createSlicer(name, context);
}
@Override
public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
ensureOpen();

28
lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
View File

@ -59,32 +59,11 @@ public class SimpleFSDirectory extends FSDirectory {
return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);
}
@Override
public IndexInputSlicer createSlicer(final String name,
final IOContext context) throws IOException {
ensureOpen();
final File file = new File(getDirectory(), name);
final RandomAccessFile descriptor = new RandomAccessFile(file, "r");
return new IndexInputSlicer() {
@Override
public void close() throws IOException {
descriptor.close();
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) {
return new SimpleFSIndexInput("SimpleFSIndexInput(" + sliceDescription + " in path=\"" + file.getPath() + "\" slice=" + offset + ":" + (offset+length) + ")", descriptor, offset,
length, BufferedIndexInput.bufferSize(context));
}
};
}
/**
* Reads bytes with {@link RandomAccessFile#seek(long)} followed by
* {@link RandomAccessFile#read(byte[], int, int)}.
*/
protected static class SimpleFSIndexInput extends BufferedIndexInput {
static final class SimpleFSIndexInput extends BufferedIndexInput {
/**
* The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs
* a native buffer outside of stack if the read buffer size is larger.
@ -129,6 +108,11 @@ public class SimpleFSDirectory extends FSDirectory {
return clone;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new SimpleFSIndexInput(sliceDescription, file, off + offset, length, getBufferSize());
}
@Override
public final long length() {
return end - off;

5
lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
View File

@ -50,11 +50,6 @@ public final class TrackingDirectoryWrapper extends FilterDirectory {
in.copy(to, src, dest, context);
}
@Override
public Directory.IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
return in.createSlicer(name, context);
}
// maybe clone before returning.... all callers are
// cloning anyway....
public Set<String> getCreatedFiles() {

6
lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
View File

@ -183,6 +183,12 @@ public class TestFieldsReader extends LuceneTestCase {
}
return i;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
IndexInput slice = delegate.slice(sliceDescription, offset, length);
return new FaultyIndexInput(slice);
}
}
// LUCENE-1262

18
lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
View File

@ -20,9 +20,11 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory;
@ -169,20 +171,11 @@ public class TestIndexInput extends LuceneTestCase {
}
}
// this test only checks BufferedIndexInput because MockIndexInput extends BufferedIndexInput
public void testBufferedIndexInputRead() throws IOException {
IndexInput is = new MockIndexInput(READ_TEST_BYTES);
checkReads(is, IOException.class);
is.close();
is = new MockIndexInput(RANDOM_TEST_BYTES);
checkRandomReads(is);
is.close();
}
// this test checks the raw IndexInput methods as it uses RAMIndexInput which extends IndexInput directly
// this test checks the IndexInput methods of any impl
public void testRawIndexInputRead() throws IOException {
for (int i = 0; i < 10; i++) {
Random random = random();
final RAMDirectory dir = new RAMDirectory();
final Directory dir = newDirectory();
IndexOutput os = dir.createOutput("foo", newIOContext(random));
os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length);
os.close();
@ -198,6 +191,7 @@ public class TestIndexInput extends LuceneTestCase {
is.close();
dir.close();
}
}
public void testByteArrayDataInput() throws IOException {
ByteArrayDataInput is = new ByteArrayDataInput(READ_TEST_BYTES);

4
lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
View File

@ -228,5 +228,9 @@ public class TestLazyProxSkipping extends LuceneTestCase {
return new SeeksCountingStream(this.input.clone());
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new SeeksCountingStream(this.input.clone());
}
}
}

4
lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
View File

@ -189,5 +189,9 @@ public class TestMultiLevelSkipList extends LuceneTestCase {
return new CountingStream(this.input.clone());
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return new CountingStream(this.input.slice(sliceDescription, offset, length));
}
}
}

5
lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
View File

@ -219,6 +219,11 @@ public class TestBufferedIndexInput extends LuceneTestCase {
public long length() {
return len;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException();
}
}
public void testSetBufferSize() throws IOException {

1
lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
View File

@ -32,7 +32,6 @@ public class TestFilterDirectory extends LuceneTestCase {
// except those under the 'exclude' list
Set<Method> exclude = new HashSet<>();
exclude.add(Directory.class.getMethod("copy", Directory.class, String.class, String.class, IOContext.class));
exclude.add(Directory.class.getMethod("createSlicer", String.class, IOContext.class));
exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class));
for (Method m : FilterDirectory.class.getMethods()) {
if (m.getDeclaringClass() == Directory.class) {

27
lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
View File

@ -26,7 +26,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory.IndexInputSlicer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TestUtil;
@ -113,9 +112,9 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
io.writeInt(1);
io.writeInt(2);
io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random()));
IndexInput one = slicer.openSlice("first int", 0, 4);
IndexInput two = slicer.openSlice("second int", 4, 4);
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput one = slicer.slice("first int", 0, 4);
IndexInput two = slicer.slice("second int", 4, 4);
IndexInput three = one.clone(); // clone of clone
IndexInput four = two.clone(); // clone of clone
slicer.close();
@ -158,9 +157,9 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
io.writeInt(1);
io.writeInt(2);
io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random()));
IndexInput one = slicer.openSlice("first int", 0, 4);
IndexInput two = slicer.openSlice("second int", 4, 4);
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput one = slicer.slice("first int", 0, 4);
IndexInput two = slicer.slice("second int", 4, 4);
one.close();
try {
one.readInt();
@ -170,7 +169,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
}
assertEquals(2, two.readInt());
// reopen a new slice "one":
one = slicer.openSlice("first int", 0, 4);
one = slicer.slice("first int", 0, 4);
assertEquals(1, one.readInt());
one.close();
two.close();
@ -195,8 +194,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceZero"), null, 1<<i);
IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random()));
io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("zeroBytes", newIOContext(random()));
IndexInput ii = slicer.openSlice("zero-length slice", 0, 0);
IndexInput slicer = mmapDir.openInput("zeroBytes", newIOContext(random()));
IndexInput ii = slicer.slice("zero-length slice", 0, 0);
ii.seek(0L);
ii.close();
slicer.close();
@ -230,8 +229,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
random().nextBytes(bytes);
io.writeBytes(bytes, bytes.length);
io.close();
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random()));
IndexInput ii = slicer.openSlice("full slice", 0, bytes.length);
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
IndexInput ii = slicer.slice("full slice", 0, bytes.length);
byte actual[] = new byte[1<<i];
ii.readBytes(actual, 0, actual.length);
assertEquals(new BytesRef(bytes), new BytesRef(actual));
@ -282,11 +281,11 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
ii.readBytes(actual, 0, actual.length);
ii.close();
assertEquals(new BytesRef(bytes), new BytesRef(actual));
IndexInputSlicer slicer = mmapDir.createSlicer("bytes", newIOContext(random()));
IndexInput slicer = mmapDir.openInput("bytes", newIOContext(random()));
for (int sliceStart = 0; sliceStart < bytes.length; sliceStart++) {
for (int sliceLength = 0; sliceLength < bytes.length - sliceStart; sliceLength++) {
byte slice[] = new byte[sliceLength];
IndexInput input = slicer.openSlice("bytesSlice", sliceStart, slice.length);
IndexInput input = slicer.slice("bytesSlice", sliceStart, slice.length);
input.readBytes(slice, 0, slice.length);
input.close();
assertEquals(new BytesRef(bytes, sliceStart, sliceLength), new BytesRef(slice));

4
lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
View File

@ -118,7 +118,11 @@ public class SlowRAMDirectory extends RAMDirectory {
ii.readBytes(b, offset, len);
}
// TODO: is it intentional that clone doesnt wrap?
@Override public IndexInput clone() { return ii.clone(); }
@Override public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return ii.slice(sliceDescription, offset, length);
}
@Override public void close() throws IOException { ii.close(); }
@Override public boolean equals(Object o) { return ii.equals(o); }
@Override public long getFilePointer() { return ii.getFilePointer(); }

6
lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
View File

@ -406,5 +406,11 @@ public class NativeUnixDirectory extends FSDirectory {
throw new RuntimeException("IOException during clone: " + this, ioe);
}
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
// TODO: is this the right thing to do?
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
}
}

70
lucene/test-framework/src/java/org/apache/lucene/index/MockIndexInput.java
View File

@ -1,70 +0,0 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.store.BufferedIndexInput;
// TODO: what is this used for? just testing BufferedIndexInput?
// if so it should be pkg-private. otherwise its a dup of ByteArrayIndexInput?
/**
* IndexInput backed by a byte[] for testing.
*/
public class MockIndexInput extends BufferedIndexInput {
private byte[] buffer;
private int pointer = 0;
private long length;
public MockIndexInput(byte[] bytes) {
super("MockIndexInput", BufferedIndexInput.BUFFER_SIZE);
buffer = bytes;
length = bytes.length;
}
@Override
protected void readInternal(byte[] dest, int destOffset, int len) {
int remainder = len;
int start = pointer;
while (remainder != 0) {
// int bufferNumber = start / buffer.length;
int bufferOffset = start % buffer.length;
int bytesInBuffer = buffer.length - bufferOffset;
int bytesToCopy = bytesInBuffer >= remainder ? remainder : bytesInBuffer;
System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy);
destOffset += bytesToCopy;
start += bytesToCopy;
remainder -= bytesToCopy;
}
pointer += len;
}
@Override
public void close() {
// ignore
}
@Override
protected void seekInternal(long pos) {
pointer = (int) pos;
}
@Override
public long length() {
return length;
}
}

5
lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java
View File

@ -75,9 +75,4 @@ public class BaseDirectoryWrapper extends FilterDirectory {
public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
in.copy(to, src, dest, context);
}
@Override
public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException {
return in.createSlicer(name, context);
}
}

39
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -947,45 +947,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
in.copy(to, src, dest, context);
}
@Override
public IndexInputSlicer createSlicer(final String name, IOContext context)
throws IOException {
maybeYield();
if (!LuceneTestCase.slowFileExists(in, name)) {
throw randomState.nextBoolean() ? new FileNotFoundException(name) : new NoSuchFileException(name);
}
// cannot open a file for input if it's still open for
// output, except for segments.gen and segments_N
if (openFilesForWrite.contains(name) && !name.startsWith("segments")) {
throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
}
final IndexInputSlicer delegateHandle = in.createSlicer(name, context);
final IndexInputSlicer handle = new IndexInputSlicer() {
private boolean isClosed;
@Override
public void close() throws IOException {
if (!isClosed) {
delegateHandle.close();
MockDirectoryWrapper.this.removeOpenFile(this, name);
isClosed = true;
}
}
@Override
public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException {
maybeYield();
IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, delegateHandle.openSlice(sliceDescription, offset, length));
addFileHandle(ii, name, Handle.Input);
return ii;
}
};
addFileHandle(handle, name, Handle.Slice);
return handle;
}
final class BufferedIndexOutputWrapper extends BufferedIndexOutput {
private final IndexOutput io;

10
lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java
View File

@ -88,6 +88,16 @@ public class MockIndexInputWrapper extends IndexInput {
return clone;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
ensureOpen();
dir.inputCloneCount.incrementAndGet();
IndexInput slice = delegate.slice(sliceDescription, offset, length);
MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, sliceDescription, slice);
clone.isClone = true;
return clone;
}
@Override
public long getFilePointer() {
ensureOpen();

6
solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
View File

@ -20,6 +20,7 @@ package org.apache.solr.store.blockcache;
import java.io.EOFException;
import java.io.IOException;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@ -255,6 +256,11 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
return clone;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
}
/**
* Flushes the in-memory bufer to the given output, copying at most
* <code>numBytes</code>.