Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Permit unbuffered index input. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150517 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Doug Cutting 2004-09-16 21:13:37 +00:00
parent faebce61db
commit 99fdf6af1f
25 changed files with 450 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
CHANGES.txt
View File

@ -48,14 +48,18 @@ $Id$
9. PhraseQuery and PhrasePrefixQuery now allow the explicit specification
of relative positions. (Christoph Goller)
10. QueryParser changes: Fix for ArrayIndexOutOfBoundsExceptions
(patch #9110); some unused method parameters removed; The ability
to specify a minimum similarity for FuzzyQuery has been added.
(Christoph Goller)
10. QueryParser changes: Fix for ArrayIndexOutOfBoundsExceptions
(patch #9110); some unused method parameters removed; The ability
to specify a minimum similarity for FuzzyQuery has been added.
(Christoph Goller)
11. Added support for binary stored fields (patch #29370)
(Drew Farris and Bernhard Messer via Christoph)
11. Added support for binary stored fields (patch #29370)
(Drew Farris and Bernhard Messer via Christoph)
12. Permit unbuffered Directory implementations (e.g., using mmap).
InputStream is replaced by the new classes IndexInput and
BufferedIndexInput. InputStream is now deprecated and FSDirectory
is now subclassable. (cutting)
1.4.1

27
src/java/org/apache/lucene/index/CompoundFileReader.java
View File

@ -17,7 +17,8 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.Lock;
import java.util.HashMap;
@ -44,7 +45,7 @@ class CompoundFileReader extends Directory {
private Directory directory;
private String fileName;
private InputStream stream;
private IndexInput stream;
private HashMap entries = new HashMap();
@ -57,7 +58,7 @@ class CompoundFileReader extends Directory {
boolean success = false;
try {
stream = dir.openFile(name);
stream = dir.openInput(name);
// read the directory and init files
int count = stream.readVInt();
@ -109,7 +110,7 @@ class CompoundFileReader extends Directory {
stream = null;
}
public synchronized InputStream openFile(String id)
public synchronized IndexInput openInput(String id)
throws IOException
{
if (stream == null)
@ -119,7 +120,7 @@ class CompoundFileReader extends Directory {
if (entry == null)
throw new IOException("No sub-file with id " + id + " found");
return new CSInputStream(stream, entry.offset, entry.length);
return new CSIndexInput(stream, entry.offset, entry.length);
}
/** Returns an array of strings, one for each file in the directory. */
@ -182,21 +183,22 @@ class CompoundFileReader extends Directory {
throw new UnsupportedOperationException();
}
/** Implementation of an InputStream that reads from a portion of the
/** Implementation of an IndexInput that reads from a portion of the
* compound file. The visibility is left as "package" *only* because
* this helps with testing since JUnit test cases in a different class
* can then access package fields of this class.
*/
static final class CSInputStream extends InputStream {
static final class CSIndexInput extends BufferedIndexInput {
InputStream base;
IndexInput base;
long fileOffset;
long length;
CSInputStream(final InputStream base, final long fileOffset, final long length)
CSIndexInput(final IndexInput base, final long fileOffset, final long length)
{
this.base = base;
this.fileOffset = fileOffset;
this.length = length; // variable in the superclass
this.length = length;
}
/** Expert: implements buffer refill. Reads bytes from the current
@ -226,5 +228,10 @@ class CompoundFileReader extends Directory {
/** Closes the stream to further operations. */
public void close() {}
public long length() {
return length;
}
}
}

6
src/java/org/apache/lucene/index/CompoundFileWriter.java
View File

@ -18,7 +18,7 @@ package org.apache.lucene.index;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import java.util.LinkedList;
import java.util.HashSet;
import java.util.Iterator;
@ -196,11 +196,11 @@ final class CompoundFileWriter {
private void copyFile(FileEntry source, OutputStream os, byte buffer[])
throws IOException
{
InputStream is = null;
IndexInput is = null;
try {
long startPtr = os.getFilePointer();
is = directory.openFile(source.file);
is = directory.openInput(source.file);
long length = is.length();
long remainder = length;
int chunk = buffer.length;

12
src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
/** Access to the Field Info file that describes document fields and whether or
* not they are indexed. Each segment has a separate Field Info file. Objects
@ -42,13 +42,13 @@ final class FieldInfos {
/**
* Construct a FieldInfos object using the directory and the name of the file
* InputStream
* @param d The directory to open the InputStream from
* @param name The name of the file to open the InputStream from in the Directory
* IndexInput
* @param d The directory to open the IndexInput from
* @param name The name of the file to open the IndexInput from in the Directory
* @throws IOException
*/
FieldInfos(Directory d, String name) throws IOException {
InputStream input = d.openFile(name);
IndexInput input = d.openInput(name);
try {
read(input);
} finally {
@ -189,7 +189,7 @@ final class FieldInfos {
}
}
private void read(InputStream input) throws IOException {
private void read(IndexInput input) throws IOException {
int size = input.readVInt();//read in the size
for (int i = 0; i < size; i++) {
String name = input.readString().intern();

10
src/java/org/apache/lucene/index/FieldsReader.java
View File

@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
/**
* Class responsible for access to stored document fields.
@ -32,15 +32,15 @@ import org.apache.lucene.store.InputStream;
*/
final class FieldsReader {
private FieldInfos fieldInfos;
private InputStream fieldsStream;
private InputStream indexStream;
private IndexInput fieldsStream;
private IndexInput indexStream;
private int size;
FieldsReader(Directory d, String segment, FieldInfos fn) throws IOException {
fieldInfos = fn;
fieldsStream = d.openFile(segment + ".fdt");
indexStream = d.openFile(segment + ".fdx");
fieldsStream = d.openInput(segment + ".fdt");
indexStream = d.openInput(segment + ".fdx");
size = (int)(indexStream.length() / 8);
}

4
src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.document.Document;
@ -606,7 +606,7 @@ public class IndexWriter {
if (!directory.fileExists("deletable"))
return result;
InputStream input = directory.openFile("deletable");
IndexInput input = directory.openInput("deletable");
try {
for (int i = input.readInt(); i > 0; i--) // read file names
result.addElement(input.readString());

6
src/java/org/apache/lucene/index/SegmentInfos.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.index;
import java.util.Vector;
import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStream;
final class SegmentInfos extends Vector {
@ -37,7 +37,7 @@ final class SegmentInfos extends Vector {
public final void read(Directory directory) throws IOException {
InputStream input = directory.openFile("segments");
IndexInput input = directory.openInput("segments");
try {
int format = input.readInt();
if(format < 0){ // file contains explicit format info
@ -103,7 +103,7 @@ final class SegmentInfos extends Vector {
public static long readCurrentVersion(Directory directory)
throws IOException {
InputStream input = directory.openFile("segments");
IndexInput input = directory.openInput("segments");
int format = 0;
long version = 0;
try {

18
src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -25,7 +25,7 @@ import java.util.Set;
import java.util.Vector;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitVector;
@ -49,20 +49,20 @@ final class SegmentReader extends IndexReader {
private boolean normsDirty = false;
private boolean undeleteAll = false;
InputStream freqStream;
InputStream proxStream;
IndexInput freqStream;
IndexInput proxStream;
// Compound File Reader when based on a compound file segment
CompoundFileReader cfsReader = null;
private class Norm {
public Norm(InputStream in, int number)
public Norm(IndexInput in, int number)
{
this.in = in;
this.number = number;
}
private InputStream in;
private IndexInput in;
private byte[] bytes;
private boolean dirty;
private int number;
@ -123,8 +123,8 @@ final class SegmentReader extends IndexReader {
// make sure that all index files have been read or are kept open
// so that if an index update removes them we'll still have them
freqStream = cfsDir.openFile(segment + ".frq");
proxStream = cfsDir.openFile(segment + ".prx");
freqStream = cfsDir.openInput(segment + ".frq");
proxStream = cfsDir.openInput(segment + ".prx");
openNorms(cfsDir);
if (fieldInfos.hasVectors()) { // open term vector files only as needed
@ -363,7 +363,7 @@ final class SegmentReader extends IndexReader {
return;
}
InputStream normStream = (InputStream) norm.in.clone();
IndexInput normStream = (IndexInput) norm.in.clone();
try { // read from disk
normStream.seek(0);
normStream.readBytes(bytes, offset, maxDoc());
@ -383,7 +383,7 @@ final class SegmentReader extends IndexReader {
fileName = segment + ".f" + fi.number;
d = cfsDir;
}
norms.put(fi.name, new Norm(d.openFile(fileName), fi.number));
norms.put(fi.name, new Norm(d.openInput(fileName), fi.number));
}
}
}

10
src/java/org/apache/lucene/index/SegmentTermDocs.java
View File

@ -18,11 +18,11 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
class SegmentTermDocs implements TermDocs {
protected SegmentReader parent;
private InputStream freqStream;
private IndexInput freqStream;
private int count;
private int df;
private BitVector deletedDocs;
@ -32,7 +32,7 @@ class SegmentTermDocs implements TermDocs {
private int skipInterval;
private int numSkips;
private int skipCount;
private InputStream skipStream;
private IndexInput skipStream;
private int skipDoc;
private long freqPointer;
private long proxPointer;
@ -41,7 +41,7 @@ class SegmentTermDocs implements TermDocs {
SegmentTermDocs(SegmentReader parent) {
this.parent = parent;
this.freqStream = (InputStream) parent.freqStream.clone();
this.freqStream = (IndexInput) parent.freqStream.clone();
this.deletedDocs = parent.deletedDocs;
this.skipInterval = parent.tis.getSkipInterval();
}
@ -147,7 +147,7 @@ class SegmentTermDocs implements TermDocs {
if (df >= skipInterval) { // optimized case
if (skipStream == null)
skipStream = (InputStream) freqStream.clone(); // lazily clone
skipStream = (IndexInput) freqStream.clone(); // lazily clone
if (!haveSkipped) { // lazily seek skip stream
skipStream.seek(skipPointer);

8
src/java/org/apache/lucene/index/SegmentTermEnum.java
View File

@ -17,10 +17,10 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
final class SegmentTermEnum extends TermEnum implements Cloneable {
private InputStream input;
private IndexInput input;
FieldInfos fieldInfos;
long size;
long position = -1;
@ -38,7 +38,7 @@ final class SegmentTermEnum extends TermEnum implements Cloneable {
private char[] buffer = {};
SegmentTermEnum(InputStream i, FieldInfos fis, boolean isi)
SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi)
throws IOException {
input = i;
fieldInfos = fis;
@ -87,7 +87,7 @@ final class SegmentTermEnum extends TermEnum implements Cloneable {
clone = (SegmentTermEnum) super.clone();
} catch (CloneNotSupportedException e) {}
clone.input = (InputStream) input.clone();
clone.input = (IndexInput) input.clone();
clone.termInfo = new TermInfo(termInfo);
if (term != null) clone.growBuffer(term.text.length());

6
src/java/org/apache/lucene/index/SegmentTermPositions.java
View File

@ -18,17 +18,17 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
final class SegmentTermPositions
extends SegmentTermDocs implements TermPositions {
private InputStream proxStream;
private IndexInput proxStream;
private int proxCount;
private int position;
SegmentTermPositions(SegmentReader p) throws IOException {
super(p);
this.proxStream = (InputStream)parent.proxStream.clone();
this.proxStream = (IndexInput)parent.proxStream.clone();
}
final void seek(TermInfo ti) throws IOException {

9
src/java/org/apache/lucene/index/TermInfosReader.java
View File

@ -39,12 +39,17 @@ final class TermInfosReader {
segment = seg;
fieldInfos = fis;
origEnum = new SegmentTermEnum(directory.openFile(segment + ".tis"),
origEnum = new SegmentTermEnum(directory.openInput(segment + ".tis"),
fieldInfos, false);
size = origEnum.size;
readIndex();
}
protected void finalize() {
// patch for pre-1.4.2 JVMs, whose ThreadLocals leak
enumerators.set(null);
}
public int getSkipInterval() {
return origEnum.skipInterval;
}
@ -74,7 +79,7 @@ final class TermInfosReader {
private final void readIndex() throws IOException {
SegmentTermEnum indexEnum =
new SegmentTermEnum(directory.openFile(segment + ".tii"),
new SegmentTermEnum(directory.openInput(segment + ".tii"),
fieldInfos, true);
try {
int indexSize = (int)indexEnum.size;

16
src/java/org/apache/lucene/index/TermVectorsReader.java
View File

@ -17,7 +17,7 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import java.io.IOException;
@ -29,19 +29,19 @@ import java.io.IOException;
class TermVectorsReader {
private FieldInfos fieldInfos;
private InputStream tvx;
private InputStream tvd;
private InputStream tvf;
private IndexInput tvx;
private IndexInput tvd;
private IndexInput tvf;
private int size;
TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos)
throws IOException {
if (d.fileExists(segment + TermVectorsWriter.TVX_EXTENSION)) {
tvx = d.openFile(segment + TermVectorsWriter.TVX_EXTENSION);
tvx = d.openInput(segment + TermVectorsWriter.TVX_EXTENSION);
checkValidFormat(tvx);
tvd = d.openFile(segment + TermVectorsWriter.TVD_EXTENSION);
tvd = d.openInput(segment + TermVectorsWriter.TVD_EXTENSION);
checkValidFormat(tvd);
tvf = d.openFile(segment + TermVectorsWriter.TVF_EXTENSION);
tvf = d.openInput(segment + TermVectorsWriter.TVF_EXTENSION);
checkValidFormat(tvf);
size = (int) tvx.length() / 8;
}
@ -49,7 +49,7 @@ class TermVectorsReader {
this.fieldInfos = fieldInfos;
}
private void checkValidFormat(InputStream in) throws IOException
private void checkValidFormat(IndexInput in) throws IOException
{
int format = in.readInt();
if (format > TermVectorsWriter.FORMAT_VERSION)

109
src/java/org/apache/lucene/store/BufferedIndexInput.java Normal file
View File

@ -0,0 +1,109 @@
package org.apache.lucene.store;
/**
* Copyright 2004 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
/** Base implementation class for buffered {@link IndexInput}. */
public abstract class BufferedIndexInput extends IndexInput {
static final int BUFFER_SIZE = OutputStream.BUFFER_SIZE;
private byte[] buffer;
private long bufferStart = 0; // position in file of buffer
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
public byte readByte() throws IOException {
if (bufferPosition >= bufferLength)
refill();
return buffer[bufferPosition++];
}
public void readBytes(byte[] b, int offset, int len)
throws IOException {
if (len < BUFFER_SIZE) {
for (int i = 0; i < len; i++) // read byte-by-byte
b[i + offset] = (byte)readByte();
} else { // read all-at-once
long start = getFilePointer();
seekInternal(start);
readInternal(b, offset, len);
bufferStart = start + len; // adjust stream variables
bufferPosition = 0;
bufferLength = 0; // trigger refill() on read
}
}
private void refill() throws IOException {
long start = bufferStart + bufferPosition;
long end = start + BUFFER_SIZE;
if (end > length()) // don't read past EOF
end = length();
bufferLength = (int)(end - start);
if (bufferLength == 0)
throw new IOException("read past EOF");
if (buffer == null)
buffer = new byte[BUFFER_SIZE]; // allocate buffer lazily
readInternal(buffer, 0, bufferLength);
bufferStart = start;
bufferPosition = 0;
}
/** Expert: implements buffer refill. Reads bytes from the current position
* in the input.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param length the number of bytes to read
*/
protected abstract void readInternal(byte[] b, int offset, int length)
throws IOException;
public long getFilePointer() { return bufferStart + bufferPosition; }
public void seek(long pos) throws IOException {
if (pos >= bufferStart && pos < (bufferStart + bufferLength))
bufferPosition = (int)(pos - bufferStart); // seek within buffer
else {
bufferStart = pos;
bufferPosition = 0;
bufferLength = 0; // trigger refill() on read()
seekInternal(pos);
}
}
/** Expert: implements seek. Sets current position in this file, where the
* next {@link #readInternal(byte[],int,int)} will occur.
* @see #readInternal(byte[],int,int)
*/
protected abstract void seekInternal(long pos) throws IOException;
public Object clone() {
BufferedIndexInput clone = (BufferedIndexInput)super.clone();
if (buffer != null) {
clone.buffer = new byte[BUFFER_SIZE];
System.arraycopy(buffer, 0, clone.buffer, 0, bufferLength);
}
return clone;
}
}

13
src/java/org/apache/lucene/store/Directory.java
View File

@ -67,9 +67,18 @@ public abstract class Directory {
public abstract OutputStream createFile(String name)
throws IOException;
/** @deprecated use {@link openInput(String)}. */
public InputStream openFile(String name) throws IOException {
return (InputStream)openInput(name);
}
/** Returns a stream reading an existing file. */
public abstract InputStream openFile(String name)
throws IOException;
public IndexInput openInput(String name)
throws IOException {
// default implementation for back compatibility
// this method should be abstract
return (IndexInput)openFile(name);
}
/** Construct a {@link Lock}.
* @param name the name of the lock file

26
src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -56,6 +56,19 @@ public class FSDirectory extends Directory {
System.getProperty("org.apache.lucene.lockdir",
System.getProperty("java.io.tmpdir"));
/** The default class which implements filesystem-based directories. */
private static final Class IMPL;
static {
try {
String name =
System.getProperty("org.apache.lucene.FSDirectory.class",
FSDirectory.class.getName());
IMPL = Class.forName(name);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
private static MessageDigest DIGESTER;
static {
@ -99,7 +112,12 @@ public class FSDirectory extends Directory {
synchronized (DIRECTORIES) {
dir = (FSDirectory)DIRECTORIES.get(file);
if (dir == null) {
dir = new FSDirectory(file, create);
try {
dir = (FSDirectory)IMPL.newInstance();
} catch (Exception e) {
throw new RuntimeException(e);
}
dir.init(file, create);
DIRECTORIES.put(file, dir);
} else if (create) {
dir.create();
@ -115,7 +133,9 @@ public class FSDirectory extends Directory {
private int refCount;
private File lockDir;
private FSDirectory(File path, boolean create) throws IOException {
protected FSDirectory() {}; // permit subclassing
private void init(File path, boolean create) throws IOException {
directory = path;
if (LOCK_DIR == null) {
@ -360,7 +380,7 @@ public class FSDirectory extends Directory {
/** For debug output. */
public String toString() {
return "FSDirectory@" + directory;
return this.getClass().getName() + "@" + directory;
}
}

156
src/java/org/apache/lucene/store/IndexInput.java Normal file
View File

@ -0,0 +1,156 @@
package org.apache.lucene.store;
/**
* Copyright 2004 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
/** Abstract base class for input from a file in a {@link Directory}. A
* random-access input stream. Used for all Lucene index input operations.
* @see Directory
*/
public abstract class IndexInput implements Cloneable {
private char[] chars; // used by readString()
/** Reads and returns a single byte.
* @see OutputStream#writeByte(byte)
*/
public abstract byte readByte() throws IOException;
/** Reads a specified number of bytes into an array at the specified offset.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param len the number of bytes to read
* @see OutputStream#writeBytes(byte[],int)
*/
public abstract void readBytes(byte[] b, int offset, int len)
throws IOException;
/** Reads four bytes and returns an int.
* @see OutputStream#writeInt(int)
*/
public int readInt() throws IOException {
return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
| ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
}
/** Reads an int stored in variable-length format. Reads between one and
* five bytes. Smaller values take fewer bytes. Negative numbers are not
* supported.
* @see OutputStream#writeVInt(int)
*/
public int readVInt() throws IOException {
byte b = readByte();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = readByte();
i |= (b & 0x7F) << shift;
}
return i;
}
/** Reads eight bytes and returns a long.
* @see OutputStream#writeLong(long)
*/
public long readLong() throws IOException {
return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
}
/** Reads a long stored in variable-length format. Reads between one and
* nine bytes. Smaller values take fewer bytes. Negative numbers are not
* supported. */
public long readVLong() throws IOException {
byte b = readByte();
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = readByte();
i |= (b & 0x7FL) << shift;
}
return i;
}
/** Reads a string.
* @see OutputStream#writeString(String)
*/
public String readString() throws IOException {
int length = readVInt();
if (chars == null || length > chars.length)
chars = new char[length];
readChars(chars, 0, length);
return new String(chars, 0, length);
}
/** Reads UTF-8 encoded characters into an array.
* @param buffer the array to read characters into
* @param start the offset in the array to start storing characters
* @param length the number of characters to read
* @see OutputStream#writeChars(String,int,int)
*/
public void readChars(char[] buffer, int start, int length)
throws IOException {
final int end = start + length;
for (int i = start; i < end; i++) {
byte b = readByte();
if ((b & 0x80) == 0)
buffer[i] = (char)(b & 0x7F);
else if ((b & 0xE0) != 0xE0) {
buffer[i] = (char)(((b & 0x1F) << 6)
| (readByte() & 0x3F));
} else
buffer[i] = (char)(((b & 0x0F) << 12)
| ((readByte() & 0x3F) << 6)
| (readByte() & 0x3F));
}
}
/** Closes the stream to futher operations. */
public abstract void close() throws IOException;
/** Returns the current position in this file, where the next read will
* occur.
* @see #seek(long)
*/
public abstract long getFilePointer();
/** Sets current position in this file, where the next read will occur.
* @see #getFilePointer()
*/
public abstract void seek(long pos) throws IOException;
/** The number of bytes in the file. */
public abstract long length();
/** Returns a clone of this stream.
*
* <p>Clones of a stream access the same data, and are positioned at the same
* point as the stream they were cloned from.
*
* <p>Expert: Subclasses must ensure that clones may be positioned at
* different points in the input from each other and from the stream they
* were cloned from.
*/
public Object clone() {
IndexInput clone = null;
try {
clone = (IndexInput)super.clone();
} catch (CloneNotSupportedException e) {}
clone.chars = null;
return clone;
}
}

210
src/java/org/apache/lucene/store/InputStream.java
View File

@ -18,217 +18,13 @@ package org.apache.lucene.store;
import java.io.IOException;
/** Abstract base class for input from a file in a {@link Directory}. A
* random-access input stream. Used for all Lucene index input operations.
* @see Directory
* @see OutputStream
*/
public abstract class InputStream implements Cloneable {
static final int BUFFER_SIZE = OutputStream.BUFFER_SIZE;
private byte[] buffer;
private char[] chars;
private long bufferStart = 0; // position in file of buffer
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
/** @deprecated Use {@link IndexInput} or {@link BufferedIndexInput} instead.*/
public abstract class InputStream extends BufferedIndexInput {
protected long length; // set by subclasses
/** Reads and returns a single byte.
* @see OutputStream#writeByte(byte)
*/
public final byte readByte() throws IOException {
if (bufferPosition >= bufferLength)
refill();
return buffer[bufferPosition++];
}
/** Reads a specified number of bytes into an array at the specified offset.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param len the number of bytes to read
* @see OutputStream#writeBytes(byte[],int)
*/
public final void readBytes(byte[] b, int offset, int len)
throws IOException {
if (len < BUFFER_SIZE) {
for (int i = 0; i < len; i++) // read byte-by-byte
b[i + offset] = (byte)readByte();
} else { // read all-at-once
long start = getFilePointer();
seekInternal(start);
readInternal(b, offset, len);
bufferStart = start + len; // adjust stream variables
bufferPosition = 0;
bufferLength = 0; // trigger refill() on read
}
}
/** Reads four bytes and returns an int.
* @see OutputStream#writeInt(int)
*/
public final int readInt() throws IOException {
return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
| ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
}
/** Reads an int stored in variable-length format. Reads between one and
* five bytes. Smaller values take fewer bytes. Negative numbers are not
* supported.
* @see OutputStream#writeVInt(int)
*/
public final int readVInt() throws IOException {
byte b = readByte();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = readByte();
i |= (b & 0x7F) << shift;
}
return i;
}
/** Reads eight bytes and returns a long.
* @see OutputStream#writeLong(long)
*/
public final long readLong() throws IOException {
return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
}
/** Reads a long stored in variable-length format. Reads between one and
* nine bytes. Smaller values take fewer bytes. Negative numbers are not
* supported. */
public final long readVLong() throws IOException {
byte b = readByte();
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = readByte();
i |= (b & 0x7FL) << shift;
}
return i;
}
/** Reads a string.
* @see OutputStream#writeString(String)
*/
public final String readString() throws IOException {
int length = readVInt();
if (chars == null || length > chars.length)
chars = new char[length];
readChars(chars, 0, length);
return new String(chars, 0, length);
}
/** Reads UTF-8 encoded characters into an array.
* @param buffer the array to read characters into
* @param start the offset in the array to start storing characters
* @param length the number of characters to read
* @see OutputStream#writeChars(String,int,int)
*/
public final void readChars(char[] buffer, int start, int length)
throws IOException {
final int end = start + length;
for (int i = start; i < end; i++) {
byte b = readByte();
if ((b & 0x80) == 0)
buffer[i] = (char)(b & 0x7F);
else if ((b & 0xE0) != 0xE0) {
buffer[i] = (char)(((b & 0x1F) << 6)
| (readByte() & 0x3F));
} else
buffer[i] = (char)(((b & 0x0F) << 12)
| ((readByte() & 0x3F) << 6)
| (readByte() & 0x3F));
}
}
private void refill() throws IOException {
long start = bufferStart + bufferPosition;
long end = start + BUFFER_SIZE;
if (end > length) // don't read past EOF
end = length;
bufferLength = (int)(end - start);
if (bufferLength == 0)
throw new IOException("read past EOF");
if (buffer == null)
buffer = new byte[BUFFER_SIZE]; // allocate buffer lazily
readInternal(buffer, 0, bufferLength);
bufferStart = start;
bufferPosition = 0;
}
/** Expert: implements buffer refill. Reads bytes from the current position
* in the input.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param length the number of bytes to read
*/
protected abstract void readInternal(byte[] b, int offset, int length)
throws IOException;
/** Closes the stream to futher operations. */
public abstract void close() throws IOException;
/** Returns the current position in this file, where the next read will
* occur.
* @see #seek(long)
*/
public final long getFilePointer() {
return bufferStart + bufferPosition;
}
/** Sets current position in this file, where the next read will occur.
* @see #getFilePointer()
*/
public final void seek(long pos) throws IOException {
if (pos >= bufferStart && pos < (bufferStart + bufferLength))
bufferPosition = (int)(pos - bufferStart); // seek within buffer
else {
bufferStart = pos;
bufferPosition = 0;
bufferLength = 0; // trigger refill() on read()
seekInternal(pos);
}
}
/** Expert: implements seek. Sets current position in this file, where the
* next {@link #readInternal(byte[],int,int)} will occur.
* @see #readInternal(byte[],int,int)
*/
protected abstract void seekInternal(long pos) throws IOException;
/** The number of bytes in the file. */
public final long length() {
public long length() {
return length;
}
/** Returns a clone of this stream.
*
* <p>Clones of a stream access the same data, and are positioned at the same
* point as the stream they were cloned from.
*
* <p>Expert: Subclasses must ensure that clones may be positioned at
* different points in the input from each other and from the stream they
* were cloned from.
*/
public Object clone() {
InputStream clone = null;
try {
clone = (InputStream)super.clone();
} catch (CloneNotSupportedException e) {}
if (buffer != null) {
clone.buffer = new byte[BUFFER_SIZE];
System.arraycopy(buffer, 0, clone.buffer, 0, bufferLength);
}
clone.chars = null;
return clone;
}
}

4
src/java/org/apache/lucene/util/BitVector.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStream;
/** Optimized implementation of a vector of bits. This is more-or-less like
@ -122,7 +122,7 @@ public final class BitVector {
<code>d</code>, as written by the {@link #write} method.
*/
public BitVector(Directory d, String name) throws IOException {
InputStream input = d.openFile(name);
IndexInput input = d.openInput(name);
try {
size = input.readInt(); // read size
count = input.readInt(); // read count

2
src/java/overview.html
View File

@ -17,7 +17,7 @@ and <a href="org/apache/lucene/util/PriorityQueue.html">PriorityQueue</a>.</li>
<b><a href="org/apache/lucene/store/package-summary.html">org.apache.lucene.store</a></b>
defines an abstract class for storing persistent data, the <a href="org/apache/lucene/store/Directory.html">Directory</a>,
a collection of named files written by an <a href="org/apache/lucene/store/OutputStream.html">OutputStream</a>
and read by an <a href="org/apache/lucene/store/InputStream.html">InputStream</a>.&nbsp;
and read by an <a href="org/apache/lucene/store/IndexInput.html">IndexInput</a>.&nbsp;
Two implementations are provided, <a href="org/apache/lucene/store/FSDirectory.html">FSDirectory</a>,
which uses a file system directory to store files, and <a href="org/apache/lucene/store/RAMDirectory.html">RAMDirectory</a>
which implements files as memory-resident data structures.</li>

4
src/test/org/apache/lucene/StoreTest.java
View File

@ -17,7 +17,7 @@ package org.apache.lucene;
*/
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
@ -84,7 +84,7 @@ class StoreTest {
byte b = (byte)(gen.nextInt() & 0x7F);
//System.out.println("reading " + name + " with " + length + " of " + b);
InputStream file = store.openFile(name);
IndexInput file = store.openInput(name);
if (file.length() != length)
throw new Exception("length incorrect");

12
src/test/org/apache/lucene/index/MockInputStream.java → src/test/org/apache/lucene/index/MockIndexInput.java
View File

@ -16,15 +16,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.BufferedIndexInput;
import java.io.IOException;
public class MockInputStream extends InputStream {
public class MockIndexInput extends BufferedIndexInput {
private byte[] buffer;
private int pointer = 0;
private long length;
public MockInputStream(byte[] bytes) {
public MockIndexInput(byte[] bytes) {
buffer = bytes;
length = bytes.length;
}
@ -53,4 +54,9 @@ public class MockInputStream extends InputStream {
protected void seekInternal(long pos) throws IOException {
pointer = (int) pos;
}
public long length() {
return length;
}
}

82
src/test/org/apache/lucene/index/TestCompoundFile.java
View File

@ -24,7 +24,7 @@ import junit.framework.TestSuite;
import junit.textui.TestRunner;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store._TestHelper;
@ -94,8 +94,8 @@ public class TestCompoundFile extends TestCase
private void assertSameStreams(String msg,
InputStream expected,
InputStream test)
IndexInput expected,
IndexInput test)
throws IOException
{
assertNotNull(msg + " null expected", expected);
@ -120,8 +120,8 @@ public class TestCompoundFile extends TestCase
private void assertSameStreams(String msg,
InputStream expected,
InputStream actual,
IndexInput expected,
IndexInput actual,
long seekTo)
throws IOException
{
@ -136,8 +136,8 @@ public class TestCompoundFile extends TestCase
private void assertSameSeekBehavior(String msg,
InputStream expected,
InputStream actual)
IndexInput expected,
IndexInput actual)
throws IOException
{
// seek to 0
@ -199,8 +199,8 @@ public class TestCompoundFile extends TestCase
csw.close();
CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
InputStream expected = dir.openFile(name);
InputStream actual = csr.openFile(name);
IndexInput expected = dir.openInput(name);
IndexInput actual = csr.openInput(name);
assertSameStreams(name, expected, actual);
assertSameSeekBehavior(name, expected, actual);
expected.close();
@ -223,15 +223,15 @@ public class TestCompoundFile extends TestCase
csw.close();
CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
InputStream expected = dir.openFile("d1");
InputStream actual = csr.openFile("d1");
IndexInput expected = dir.openInput("d1");
IndexInput actual = csr.openInput("d1");
assertSameStreams("d1", expected, actual);
assertSameSeekBehavior("d1", expected, actual);
expected.close();
actual.close();
expected = dir.openFile("d2");
actual = csr.openFile("d2");
expected = dir.openInput("d2");
actual = csr.openInput("d2");
assertSameStreams("d2", expected, actual);
assertSameSeekBehavior("d2", expected, actual);
expected.close();
@ -279,8 +279,8 @@ public class TestCompoundFile extends TestCase
CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
for (int i=0; i<data.length; i++) {
InputStream check = dir.openFile(segment + data[i]);
InputStream test = csr.openFile(segment + data[i]);
IndexInput check = dir.openInput(segment + data[i]);
IndexInput test = csr.openInput(segment + data[i]);
assertSameStreams(data[i], check, test);
assertSameSeekBehavior(data[i], check, test);
test.close();
@ -319,9 +319,9 @@ public class TestCompoundFile extends TestCase
}
os.close();
InputStream in = fsdir.openFile(file);
IndexInput in = fsdir.openInput(file);
// This read primes the buffer in InputStream
// This read primes the buffer in IndexInput
byte b = in.readByte();
// Close the file
@ -343,14 +343,14 @@ public class TestCompoundFile extends TestCase
}
static boolean isCSInputStream(InputStream is) {
return is instanceof CompoundFileReader.CSInputStream;
static boolean isCSIndexInput(IndexInput is) {
return is instanceof CompoundFileReader.CSIndexInput;
}
static boolean isCSInputStreamOpen(InputStream is) throws IOException {
if (isCSInputStream(is)) {
CompoundFileReader.CSInputStream cis =
(CompoundFileReader.CSInputStream) is;
static boolean isCSIndexInputOpen(IndexInput is) throws IOException {
if (isCSIndexInput(is)) {
CompoundFileReader.CSIndexInput cis =
(CompoundFileReader.CSIndexInput) is;
return _TestHelper.isFSInputStreamOpen(cis.base);
} else {
@ -364,14 +364,14 @@ public class TestCompoundFile extends TestCase
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// basic clone
InputStream expected = dir.openFile("f11");
IndexInput expected = dir.openInput("f11");
assertTrue(_TestHelper.isFSInputStreamOpen(expected));
InputStream one = cr.openFile("f11");
assertTrue(isCSInputStreamOpen(one));
IndexInput one = cr.openInput("f11");
assertTrue(isCSIndexInputOpen(one));
InputStream two = (InputStream) one.clone();
assertTrue(isCSInputStreamOpen(two));
IndexInput two = (IndexInput) one.clone();
assertTrue(isCSIndexInputOpen(two));
assertSameStreams("basic clone one", expected, one);
expected.seek(0);
@ -379,7 +379,7 @@ public class TestCompoundFile extends TestCase
// Now close the first stream
one.close();
assertTrue("Only close when cr is closed", isCSInputStreamOpen(one));
assertTrue("Only close when cr is closed", isCSIndexInputOpen(one));
// The following should really fail since we couldn't expect to
// access a file once close has been called on it (regardless of
@ -391,8 +391,8 @@ public class TestCompoundFile extends TestCase
// Now close the compound reader
cr.close();
assertFalse("Now closed one", isCSInputStreamOpen(one));
assertFalse("Now closed two", isCSInputStreamOpen(two));
assertFalse("Now closed one", isCSIndexInputOpen(one));
assertFalse("Now closed two", isCSIndexInputOpen(two));
// The following may also fail since the compound stream is closed
expected.seek(0);
@ -418,11 +418,11 @@ public class TestCompoundFile extends TestCase
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
InputStream e1 = dir.openFile("f11");
InputStream e2 = dir.openFile("f3");
IndexInput e1 = dir.openInput("f11");
IndexInput e2 = dir.openInput("f3");
InputStream a1 = cr.openFile("f11");
InputStream a2 = dir.openFile("f3");
IndexInput a1 = cr.openInput("f11");
IndexInput a2 = dir.openInput("f3");
// Seek the first pair
e1.seek(100);
@ -497,11 +497,11 @@ public class TestCompoundFile extends TestCase
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
// Open two files
InputStream e1 = cr.openFile("f11");
InputStream e2 = cr.openFile("f3");
IndexInput e1 = cr.openInput("f11");
IndexInput e2 = cr.openInput("f3");
InputStream a1 = (InputStream) e1.clone();
InputStream a2 = (InputStream) e2.clone();
IndexInput a1 = (IndexInput) e1.clone();
IndexInput a2 = (IndexInput) e2.clone();
// Seek the first pair
e1.seek(100);
@ -575,7 +575,7 @@ public class TestCompoundFile extends TestCase
// Open two files
try {
InputStream e1 = cr.openFile("bogus");
IndexInput e1 = cr.openInput("bogus");
fail("File not found");
} catch (IOException e) {
@ -590,7 +590,7 @@ public class TestCompoundFile extends TestCase
public void testReadPastEOF() throws IOException {
setUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
InputStream is = cr.openFile("f2");
IndexInput is = cr.openInput("f2");
is.seek(is.length() - 10);
byte b[] = new byte[100];
is.readBytes(b, 0, 10);

14
src/test/org/apache/lucene/index/TestInputStream.java → src/test/org/apache/lucene/index/TestIndexInput.java
View File

@ -17,17 +17,17 @@ package org.apache.lucene.index;
*/
import junit.framework.TestCase;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.IndexInput;
import java.io.IOException;
public class TestInputStream extends TestCase {
public class TestIndexInput extends TestCase {
public void testRead() throws IOException {
InputStream is = new MockInputStream(new byte[] { (byte) 0x80, 0x01,
(byte) 0xFF, 0x7F,
(byte) 0x80, (byte) 0x80, 0x01,
(byte) 0x81, (byte) 0x80, 0x01,
0x06, 'L', 'u', 'c', 'e', 'n', 'e'});
IndexInput is = new MockIndexInput(new byte[] { (byte) 0x80, 0x01,
(byte) 0xFF, 0x7F,
(byte) 0x80, (byte) 0x80, 0x01,
(byte) 0x81, (byte) 0x80, 0x01,
0x06, 'L', 'u', 'c', 'e', 'n', 'e'});
assertEquals(128,is.readVInt());
assertEquals(16383,is.readVInt());
assertEquals(16384,is.readVInt());

6
src/test/org/apache/lucene/store/_TestHelper.java
View File

@ -10,14 +10,14 @@ public class _TestHelper {
/** Returns true if the instance of the provided input stream is actually
* an FSInputStream.
*/
public static boolean isFSInputStream(InputStream is) {
public static boolean isFSInputStream(IndexInput is) {
return is instanceof FSInputStream;
}
/** Returns true if the provided input stream is an FSInputStream and
* is a clone, that is it does not own its underlying file descriptor.
*/
public static boolean isFSInputStreamClone(InputStream is) {
public static boolean isFSInputStreamClone(IndexInput is) {
if (isFSInputStream(is)) {
return ((FSInputStream) is).isClone;
} else {
@ -32,7 +32,7 @@ public class _TestHelper {
* FSInputStream that owns this descriptor is closed. However, the
* descriptor may possibly become invalid in other ways as well.
*/
public static boolean isFSInputStreamOpen(InputStream is)
public static boolean isFSInputStreamOpen(IndexInput is)
throws IOException
{
if (isFSInputStream(is)) {