Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- Added ASL and removed trailing spaces. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150094 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Otis Gospodnetic 2003-10-13 14:18:04 +00:00
parent 34df355b2d
commit 59779c7455
2 changed files with 227 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
src/java/org/apache/lucene/index/CompoundFileReader.java
View File

@ -1,4 +1,59 @@
package org.apache.lucene.index;
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.InputStream;
import org.apache.lucene.store.OutputStream;
@ -9,133 +64,134 @@ import java.util.Iterator;
import java.io.IOException;
/** Class for accessing a compound stream.
* This class implements a directory, but is limited to only read operations.
* Directory methods that would normally modify data throw an exception.
/**
* Class for accessing a compound stream.
* This class implements a directory, but is limited to only read operations.
* Directory methods that would normally modify data throw an exception.
*
* @author Dmitry Serebrennikov
* @version $Id$
*/
public class CompoundFileReader extends Directory {
private static final class FileEntry {
long offset;
long length;
};
}
// Base info
private Directory directory;
private String fileName;
// Reference count
private boolean open;
private InputStream stream;
private HashMap entries = new HashMap();
public CompoundFileReader(Directory dir, String name)
public CompoundFileReader(Directory dir, String name)
throws IOException
{
directory = dir;
fileName = name;
boolean success = false;
try {
stream = dir.openFile(name);
// read the directory and init files
int count = stream.readVInt();
FileEntry entry = null;
for (int i=0; i<count; i++) {
long offset = stream.readLong();
String id = stream.readString();
if (entry != null) {
// set length of the previous entry
entry.length = offset - entry.offset;
}
entry = new FileEntry();
entry.offset = offset;
entries.put(id, entry);
}
// set the length of the final entry
if (entry != null) {
entry.length = stream.length() - entry.offset;
}
success = true;
} finally {
if (! success) {
try {
stream.close();
} catch (IOException e) { }
}
}
}
}
public Directory getDirectory() {
return directory;
}
public String getName() {
return fileName;
}
public synchronized void close() throws IOException {
if (stream == null)
throw new IOException("Already closed");
entries.clear();
stream.close();
stream = null;
}
public synchronized InputStream openFile(String id)
public synchronized InputStream openFile(String id)
throws IOException
{
if (stream == null)
if (stream == null)
throw new IOException("Stream closed");
FileEntry entry = (FileEntry) entries.get(id);
if (entry == null)
throw new IOException("No sub-file with id " + id + " found");
return new CSInputStream(stream, entry.offset, entry.length);
}
/** Returns an array of strings, one for each file in the directory. */
public String[] list() {
String res[] = new String[entries.size()];
return (String[]) entries.keySet().toArray(res);
}
/** Returns true iff a file with the given name exists. */
public boolean fileExists(String name) {
return entries.containsKey(name);
}
/** Returns the time the named file was last modified. */
public long fileModified(String name) throws IOException {
return directory.fileModified(fileName);
}
/** Set the modified time of an existing file to now. */
public void touchFile(String name) throws IOException {
directory.touchFile(fileName);
}
/** Removes an existing file in the directory. */
public void deleteFile(String name)
public void deleteFile(String name)
{
throw new UnsupportedOperationException();
}
/** Renames an existing file in the directory.
If a file already exists with the new name, then it is replaced.
This replacement should be atomic. */
@ -143,8 +199,7 @@ public class CompoundFileReader extends Directory {
{
throw new UnsupportedOperationException();
}
/** Returns the length of a file in the directory. */
public long fileLength(String name)
throws IOException
@ -154,22 +209,21 @@ public class CompoundFileReader extends Directory {
throw new IOException("File " + name + " does not exist");
return e.length;
}
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
public OutputStream createFile(String name)
{
throw new UnsupportedOperationException();
}
/** Construct a {@link Lock}.
* @param name the name of the lock file
*/
public Lock makeLock(String name)
public Lock makeLock(String name)
{
throw new UnsupportedOperationException();
}
/** Implementation of an InputStream that reads from a portion of the
* compound file. The visibility is left as "package" *only* because
@ -177,13 +231,13 @@ public class CompoundFileReader extends Directory {
* can then access package fields of this class.
*/
static final class CSInputStream extends InputStream {
InputStream base;
long fileOffset;
CSInputStream(final InputStream base,
final long fileOffset,
final long length)
CSInputStream(final InputStream base,
final long fileOffset,
final long length)
throws IOException
{
this.base = (InputStream) base.clone();
@ -191,21 +245,20 @@ public class CompoundFileReader extends Directory {
this.length = length; // variable in the superclass
seekInternal(0); // position to the adjusted 0th byte
}
/** Expert: implements buffer refill. Reads bytes from the current
/** Expert: implements buffer refill. Reads bytes from the current
* position in the input.
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param length the number of bytes to read
*/
protected void readInternal(byte[] b, int offset, int len)
throws IOException
throws IOException
{
base.readBytes(b, offset, len);
}
/** Expert: implements seek. Sets current position in this file, where
/** Expert: implements seek. Sets current position in this file, where
* the next {@link #readInternal(byte[],int,int)} will occur.
* @see #readInternal(byte[],int,int)
*/
@ -213,10 +266,10 @@ public class CompoundFileReader extends Directory {
{
if (pos > 0 && pos >= length)
throw new IOException("Seek past the end of file");
if (pos < 0)
throw new IOException("Seek to a negative offset");
base.seek(fileOffset + pos);
}
@ -225,8 +278,7 @@ public class CompoundFileReader extends Directory {
{
base.close();
}
/** Returns a clone of this stream.
*
* <p>Clones of a stream access the same data, and are positioned at the same
@ -240,8 +292,6 @@ public class CompoundFileReader extends Directory {
CSInputStream other = (CSInputStream) super.clone();
other.base = (InputStream) base.clone();
return other;
}
}
}
}
}

177
src/java/org/apache/lucene/index/CompoundFileWriter.java
View File

@ -1,4 +1,59 @@
package org.apache.lucene.index;
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.OutputStream;
import org.apache.lucene.store.InputStream;
@ -8,36 +63,40 @@ import java.util.Iterator;
import java.io.IOException;
/** Combines multiple files into a single compound file.
* The file format:<br>
* <ul>
* <li>VInt fileCount</li>
* <li>{Directory}
* fileCount entries with the following structure:</li>
* <ul>
* <li>long dataOffset</li>
* <li>UTFString extension</li>
* </ul>
* <li>{File Data}
* fileCount entries with the raw data of the corresponding file</li>
* </ul>
*
* The fileCount integer indicates how many files are contained in this compound
* file. The {directory} that follows has that many entries. Each directory entry
* contains an encoding identifier, an long pointer to the start of this file's
* data section, and a UTF String with that file's extension.
/**
* Combines multiple files into a single compound file.
* The file format:<br>
* <ul>
* <li>VInt fileCount</li>
* <li>{Directory}
* fileCount entries with the following structure:</li>
* <ul>
* <li>long dataOffset</li>
* <li>UTFString extension</li>
* </ul>
* <li>{File Data}
* fileCount entries with the raw data of the corresponding file</li>
* </ul>
*
* The fileCount integer indicates how many files are contained in this compound
* file. The {directory} that follows has that many entries. Each directory entry
* contains an encoding identifier, an long pointer to the start of this file's
* data section, and a UTF String with that file's extension.
*
* @author Dmitry Serebrennikov
* @version $Id$
*/
final class CompoundFileWriter {
private static final class FileEntry {
/** source file */
String file;
String file;
/** temporary holder for the start of directory entry for this file */
long directoryOffset;
long directoryOffset;
/** temporary holder for the start of this file's data section */
long dataOffset;
long dataOffset;
}
@ -46,7 +105,7 @@ final class CompoundFileWriter {
private HashSet ids;
private LinkedList entries;
private boolean merged = false;
/** Create the compound stream in the specified file. The file name is the
* entire name (no extensions are added).
@ -56,24 +115,23 @@ final class CompoundFileWriter {
throw new IllegalArgumentException("Missing directory");
if (name == null)
throw new IllegalArgumentException("Missing name");
directory = dir;
fileName = name;
ids = new HashSet();
entries = new LinkedList();
}
/** Returns the directory of the compound file. */
public Directory getDirectory() {
return directory;
}
/** Returns the name of the compound file. */
public String getName() {
return fileName;
}
/** Add a source stream. If sourceDir is null, it is set to the
* same value as the directory where this compound stream exists.
* The id is the string by which the sub-stream will be know in the
@ -84,44 +142,44 @@ final class CompoundFileWriter {
if (merged)
throw new IllegalStateException(
"Can't add extensions after merge has been called");
if (file == null)
throw new IllegalArgumentException(
"Missing source file");
if (! ids.add(file))
if (! ids.add(file))
throw new IllegalArgumentException(
"File " + file + " already added");
FileEntry entry = new FileEntry();
entry.file = file;
entries.add(entry);
}
/** Merge files with the extensions added up to now.
* All files with these extensions are combined sequentially into the
* All files with these extensions are combined sequentially into the
* compound stream. After successful merge, the source files
* are deleted.
*/
*/
public void close() throws IOException {
if (merged)
throw new IllegalStateException(
"Merge already performed");
if (entries.isEmpty())
throw new IllegalStateException(
"No entries to merge have been defined");
merged = true;
// open the compound stream
OutputStream os = null;
try {
os = directory.createFile(fileName);
// Write the number of entries
os.writeVInt(entries.size());
// Write the directory with all offsets at 0.
// Remember the positions of directory entries so that we can
// adjust the offsets later
@ -132,17 +190,17 @@ final class CompoundFileWriter {
os.writeLong(0); // for now
os.writeString(fe.file);
}
// Open the files and copy their data into the stream.
// Remeber the locations of each file's data section.
byte buffer[] = new byte[1024];
it = entries.iterator();
while(it.hasNext()) {
FileEntry fe = (FileEntry) it.next();
fe.dataOffset = os.getFilePointer();
fe.dataOffset = os.getFilePointer();
copyFile(fe, os, buffer);
}
// Write the data offsets into the directory of the compound stream
it = entries.iterator();
while(it.hasNext()) {
@ -150,61 +208,60 @@ final class CompoundFileWriter {
os.seek(fe.directoryOffset);
os.writeLong(fe.dataOffset);
}
// Close the output stream. Set the os to null before trying to
// close so that if an exception occurs during the close, the
// close so that if an exception occurs during the close, the
// finally clause below will not attempt to close the stream
// the second time.
OutputStream tmp = os;
os = null;
tmp.close();
} finally {
if (os != null) try { os.close(); } catch (IOException e) { }
}
}
/** Copy the contents of the file with specified extension into the
* provided output stream. Use the provided buffer for moving data
* to reduce memory allocation.
*/
private void copyFile(FileEntry source, OutputStream os, byte buffer[])
private void copyFile(FileEntry source, OutputStream os, byte buffer[])
throws IOException
{
InputStream is = null;
try {
long startPtr = os.getFilePointer();
is = directory.openFile(source.file);
long length = is.length();
long remainder = length;
int chunk = buffer.length;
while(remainder > 0) {
int len = (int) Math.min(chunk, remainder);
is.readBytes(buffer, 0, len);
os.writeBytes(buffer, len);
remainder -= len;
}
// Verify that remainder is 0
if (remainder != 0)
if (remainder != 0)
throw new IOException(
"Non-zero remainder length after copying: " + remainder
+ " (id: " + source.file + ", length: " + length
+ " (id: " + source.file + ", length: " + length
+ ", buffer size: " + chunk + ")");
// Verify that the output length diff is equal to original file
long endPtr = os.getFilePointer();
long diff = endPtr - startPtr;
if (diff != length)
if (diff != length)
throw new IOException(
"Difference in the output file offsets " + diff
+ " does not match the original file length " + length);
} finally {
} finally {
if (is != null) is.close();
}
}
}
}