Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

building of compound file taken out of commit lock 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150419 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Christoph Goller 2004-08-11 17:38:35 +00:00
parent a715799797
commit 76e2362954
4 changed files with 84 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -399,8 +399,8 @@ public class IndexWriter {
optimize(); // start with zero or 1 seg optimize(); // start with zero or 1 seg
String mergedName = newSegmentName(); final String mergedName = newSegmentName();
final SegmentMerger merger = new SegmentMerger(directory, mergedName); SegmentMerger merger = new SegmentMerger(directory, mergedName);
final Vector segmentsToDelete = new Vector(); final Vector segmentsToDelete = new Vector();
IndexReader sReader = null; IndexReader sReader = null;
@ -422,16 +422,29 @@ public class IndexWriter {
sReader.close(); sReader.close();
synchronized (directory) { // in- & inter-process sync synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock("commit.lock"), COMMIT_LOCK_TIMEOUT) { new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException { public Object doBody() throws IOException {
segmentInfos.write(directory); // commit changes segmentInfos.write(directory); // commit changes
deleteSegments(segmentsToDelete); // delete now-unused segments deleteSegments(segmentsToDelete); // delete now-unused segments
if(useCompoundFile)
merger.createCompoundFile();
return null; return null;
} }
}.run(); }.run();
} }
if (useCompoundFile) {
final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
// make compound file visible for SegmentReaders
directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
// delete now unused files of segment
deleteFiles(filesToDelete);
return null;
}
}.run();
}
}
} }
/** Merges all RAM-resident segments. */ /** Merges all RAM-resident segments. */
@ -479,9 +492,9 @@ public class IndexWriter {
and pushes the merged index onto the top of the segmentInfos stack. */ and pushes the merged index onto the top of the segmentInfos stack. */
private final void mergeSegments(int minSegment) private final void mergeSegments(int minSegment)
throws IOException { throws IOException {
String mergedName = newSegmentName(); final String mergedName = newSegmentName();
if (infoStream != null) infoStream.print("merging segments"); if (infoStream != null) infoStream.print("merging segments");
final SegmentMerger merger = SegmentMerger merger =
new SegmentMerger(directory, mergedName); new SegmentMerger(directory, mergedName);
final Vector segmentsToDelete = new Vector(); final Vector segmentsToDelete = new Vector();
@ -510,23 +523,37 @@ public class IndexWriter {
merger.closeReaders(); merger.closeReaders();
synchronized (directory) { // in- & inter-process sync synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock(IndexWriter.COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) { new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException { public Object doBody() throws IOException {
segmentInfos.write(directory); // commit before deleting segmentInfos.write(directory); // commit before deleting
deleteSegments(segmentsToDelete); // delete now-unused segments deleteSegments(segmentsToDelete); // delete now-unused segments
if(useCompoundFile)
merger.createCompoundFile();
return null; return null;
} }
}.run(); }.run();
} }
if (useCompoundFile) {
final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
// make compound file visible for SegmentReaders
directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
// delete now unused files of segment
deleteFiles(filesToDelete);
return null;
}
}.run();
}
}
} }
/* Some operating systems (e.g. Windows) don't permit a file to be deleted /*
while it is opened for read (e.g. by another process or thread). So we * Some operating systems (e.g. Windows) don't permit a file to be deleted
assume that when a delete fails it is because the file is open in another * while it is opened for read (e.g. by another process or thread). So we
process, and queue the file for subsequent deletion. */ * assume that when a delete fails it is because the file is open in another
* process, and queue the file for subsequent deletion.
*/
private final void deleteSegments(Vector segments) throws IOException { private final void deleteSegments(Vector segments) throws IOException {
Vector deletable = new Vector(); Vector deletable = new Vector();
@ -544,6 +571,13 @@ public class IndexWriter {
writeDeleteableFiles(deletable); // note files we can't delete writeDeleteableFiles(deletable); // note files we can't delete
} }
private final void deleteFiles(Vector files) throws IOException {
Vector deletable = new Vector();
deleteFiles(readDeleteableFiles(), deletable); // try to delete deleteable
deleteFiles(files, deletable); // try to delete our files
writeDeleteableFiles(deletable); // note files we can't delete
}
private final void deleteFiles(Vector files, Directory directory) private final void deleteFiles(Vector files, Directory directory)
throws IOException { throws IOException {
for (int i = 0; i < files.size(); i++) for (int i = 0; i < files.size(); i++)

14
src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -108,13 +108,13 @@ final class SegmentMerger {
} }
} }
final void createCompoundFile() final Vector createCompoundFile(String fileName)
throws IOException { throws IOException {
CompoundFileWriter cfsWriter = CompoundFileWriter cfsWriter =
new CompoundFileWriter(directory, segment + ".cfs"); new CompoundFileWriter(directory, fileName);
ArrayList files = Vector files =
new ArrayList(COMPOUND_EXTENSIONS.length + fieldInfos.size()); new Vector(COMPOUND_EXTENSIONS.length + fieldInfos.size());
// Basic files // Basic files
for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) { for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) {
@ -145,11 +145,7 @@ final class SegmentMerger {
// Perform the merge // Perform the merge
cfsWriter.close(); cfsWriter.close();
// Now delete the source files return files;
it = files.iterator();
while (it.hasNext()) {
directory.deleteFile((String) it.next());
}
} }
/** /**

31
src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -53,7 +53,7 @@ final class SegmentReader extends IndexReader {
InputStream proxStream; InputStream proxStream;
// Compound File Reader when based on a compound file segment // Compound File Reader when based on a compound file segment
CompoundFileReader cfsReader; CompoundFileReader cfsReader = null;
private class Norm { private class Norm {
public Norm(InputStream in, int number) public Norm(InputStream in, int number)
@ -75,7 +75,13 @@ final class SegmentReader extends IndexReader {
} finally { } finally {
out.close(); out.close();
} }
String fileName = segment + ".f" + number; String fileName;
if(cfsReader == null)
fileName = segment + ".f" + number;
else{
// use a different file name if we have compound format
fileName = segment + ".s" + number;
}
directory().renameFile(segment + ".tmp", fileName); directory().renameFile(segment + ".tmp", fileName);
this.dirty = false; this.dirty = false;
} }
@ -216,8 +222,15 @@ final class SegmentReader extends IndexReader {
for (int i = 0; i < fieldInfos.size(); i++) { for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i); FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed) if (fi.isIndexed){
files.addElement(segment + ".f" + i); String name;
if(cfsReader == null)
name = segment + ".f" + i;
else
name = segment + ".s" + i;
if (directory().fileExists(name))
files.addElement(name);
}
} }
return files; return files;
} }
@ -363,9 +376,13 @@ final class SegmentReader extends IndexReader {
for (int i = 0; i < fieldInfos.size(); i++) { for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i); FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed) { if (fi.isIndexed) {
String fileName = segment + ".f" + fi.number; // look first if there are separate norms in compound format
// look first for re-written file, then in compound format String fileName = segment + ".s" + fi.number;
Directory d = directory().fileExists(fileName) ? directory() : cfsDir; Directory d = directory();
if(!d.fileExists(fileName)){
fileName = segment + ".f" + fi.number;
d = cfsDir;
}
norms.put(fi.name, new Norm(d.openFile(fileName), fi.number)); norms.put(fi.name, new Norm(d.openFile(fileName), fi.number));
} }
} }

7
src/test/org/apache/lucene/index/TestDoc.java
View File

@ -188,8 +188,11 @@ public class TestDoc extends TestCase {
merger.merge(); merger.merge();
merger.closeReaders(); merger.closeReaders();
if(useCompoundFile) if (useCompoundFile) {
merger.createCompoundFile(); Vector filesToDelete = merger.createCompoundFile(merged + ".cfs");
for (Iterator iter = filesToDelete.iterator(); iter.hasNext();)
directory.deleteFile((String) iter.next());
}
directory.close(); directory.close();
} }