Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

building of compound file taken out of commit lock 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150419 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Christoph Goller 2004-08-11 17:38:35 +00:00
parent a715799797
commit 76e2362954
4 changed files with 84 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -399,8 +399,8 @@ public class IndexWriter {
optimize(); // start with zero or 1 seg
String mergedName = newSegmentName();
final SegmentMerger merger = new SegmentMerger(directory, mergedName);
final String mergedName = newSegmentName();
SegmentMerger merger = new SegmentMerger(directory, mergedName);
final Vector segmentsToDelete = new Vector();
IndexReader sReader = null;
@ -422,16 +422,29 @@ public class IndexWriter {
sReader.close();
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock("commit.lock"), COMMIT_LOCK_TIMEOUT) {
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
segmentInfos.write(directory); // commit changes
deleteSegments(segmentsToDelete); // delete now-unused segments
if(useCompoundFile)
merger.createCompoundFile();
return null;
}
}.run();
}
if (useCompoundFile) {
final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
// make compound file visible for SegmentReaders
directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
// delete now unused files of segment
deleteFiles(filesToDelete);
return null;
}
}.run();
}
}
}
/** Merges all RAM-resident segments. */
@ -479,9 +492,9 @@ public class IndexWriter {
and pushes the merged index onto the top of the segmentInfos stack. */
private final void mergeSegments(int minSegment)
throws IOException {
String mergedName = newSegmentName();
final String mergedName = newSegmentName();
if (infoStream != null) infoStream.print("merging segments");
final SegmentMerger merger =
SegmentMerger merger =
new SegmentMerger(directory, mergedName);
final Vector segmentsToDelete = new Vector();
@ -510,23 +523,37 @@ public class IndexWriter {
merger.closeReaders();
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock(IndexWriter.COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
segmentInfos.write(directory); // commit before deleting
deleteSegments(segmentsToDelete); // delete now-unused segments
if(useCompoundFile)
merger.createCompoundFile();
return null;
}
}.run();
}
if (useCompoundFile) {
final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
// make compound file visible for SegmentReaders
directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
// delete now unused files of segment
deleteFiles(filesToDelete);
return null;
}
}.run();
}
}
}
/* Some operating systems (e.g. Windows) don't permit a file to be deleted
while it is opened for read (e.g. by another process or thread). So we
assume that when a delete fails it is because the file is open in another
process, and queue the file for subsequent deletion. */
/*
* Some operating systems (e.g. Windows) don't permit a file to be deleted
* while it is opened for read (e.g. by another process or thread). So we
* assume that when a delete fails it is because the file is open in another
* process, and queue the file for subsequent deletion.
*/
private final void deleteSegments(Vector segments) throws IOException {
Vector deletable = new Vector();
@ -544,6 +571,13 @@ public class IndexWriter {
writeDeleteableFiles(deletable); // note files we can't delete
}
private final void deleteFiles(Vector files) throws IOException {
Vector deletable = new Vector();
deleteFiles(readDeleteableFiles(), deletable); // try to delete deleteable
deleteFiles(files, deletable); // try to delete our files
writeDeleteableFiles(deletable); // note files we can't delete
}
private final void deleteFiles(Vector files, Directory directory)
throws IOException {
for (int i = 0; i < files.size(); i++)

14
src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -108,13 +108,13 @@ final class SegmentMerger {
}
}
final void createCompoundFile()
final Vector createCompoundFile(String fileName)
throws IOException {
CompoundFileWriter cfsWriter =
new CompoundFileWriter(directory, segment + ".cfs");
new CompoundFileWriter(directory, fileName);
ArrayList files =
new ArrayList(COMPOUND_EXTENSIONS.length + fieldInfos.size());
Vector files =
new Vector(COMPOUND_EXTENSIONS.length + fieldInfos.size());
// Basic files
for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) {
@ -145,11 +145,7 @@ final class SegmentMerger {
// Perform the merge
cfsWriter.close();
// Now delete the source files
it = files.iterator();
while (it.hasNext()) {
directory.deleteFile((String) it.next());
}
return files;
}
/**

31
src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -53,7 +53,7 @@ final class SegmentReader extends IndexReader {
InputStream proxStream;
// Compound File Reader when based on a compound file segment
CompoundFileReader cfsReader;
CompoundFileReader cfsReader = null;
private class Norm {
public Norm(InputStream in, int number)
@ -75,7 +75,13 @@ final class SegmentReader extends IndexReader {
} finally {
out.close();
}
String fileName = segment + ".f" + number;
String fileName;
if(cfsReader == null)
fileName = segment + ".f" + number;
else{
// use a different file name if we have compound format
fileName = segment + ".s" + number;
}
directory().renameFile(segment + ".tmp", fileName);
this.dirty = false;
}
@ -216,8 +222,15 @@ final class SegmentReader extends IndexReader {
for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed)
files.addElement(segment + ".f" + i);
if (fi.isIndexed){
String name;
if(cfsReader == null)
name = segment + ".f" + i;
else
name = segment + ".s" + i;
if (directory().fileExists(name))
files.addElement(name);
}
}
return files;
}
@ -363,9 +376,13 @@ final class SegmentReader extends IndexReader {
for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed) {
String fileName = segment + ".f" + fi.number;
// look first for re-written file, then in compound format
Directory d = directory().fileExists(fileName) ? directory() : cfsDir;
// look first if there are separate norms in compound format
String fileName = segment + ".s" + fi.number;
Directory d = directory();
if(!d.fileExists(fileName)){
fileName = segment + ".f" + fi.number;
d = cfsDir;
}
norms.put(fi.name, new Norm(d.openFile(fileName), fi.number));
}
}

7
src/test/org/apache/lucene/index/TestDoc.java
View File

@ -188,8 +188,11 @@ public class TestDoc extends TestCase {
merger.merge();
merger.closeReaders();
if(useCompoundFile)
merger.createCompoundFile();
if (useCompoundFile) {
Vector filesToDelete = merger.createCompoundFile(merged + ".cfs");
for (Iterator iter = filesToDelete.iterator(); iter.hasNext();)
directory.deleteFile((String) iter.next());
}
directory.close();
}