From 76e2362954915e374eb9613f94c3e41eaf2f2b05 Mon Sep 17 00:00:00 2001 From: Christoph Goller Date: Wed, 11 Aug 2004 17:38:35 +0000 Subject: [PATCH] building of compound file taken out of commit lock git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150419 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/index/IndexWriter.java | 64 ++++++++++++++----- .../apache/lucene/index/SegmentMerger.java | 16 ++--- .../apache/lucene/index/SegmentReader.java | 31 +++++++-- src/test/org/apache/lucene/index/TestDoc.java | 7 +- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/java/org/apache/lucene/index/IndexWriter.java b/src/java/org/apache/lucene/index/IndexWriter.java index 23e7d1b9b6e..f8cd26a7751 100644 --- a/src/java/org/apache/lucene/index/IndexWriter.java +++ b/src/java/org/apache/lucene/index/IndexWriter.java @@ -399,8 +399,8 @@ public class IndexWriter { optimize(); // start with zero or 1 seg - String mergedName = newSegmentName(); - final SegmentMerger merger = new SegmentMerger(directory, mergedName); + final String mergedName = newSegmentName(); + SegmentMerger merger = new SegmentMerger(directory, mergedName); final Vector segmentsToDelete = new Vector(); IndexReader sReader = null; @@ -422,16 +422,29 @@ public class IndexWriter { sReader.close(); synchronized (directory) { // in- & inter-process sync - new Lock.With(directory.makeLock("commit.lock"), COMMIT_LOCK_TIMEOUT) { + new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) { public Object doBody() throws IOException { segmentInfos.write(directory); // commit changes deleteSegments(segmentsToDelete); // delete now-unused segments - if(useCompoundFile) - merger.createCompoundFile(); return null; } }.run(); } + + if (useCompoundFile) { + final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp"); + synchronized (directory) { // in- & inter-process sync + new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) { + public Object doBody() throws IOException { + // make compound file visible for SegmentReaders + directory.renameFile(mergedName + ".tmp", mergedName + ".cfs"); + // delete now unused files of segment + deleteFiles(filesToDelete); + return null; + } + }.run(); + } + } } /** Merges all RAM-resident segments. */ @@ -479,9 +492,9 @@ public class IndexWriter { and pushes the merged index onto the top of the segmentInfos stack. */ private final void mergeSegments(int minSegment) throws IOException { - String mergedName = newSegmentName(); + final String mergedName = newSegmentName(); if (infoStream != null) infoStream.print("merging segments"); - final SegmentMerger merger = + SegmentMerger merger = new SegmentMerger(directory, mergedName); final Vector segmentsToDelete = new Vector(); @@ -510,23 +523,37 @@ public class IndexWriter { merger.closeReaders(); synchronized (directory) { // in- & inter-process sync - new Lock.With(directory.makeLock(IndexWriter.COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) { + new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) { public Object doBody() throws IOException { segmentInfos.write(directory); // commit before deleting deleteSegments(segmentsToDelete); // delete now-unused segments - if(useCompoundFile) - merger.createCompoundFile(); return null; } }.run(); } - + + if (useCompoundFile) { + final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp"); + synchronized (directory) { // in- & inter-process sync + new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) { + public Object doBody() throws IOException { + // make compound file visible for SegmentReaders + directory.renameFile(mergedName + ".tmp", mergedName + ".cfs"); + // delete now unused files of segment + deleteFiles(filesToDelete); + return null; + } + }.run(); + } + } } - /* Some operating systems (e.g. Windows) don't permit a file to be deleted - while it is opened for read (e.g. by another process or thread). So we - assume that when a delete fails it is because the file is open in another - process, and queue the file for subsequent deletion. */ + /* + * Some operating systems (e.g. Windows) don't permit a file to be deleted + * while it is opened for read (e.g. by another process or thread). So we + * assume that when a delete fails it is because the file is open in another + * process, and queue the file for subsequent deletion. + */ private final void deleteSegments(Vector segments) throws IOException { Vector deletable = new Vector(); @@ -543,6 +570,13 @@ public class IndexWriter { writeDeleteableFiles(deletable); // note files we can't delete } + + private final void deleteFiles(Vector files) throws IOException { + Vector deletable = new Vector(); + deleteFiles(readDeleteableFiles(), deletable); // try to delete deleteable + deleteFiles(files, deletable); // try to delete our files + writeDeleteableFiles(deletable); // note files we can't delete + } private final void deleteFiles(Vector files, Directory directory) throws IOException { diff --git a/src/java/org/apache/lucene/index/SegmentMerger.java b/src/java/org/apache/lucene/index/SegmentMerger.java index 565ec8c9fa0..2b81b802a8e 100644 --- a/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/src/java/org/apache/lucene/index/SegmentMerger.java @@ -108,13 +108,13 @@ final class SegmentMerger { } } - final void createCompoundFile() + final Vector createCompoundFile(String fileName) throws IOException { CompoundFileWriter cfsWriter = - new CompoundFileWriter(directory, segment + ".cfs"); + new CompoundFileWriter(directory, fileName); - ArrayList files = - new ArrayList(COMPOUND_EXTENSIONS.length + fieldInfos.size()); + Vector files = + new Vector(COMPOUND_EXTENSIONS.length + fieldInfos.size()); // Basic files for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) { @@ -144,12 +144,8 @@ final class SegmentMerger { // Perform the merge cfsWriter.close(); - - // Now delete the source files - it = files.iterator(); - while (it.hasNext()) { - directory.deleteFile((String) it.next()); - } + + return files; } /** diff --git a/src/java/org/apache/lucene/index/SegmentReader.java b/src/java/org/apache/lucene/index/SegmentReader.java index cae871cbd76..b98a5989f39 100644 --- a/src/java/org/apache/lucene/index/SegmentReader.java +++ b/src/java/org/apache/lucene/index/SegmentReader.java @@ -53,7 +53,7 @@ final class SegmentReader extends IndexReader { InputStream proxStream; // Compound File Reader when based on a compound file segment - CompoundFileReader cfsReader; + CompoundFileReader cfsReader = null; private class Norm { public Norm(InputStream in, int number) @@ -75,7 +75,13 @@ final class SegmentReader extends IndexReader { } finally { out.close(); } - String fileName = segment + ".f" + number; + String fileName; + if(cfsReader == null) + fileName = segment + ".f" + number; + else{ + // use a different file name if we have compound format + fileName = segment + ".s" + number; + } directory().renameFile(segment + ".tmp", fileName); this.dirty = false; } @@ -216,8 +222,15 @@ final class SegmentReader extends IndexReader { for (int i = 0; i < fieldInfos.size(); i++) { FieldInfo fi = fieldInfos.fieldInfo(i); - if (fi.isIndexed) - files.addElement(segment + ".f" + i); + if (fi.isIndexed){ + String name; + if(cfsReader == null) + name = segment + ".f" + i; + else + name = segment + ".s" + i; + if (directory().fileExists(name)) + files.addElement(name); + } } return files; } @@ -363,9 +376,13 @@ final class SegmentReader extends IndexReader { for (int i = 0; i < fieldInfos.size(); i++) { FieldInfo fi = fieldInfos.fieldInfo(i); if (fi.isIndexed) { - String fileName = segment + ".f" + fi.number; - // look first for re-written file, then in compound format - Directory d = directory().fileExists(fileName) ? directory() : cfsDir; + // look first if there are separate norms in compound format + String fileName = segment + ".s" + fi.number; + Directory d = directory(); + if(!d.fileExists(fileName)){ + fileName = segment + ".f" + fi.number; + d = cfsDir; + } norms.put(fi.name, new Norm(d.openFile(fileName), fi.number)); } } diff --git a/src/test/org/apache/lucene/index/TestDoc.java b/src/test/org/apache/lucene/index/TestDoc.java index c8973e41107..9e4a177b9a6 100644 --- a/src/test/org/apache/lucene/index/TestDoc.java +++ b/src/test/org/apache/lucene/index/TestDoc.java @@ -188,8 +188,11 @@ public class TestDoc extends TestCase { merger.merge(); merger.closeReaders(); - if(useCompoundFile) - merger.createCompoundFile(); + if (useCompoundFile) { + Vector filesToDelete = merger.createCompoundFile(merged + ".cfs"); + for (Iterator iter = filesToDelete.iterator(); iter.hasNext();) + directory.deleteFile((String) iter.next()); + } directory.close(); }