mirror of https://github.com/apache/lucene.git
building of compound file taken out of commit lock
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150419 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
a715799797
commit
76e2362954
|
@ -399,8 +399,8 @@ public class IndexWriter {
|
||||||
|
|
||||||
optimize(); // start with zero or 1 seg
|
optimize(); // start with zero or 1 seg
|
||||||
|
|
||||||
String mergedName = newSegmentName();
|
final String mergedName = newSegmentName();
|
||||||
final SegmentMerger merger = new SegmentMerger(directory, mergedName);
|
SegmentMerger merger = new SegmentMerger(directory, mergedName);
|
||||||
|
|
||||||
final Vector segmentsToDelete = new Vector();
|
final Vector segmentsToDelete = new Vector();
|
||||||
IndexReader sReader = null;
|
IndexReader sReader = null;
|
||||||
|
@ -422,16 +422,29 @@ public class IndexWriter {
|
||||||
sReader.close();
|
sReader.close();
|
||||||
|
|
||||||
synchronized (directory) { // in- & inter-process sync
|
synchronized (directory) { // in- & inter-process sync
|
||||||
new Lock.With(directory.makeLock("commit.lock"), COMMIT_LOCK_TIMEOUT) {
|
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
|
||||||
public Object doBody() throws IOException {
|
public Object doBody() throws IOException {
|
||||||
segmentInfos.write(directory); // commit changes
|
segmentInfos.write(directory); // commit changes
|
||||||
deleteSegments(segmentsToDelete); // delete now-unused segments
|
deleteSegments(segmentsToDelete); // delete now-unused segments
|
||||||
if(useCompoundFile)
|
|
||||||
merger.createCompoundFile();
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}.run();
|
}.run();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (useCompoundFile) {
|
||||||
|
final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
|
||||||
|
synchronized (directory) { // in- & inter-process sync
|
||||||
|
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
|
||||||
|
public Object doBody() throws IOException {
|
||||||
|
// make compound file visible for SegmentReaders
|
||||||
|
directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
|
||||||
|
// delete now unused files of segment
|
||||||
|
deleteFiles(filesToDelete);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}.run();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Merges all RAM-resident segments. */
|
/** Merges all RAM-resident segments. */
|
||||||
|
@ -479,9 +492,9 @@ public class IndexWriter {
|
||||||
and pushes the merged index onto the top of the segmentInfos stack. */
|
and pushes the merged index onto the top of the segmentInfos stack. */
|
||||||
private final void mergeSegments(int minSegment)
|
private final void mergeSegments(int minSegment)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
String mergedName = newSegmentName();
|
final String mergedName = newSegmentName();
|
||||||
if (infoStream != null) infoStream.print("merging segments");
|
if (infoStream != null) infoStream.print("merging segments");
|
||||||
final SegmentMerger merger =
|
SegmentMerger merger =
|
||||||
new SegmentMerger(directory, mergedName);
|
new SegmentMerger(directory, mergedName);
|
||||||
|
|
||||||
final Vector segmentsToDelete = new Vector();
|
final Vector segmentsToDelete = new Vector();
|
||||||
|
@ -510,23 +523,37 @@ public class IndexWriter {
|
||||||
merger.closeReaders();
|
merger.closeReaders();
|
||||||
|
|
||||||
synchronized (directory) { // in- & inter-process sync
|
synchronized (directory) { // in- & inter-process sync
|
||||||
new Lock.With(directory.makeLock(IndexWriter.COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
|
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
|
||||||
public Object doBody() throws IOException {
|
public Object doBody() throws IOException {
|
||||||
segmentInfos.write(directory); // commit before deleting
|
segmentInfos.write(directory); // commit before deleting
|
||||||
deleteSegments(segmentsToDelete); // delete now-unused segments
|
deleteSegments(segmentsToDelete); // delete now-unused segments
|
||||||
if(useCompoundFile)
|
|
||||||
merger.createCompoundFile();
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}.run();
|
}.run();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (useCompoundFile) {
|
||||||
|
final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
|
||||||
|
synchronized (directory) { // in- & inter-process sync
|
||||||
|
new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
|
||||||
|
public Object doBody() throws IOException {
|
||||||
|
// make compound file visible for SegmentReaders
|
||||||
|
directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
|
||||||
|
// delete now unused files of segment
|
||||||
|
deleteFiles(filesToDelete);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}.run();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Some operating systems (e.g. Windows) don't permit a file to be deleted
|
/*
|
||||||
while it is opened for read (e.g. by another process or thread). So we
|
* Some operating systems (e.g. Windows) don't permit a file to be deleted
|
||||||
assume that when a delete fails it is because the file is open in another
|
* while it is opened for read (e.g. by another process or thread). So we
|
||||||
process, and queue the file for subsequent deletion. */
|
* assume that when a delete fails it is because the file is open in another
|
||||||
|
* process, and queue the file for subsequent deletion.
|
||||||
|
*/
|
||||||
|
|
||||||
private final void deleteSegments(Vector segments) throws IOException {
|
private final void deleteSegments(Vector segments) throws IOException {
|
||||||
Vector deletable = new Vector();
|
Vector deletable = new Vector();
|
||||||
|
@ -543,6 +570,13 @@ public class IndexWriter {
|
||||||
|
|
||||||
writeDeleteableFiles(deletable); // note files we can't delete
|
writeDeleteableFiles(deletable); // note files we can't delete
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private final void deleteFiles(Vector files) throws IOException {
|
||||||
|
Vector deletable = new Vector();
|
||||||
|
deleteFiles(readDeleteableFiles(), deletable); // try to delete deleteable
|
||||||
|
deleteFiles(files, deletable); // try to delete our files
|
||||||
|
writeDeleteableFiles(deletable); // note files we can't delete
|
||||||
|
}
|
||||||
|
|
||||||
private final void deleteFiles(Vector files, Directory directory)
|
private final void deleteFiles(Vector files, Directory directory)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
|
@ -108,13 +108,13 @@ final class SegmentMerger {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
final void createCompoundFile()
|
final Vector createCompoundFile(String fileName)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
CompoundFileWriter cfsWriter =
|
CompoundFileWriter cfsWriter =
|
||||||
new CompoundFileWriter(directory, segment + ".cfs");
|
new CompoundFileWriter(directory, fileName);
|
||||||
|
|
||||||
ArrayList files =
|
Vector files =
|
||||||
new ArrayList(COMPOUND_EXTENSIONS.length + fieldInfos.size());
|
new Vector(COMPOUND_EXTENSIONS.length + fieldInfos.size());
|
||||||
|
|
||||||
// Basic files
|
// Basic files
|
||||||
for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) {
|
for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) {
|
||||||
|
@ -144,12 +144,8 @@ final class SegmentMerger {
|
||||||
|
|
||||||
// Perform the merge
|
// Perform the merge
|
||||||
cfsWriter.close();
|
cfsWriter.close();
|
||||||
|
|
||||||
// Now delete the source files
|
return files;
|
||||||
it = files.iterator();
|
|
||||||
while (it.hasNext()) {
|
|
||||||
directory.deleteFile((String) it.next());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -53,7 +53,7 @@ final class SegmentReader extends IndexReader {
|
||||||
InputStream proxStream;
|
InputStream proxStream;
|
||||||
|
|
||||||
// Compound File Reader when based on a compound file segment
|
// Compound File Reader when based on a compound file segment
|
||||||
CompoundFileReader cfsReader;
|
CompoundFileReader cfsReader = null;
|
||||||
|
|
||||||
private class Norm {
|
private class Norm {
|
||||||
public Norm(InputStream in, int number)
|
public Norm(InputStream in, int number)
|
||||||
|
@ -75,7 +75,13 @@ final class SegmentReader extends IndexReader {
|
||||||
} finally {
|
} finally {
|
||||||
out.close();
|
out.close();
|
||||||
}
|
}
|
||||||
String fileName = segment + ".f" + number;
|
String fileName;
|
||||||
|
if(cfsReader == null)
|
||||||
|
fileName = segment + ".f" + number;
|
||||||
|
else{
|
||||||
|
// use a different file name if we have compound format
|
||||||
|
fileName = segment + ".s" + number;
|
||||||
|
}
|
||||||
directory().renameFile(segment + ".tmp", fileName);
|
directory().renameFile(segment + ".tmp", fileName);
|
||||||
this.dirty = false;
|
this.dirty = false;
|
||||||
}
|
}
|
||||||
|
@ -216,8 +222,15 @@ final class SegmentReader extends IndexReader {
|
||||||
|
|
||||||
for (int i = 0; i < fieldInfos.size(); i++) {
|
for (int i = 0; i < fieldInfos.size(); i++) {
|
||||||
FieldInfo fi = fieldInfos.fieldInfo(i);
|
FieldInfo fi = fieldInfos.fieldInfo(i);
|
||||||
if (fi.isIndexed)
|
if (fi.isIndexed){
|
||||||
files.addElement(segment + ".f" + i);
|
String name;
|
||||||
|
if(cfsReader == null)
|
||||||
|
name = segment + ".f" + i;
|
||||||
|
else
|
||||||
|
name = segment + ".s" + i;
|
||||||
|
if (directory().fileExists(name))
|
||||||
|
files.addElement(name);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return files;
|
return files;
|
||||||
}
|
}
|
||||||
|
@ -363,9 +376,13 @@ final class SegmentReader extends IndexReader {
|
||||||
for (int i = 0; i < fieldInfos.size(); i++) {
|
for (int i = 0; i < fieldInfos.size(); i++) {
|
||||||
FieldInfo fi = fieldInfos.fieldInfo(i);
|
FieldInfo fi = fieldInfos.fieldInfo(i);
|
||||||
if (fi.isIndexed) {
|
if (fi.isIndexed) {
|
||||||
String fileName = segment + ".f" + fi.number;
|
// look first if there are separate norms in compound format
|
||||||
// look first for re-written file, then in compound format
|
String fileName = segment + ".s" + fi.number;
|
||||||
Directory d = directory().fileExists(fileName) ? directory() : cfsDir;
|
Directory d = directory();
|
||||||
|
if(!d.fileExists(fileName)){
|
||||||
|
fileName = segment + ".f" + fi.number;
|
||||||
|
d = cfsDir;
|
||||||
|
}
|
||||||
norms.put(fi.name, new Norm(d.openFile(fileName), fi.number));
|
norms.put(fi.name, new Norm(d.openFile(fileName), fi.number));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -188,8 +188,11 @@ public class TestDoc extends TestCase {
|
||||||
merger.merge();
|
merger.merge();
|
||||||
merger.closeReaders();
|
merger.closeReaders();
|
||||||
|
|
||||||
if(useCompoundFile)
|
if (useCompoundFile) {
|
||||||
merger.createCompoundFile();
|
Vector filesToDelete = merger.createCompoundFile(merged + ".cfs");
|
||||||
|
for (Iterator iter = filesToDelete.iterator(); iter.hasNext();)
|
||||||
|
directory.deleteFile((String) iter.next());
|
||||||
|
}
|
||||||
|
|
||||||
directory.close();
|
directory.close();
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue