diff --git a/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosWriter.java index 9f4b39e96e2..5dd7e10fd38 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/SegmentInfosWriter.java @@ -47,5 +47,5 @@ public abstract class SegmentInfosWriter { * phase commit" operations as described above. * @throws IOException */ - public abstract void write(SegmentInfo info, FieldInfos fis) throws IOException; + public abstract void write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext) throws IOException; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java index 6c2c73a1675..e4a6b158c31 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosFormat.java @@ -74,9 +74,8 @@ public class Lucene3xSegmentInfosFormat extends SegmentInfosFormat { @Override public void files(SegmentInfo info, Set files) { - // nocommit hacky! - if (true || !info.getVersion().startsWith("3.")) { - files.add(IndexFileNames.segmentFileName(info.name, "", SI_EXTENSION)); - } + // nocommit must take care to filter this out if we are + // "really" an old 3.x index + files.add(IndexFileNames.segmentFileName(info.name, "", SI_EXTENSION)); } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java index 1fd5c8bc8a0..172f7184555 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfosReader.java @@ -48,7 +48,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader { infos.counter = input.readInt(); // read counter Lucene3xSegmentInfosReader reader = new Lucene3xSegmentInfosReader(); for (int i = input.readInt(); i > 0; i--) { // read segmentInfos - SegmentInfo si = reader.readSegmentInfo(directory, format, input); + SegmentInfo si = reader.readSegmentInfo(null, directory, format, input); if (si.getVersion() == null) { // Could be a 3.0 - try to open the doc stores - if it fails, it's a // 2.x segment, and an IndexFormatTooOldException will be thrown, @@ -104,7 +104,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader { IndexInput input = directory.openInput(fileName, IOContext.READONCE); try { - SegmentInfo si = readSegmentInfo(directory, format, input); + SegmentInfo si = readSegmentInfo(segmentName, directory, format, input); success = true; return si; } finally { @@ -116,7 +116,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader { } } - private SegmentInfo readSegmentInfo(Directory dir, int format, IndexInput input) throws IOException { + private SegmentInfo readSegmentInfo(String segmentName, Directory dir, int format, IndexInput input) throws IOException { // check that it is a format we can understand if (format > Lucene3xSegmentInfosFormat.FORMAT_MINIMUM) { throw new IndexFormatTooOldException(input, format, @@ -132,7 +132,14 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader { } else { version = null; } + + // NOTE: we ignore this and use the incoming arg + // instead, if it's non-null: final String name = input.readString(); + if (segmentName == null) { + segmentName = name; + } + final int docCount = input.readInt(); final long delGen = input.readLong(); final int docStoreOffset = input.readInt(); @@ -149,7 +156,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader { // pre-4.0 indexes write a byte if there is a single norms file byte b = input.readByte(); - System.out.println("version=" + version + " name=" + name + " docCount=" + docCount + " delGen=" + delGen + " dso=" + docStoreOffset + " dss=" + docStoreSegment + " dssCFs=" + docStoreIsCompoundFile + " b=" + b + " format=" + format); + //System.out.println("version=" + version + " name=" + name + " docCount=" + docCount + " delGen=" + delGen + " dso=" + docStoreOffset + " dss=" + docStoreSegment + " dssCFs=" + docStoreIsCompoundFile + " b=" + b + " format=" + format); assert 1 == b : "expected 1 but was: "+ b + " format: " + format; final int numNormGen = input.readInt(); @@ -178,7 +185,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader { // nocommit we can use hasProx/hasVectors from the 3.x // si... if we can pass this to the other components...? - SegmentInfo info = new SegmentInfo(dir, version, name, docCount, docStoreOffset, + SegmentInfo info = new SegmentInfo(dir, version, segmentName, docCount, docStoreOffset, docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile, delCount, null, diagnostics); info.setDelGen(delGen); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java index 1a85e2302b8..5c66e91f7a3 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java @@ -48,37 +48,20 @@ public class Lucene40SegmentInfosReader extends SegmentInfosReader { try { final String version = input.readString(); final int docCount = input.readInt(); - // this is still written in 4.0 if we open a 3.x and upgrade the SI - final int docStoreOffset = input.readInt(); - final String docStoreSegment; - final boolean docStoreIsCompoundFile; - if (docStoreOffset != -1) { - docStoreSegment = input.readString(); - docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES; - } else { - docStoreSegment = segment; - docStoreIsCompoundFile = false; - } - final int numNormGen = input.readInt(); - final Map normGen; - if (numNormGen == SegmentInfo.NO) { - normGen = null; - } else { - normGen = new HashMap(); - for(int j=0;j normGen = null; final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; - - final int delCount = input.readInt(); - assert delCount <= docCount; final Map diagnostics = input.readStringStringMap(); + final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, docStoreOffset, + docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile, + 0, null, diagnostics); success = true; - return new SegmentInfo(dir, version, segment, docCount, docStoreOffset, - docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile, - delCount, null, diagnostics); + + return si; + } finally { if (!success) { IOUtils.closeWhileHandlingException(input); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java index 2b53368e78c..68552b93f45 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosWriter.java @@ -43,38 +43,21 @@ public class Lucene40SegmentInfosWriter extends SegmentInfosWriter { /** Save a single segment's info. */ @Override - public void write(SegmentInfo si, FieldInfos fis) throws IOException { + public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException { assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name; final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene40SegmentInfosFormat.SI_EXTENSION); - // nocommit what ioctxt to pass? cannot call .sizeInBytes()! - final IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(si.docCount, 0))); + final IndexOutput output = dir.createOutput(fileName, ioContext); boolean success = false; try { // Write the Lucene version that created this segment, since 3.1 output.writeString(si.getVersion()); output.writeInt(si.docCount); - // we still need to write this in 4.0 since we can open a 3.x with shared docStores - output.writeInt(si.getDocStoreOffset()); - if (si.getDocStoreOffset() != -1) { - output.writeString(si.getDocStoreSegment()); - output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0)); - } - // nocommit remove (4.0 doesn't write normGen)... - Map normGen = si.getNormGen(); - if (normGen == null) { - output.writeInt(SegmentInfo.NO); - } else { - output.writeInt(normGen.size()); - for (Entry entry : normGen.entrySet()) { - output.writeInt(entry.getKey()); - output.writeLong(entry.getValue()); - } - } + assert si.getDocStoreOffset() == -1; + assert si.getNormGen() == null; output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO)); - output.writeInt(si.getDelCount()); output.writeStringStringMap(si.getDiagnostics()); success = true; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java index fc9eaecb736..47b8f00b5fc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosReader.java @@ -46,43 +46,6 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfosWriter.* */ public class SimpleTextSegmentInfosReader extends SegmentInfosReader { - /* - final BytesRef scratch = new BytesRef(); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, VERSION); - infos.version = Long.parseLong(readString(VERSION.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, COUNTER); - infos.counter = Integer.parseInt(readString(COUNTER.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, NUM_USERDATA); - int numUserData = Integer.parseInt(readString(NUM_USERDATA.length, scratch)); - infos.userData = new HashMap(); - - for (int i = 0; i < numUserData; i++) { - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, USERDATA_KEY); - String key = readString(USERDATA_KEY.length, scratch); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, USERDATA_VALUE); - String value = readString(USERDATA_VALUE.length, scratch); - infos.userData.put(key, value); - } - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, NUM_SEGMENTS); - int numSegments = Integer.parseInt(readString(NUM_SEGMENTS.length, scratch)); - - for (int i = 0; i < numSegments; i++) { - infos.add(readSegmentInfo(directory, input, scratch)); - } - } - */ - @Override public SegmentInfo read(Directory directory, String segmentName) throws IOException { BytesRef scratch = new BytesRef(); @@ -98,50 +61,10 @@ public class SimpleTextSegmentInfosReader extends SegmentInfosReader { assert StringHelper.startsWith(scratch, SI_DOCCOUNT); final int docCount = Integer.parseInt(readString(SI_DOCCOUNT.length, scratch)); - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_DELCOUNT); - final int delCount = Integer.parseInt(readString(SI_DELCOUNT.length, scratch)); - SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, SI_USECOMPOUND); final boolean isCompoundFile = Boolean.parseBoolean(readString(SI_USECOMPOUND.length, scratch)); - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_DSOFFSET); - final int dsOffset = Integer.parseInt(readString(SI_DSOFFSET.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_DSSEGMENT); - final String dsSegment = readString(SI_DSSEGMENT.length, scratch); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_DSCOMPOUND); - final boolean dsCompoundFile = Boolean.parseBoolean(readString(SI_DSCOMPOUND.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_DELGEN); - final long delGen = Long.parseLong(readString(SI_DELGEN.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_NUM_NORMGEN); - final int numNormGen = Integer.parseInt(readString(SI_NUM_NORMGEN.length, scratch)); - final Map normGen; - if (numNormGen == 0) { - normGen = null; - } else { - normGen = new HashMap(); - for (int i = 0; i < numNormGen; i++) { - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_NORMGEN_KEY); - int key = Integer.parseInt(readString(SI_NORMGEN_KEY.length, scratch)); - - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, SI_NORMGEN_VALUE); - long value = Long.parseLong(readString(SI_NORMGEN_VALUE.length, scratch)); - normGen.put(key, value); - } - } - SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, SI_NUM_DIAG); int numDiag = Integer.parseInt(readString(SI_NUM_DIAG.length, scratch)); @@ -159,9 +82,9 @@ public class SimpleTextSegmentInfosReader extends SegmentInfosReader { } success = true; - return new SegmentInfo(directory, version, segmentName, docCount, dsOffset, - dsSegment, dsCompoundFile, normGen, isCompoundFile, - delCount, null, diagnostics); + return new SegmentInfo(directory, version, segmentName, docCount, -1, + segmentName, false, null, isCompoundFile, + 0, null, diagnostics); } finally { if (!success) { IOUtils.closeWhileHandlingException(input); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosWriter.java index c86389546dc..bfd095e8b53 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfosWriter.java @@ -42,92 +42,20 @@ import org.apache.lucene.util.IOUtils; */ public class SimpleTextSegmentInfosWriter extends SegmentInfosWriter { - final static BytesRef VERSION = new BytesRef("version "); - final static BytesRef COUNTER = new BytesRef("counter "); - final static BytesRef NUM_USERDATA = new BytesRef("user data entries "); - final static BytesRef USERDATA_KEY = new BytesRef(" key "); - final static BytesRef USERDATA_VALUE = new BytesRef(" value "); - final static BytesRef NUM_SEGMENTS = new BytesRef("number of segments "); final static BytesRef SI_VERSION = new BytesRef(" version "); final static BytesRef SI_DOCCOUNT = new BytesRef(" number of documents "); - final static BytesRef SI_DELCOUNT = new BytesRef(" number of deletions "); final static BytesRef SI_USECOMPOUND = new BytesRef(" uses compound file "); - final static BytesRef SI_DSOFFSET = new BytesRef(" docstore offset "); - final static BytesRef SI_DSSEGMENT = new BytesRef(" docstore segment "); - final static BytesRef SI_DSCOMPOUND = new BytesRef(" docstore is compound file "); - final static BytesRef SI_DELGEN = new BytesRef(" deletion generation "); - final static BytesRef SI_NUM_NORMGEN = new BytesRef(" norms generations "); - final static BytesRef SI_NORMGEN_KEY = new BytesRef(" key "); - final static BytesRef SI_NORMGEN_VALUE = new BytesRef(" value "); final static BytesRef SI_NUM_DIAG = new BytesRef(" diagnostics "); final static BytesRef SI_DIAG_KEY = new BytesRef(" key "); final static BytesRef SI_DIAG_VALUE = new BytesRef(" value "); - /* - BytesRef scratch = new BytesRef(); - IndexOutput out = new ChecksumIndexOutput(dir.createOutput(segmentsFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount())))); - boolean success = false; - try { - // required preamble: - out.writeInt(SegmentInfos.FORMAT_CURRENT); // write FORMAT - out.writeString(codecID); // write codecID - // end preamble - - // version - SimpleTextUtil.write(out, VERSION); - SimpleTextUtil.write(out, Long.toString(infos.version), scratch); - SimpleTextUtil.writeNewline(out); - - // counter - SimpleTextUtil.write(out, COUNTER); - SimpleTextUtil.write(out, Integer.toString(infos.counter), scratch); - SimpleTextUtil.writeNewline(out); - - // user data - int numUserDataEntries = infos.getUserData() == null ? 0 : infos.getUserData().size(); - SimpleTextUtil.write(out, NUM_USERDATA); - SimpleTextUtil.write(out, Integer.toString(numUserDataEntries), scratch); - SimpleTextUtil.writeNewline(out); - - if (numUserDataEntries > 0) { - for (Map.Entry userEntry : infos.getUserData().entrySet()) { - SimpleTextUtil.write(out, USERDATA_KEY); - SimpleTextUtil.write(out, userEntry.getKey(), scratch); - SimpleTextUtil.writeNewline(out); - - SimpleTextUtil.write(out, USERDATA_VALUE); - SimpleTextUtil.write(out, userEntry.getValue(), scratch); - SimpleTextUtil.writeNewline(out); - } - } - - // infos size - SimpleTextUtil.write(out, NUM_SEGMENTS); - SimpleTextUtil.write(out, Integer.toString(infos.size()), scratch); - SimpleTextUtil.writeNewline(out); - - for (SegmentInfo si : infos) { - writeInfo(out, si); - } - - success = true; - return out; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(out); - } - } - } - */ - @Override - public void write(SegmentInfo si, FieldInfos fis) throws IOException { + public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException { assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name; String fileName = IndexFileNames.segmentFileName(si.name, "", SimpleTextSegmentInfosFormat.SI_EXTENSION); - // nocommit what IOCtx boolean success = false; - IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(0, 0))); + IndexOutput output = dir.createOutput(fileName, ioContext); try { BytesRef scratch = new BytesRef(); @@ -140,49 +68,10 @@ public class SimpleTextSegmentInfosWriter extends SegmentInfosWriter { SimpleTextUtil.write(output, Integer.toString(si.docCount), scratch); SimpleTextUtil.writeNewline(output); - SimpleTextUtil.write(output, SI_DELCOUNT); - SimpleTextUtil.write(output, Integer.toString(si.getDelCount()), scratch); - SimpleTextUtil.writeNewline(output); - SimpleTextUtil.write(output, SI_USECOMPOUND); SimpleTextUtil.write(output, Boolean.toString(si.getUseCompoundFile()), scratch); SimpleTextUtil.writeNewline(output); - SimpleTextUtil.write(output, SI_DSOFFSET); - SimpleTextUtil.write(output, Integer.toString(si.getDocStoreOffset()), scratch); - SimpleTextUtil.writeNewline(output); - - SimpleTextUtil.write(output, SI_DSSEGMENT); - SimpleTextUtil.write(output, si.getDocStoreSegment(), scratch); - SimpleTextUtil.writeNewline(output); - - SimpleTextUtil.write(output, SI_DSCOMPOUND); - SimpleTextUtil.write(output, Boolean.toString(si.getDocStoreIsCompoundFile()), scratch); - SimpleTextUtil.writeNewline(output); - - SimpleTextUtil.write(output, SI_DELGEN); - SimpleTextUtil.write(output, Long.toString(si.getDelGen()), scratch); - SimpleTextUtil.writeNewline(output); - - Map normGen = si.getNormGen(); - int numNormGen = normGen == null ? 0 : normGen.size(); - SimpleTextUtil.write(output, SI_NUM_NORMGEN); - SimpleTextUtil.write(output, Integer.toString(numNormGen), scratch); - SimpleTextUtil.writeNewline(output); - - // nocommit no more: - if (numNormGen > 0) { - for (Entry entry : normGen.entrySet()) { - SimpleTextUtil.write(output, SI_NORMGEN_KEY); - SimpleTextUtil.write(output, Integer.toString(entry.getKey()), scratch); - SimpleTextUtil.writeNewline(output); - - SimpleTextUtil.write(output, SI_NORMGEN_VALUE); - SimpleTextUtil.write(output, Long.toString(entry.getValue()), scratch); - SimpleTextUtil.writeNewline(output); - } - } - Map diagnostics = si.getDiagnostics(); int numDiagnostics = diagnostics == null ? 0 : diagnostics.size(); SimpleTextUtil.write(output, SI_NUM_DIAG); diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 2606ba08fc2..eedd17a5db5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -478,7 +478,7 @@ class DocumentsWriterPerThread { consumer.flush(flushState); pendingDeletes.terms.clear(); final SegmentInfo newSegment = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, segment, flushState.numDocs, - 0, segment, false, null, false, 0, + -1, segment, false, null, false, 0, flushState.codec, null); diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 7fb9af3a581..28f6bc4a7af 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfosFormat; import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -46,6 +47,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MergeInfo; +import org.apache.lucene.store.TrackingDirectoryWrapper; import org.apache.lucene.util.Bits; import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; @@ -2052,7 +2054,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: - codec.segmentInfosFormat().getSegmentInfosWriter().write(newSegment, flushedSegment.fieldInfos); + codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, newSegment, flushedSegment.fieldInfos, context); newSegment.clearFilesCache(); // nocommit ideally we would freeze merge.info here!! @@ -2338,7 +2340,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: - codec.segmentInfosFormat().getSegmentInfosWriter().write(info, mergeState.fieldInfos); + codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, info, mergeState.fieldInfos, context); info.clearFilesCache(); // Register the new segment @@ -2376,23 +2378,40 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } Set codecDocStoreFiles = new HashSet(); - if (info.getDocStoreOffset() != -1) { - // only violate the codec this way if its preflex + final boolean hasSharedDocStore = info.getDocStoreOffset() != -1; + final String segmentInfoFileName3X = IndexFileNames.segmentFileName(info.name, + "", + Lucene3xSegmentInfosFormat.SI_EXTENSION); + + if (hasSharedDocStore) { + // only violate the codec this way if its preflex & + // shares doc stores info.getCodec().storedFieldsFormat().files(info, codecDocStoreFiles); info.getCodec().termVectorsFormat().files(info, codecDocStoreFiles); } //System.out.println("copy seg=" + info.name + " version=" + info.getVersion()); - - // Copy the segment files - for (String file: info.files()) { - // nocommit messy: insteda we should pull .files() - // from the codec's SIFormat and check if it's in - // there... - if (file.endsWith(".si")) { - continue; - } + // Same SI as before but we change directory, name and docStoreSegment: + SegmentInfo newInfo = new SegmentInfo(directory, info.getVersion(), segName, info.docCount, info.getDocStoreOffset(), + newDsName, info.getDocStoreIsCompoundFile(), info.getNormGen(), info.getUseCompoundFile(), + info.getDelCount(), info.getCodec(), info.getDiagnostics()); + newInfo.setDelGen(info.getDelGen()); + + // We must rewrite the SI file because it references + // segment name (its own name, if its 3.x, and doc + // store segment name): + TrackingDirectoryWrapper dirWrapper = new TrackingDirectoryWrapper(directory); + try { + newInfo.getCodec().segmentInfosFormat().getSegmentInfosWriter().write(dirWrapper, newInfo, null, context); + } catch (UnsupportedOperationException uoe) { + // OK: 3x codec cannot write a new SI file; + // SegmentInfos will write this on commit + } + final Set siFileNames = dirWrapper.getCreatedFiles(); + + // Copy the segment's files + for (String file: info.files()) { final String newFileName; if (codecDocStoreFiles.contains(file)) { @@ -2405,28 +2424,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { newFileName = segName + IndexFileNames.stripSegmentName(file); } + if (siFileNames != null && siFileNames.contains(newFileName)) { + // We already rewwrote this above + continue; + } + assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists"; assert !copiedFiles.contains(file): "file \"" + file + "\" is being copied more than once"; copiedFiles.add(file); - //System.out.println("COPY " + file + " -> " + newFileName); info.dir.copy(directory, file, newFileName, context); } - // Same SI as before but we change directory and name: - SegmentInfo newInfo = new SegmentInfo(directory, info.getVersion(), segName, info.docCount, info.getDocStoreOffset(), - newDsName, info.getDocStoreIsCompoundFile(), info.getNormGen(), info.getUseCompoundFile(), - info.getDelCount(), info.getCodec(), info.getDiagnostics()); - newInfo.setDelGen(info.getDelGen()); - - // nocommit need to pass real FIS... - // nocommit maybe we don't pass FIS......? - // nocommit messy.... - //if (!newInfo.getCodec().getName().equals("Lucene3x")) { - if (!newInfo.getVersion().startsWith("3.")) { - //System.out.println(" now write si for seg=" + newInfo.name + " codec=" + newInfo.getCodec()); - newInfo.getCodec().segmentInfosFormat().getSegmentInfosWriter().write(newInfo, null); - } - return newInfo; } @@ -3511,7 +3519,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { synchronized (this) { // Guard segmentInfos useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, merge.info); } - + if (useCompoundFile) { success = false; final String compoundFileName = IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION); @@ -3520,7 +3528,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "create compound file " + compoundFileName); } - createCompoundFile(directory, compoundFileName, checkAbort, merge.info, new IOContext(merge.getMergeInfo())); + createCompoundFile(directory, compoundFileName, checkAbort, merge.info, context); success = true; } catch (IOException ioe) { synchronized(this) { @@ -3575,7 +3583,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // creating CFS so that 1) .si isn't slurped into CFS, // and 2) .si reflects useCompoundFile=true change // above: - codec.segmentInfosFormat().getSegmentInfosWriter().write(merge.info, mergeState.fieldInfos); + codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, merge.info, mergeState.fieldInfos, context); merge.info.clearFilesCache(); // nocommit ideally we would freeze merge.info here!! diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java index 2ab6ab0b3f4..0fb4c142f04 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java @@ -284,6 +284,7 @@ public final class SegmentInfos implements Cloneable, Iterable { info.setCodec(codec); info.setDelGen(input.readLong()); info.setDelCount(input.readInt()); + assert info.getDelCount() <= info.docCount; add(info); } userData = input.readStringStringMap(); @@ -297,9 +298,6 @@ public final class SegmentInfos implements Cloneable, Iterable { } } - // nocommit all 3.x indices have checksum right...???? - // ie we added it during 2.x? i think so! - final long checksumNow = input.getChecksum(); final long checksumThen = input.readLong(); if (checksumNow != checksumThen) { @@ -376,8 +374,6 @@ public final class SegmentInfos implements Cloneable, Iterable { if (!directory.fileExists(fileName)) { //System.out.println("write 3x info seg=" + si.name + " version=" + si.getVersion() + " codec=" + si.getCodec().getName()); write3xInfo(si); - // nocommit do this after, on success... - //si.setVersion("4.0"); si.clearFilesCache(); } } diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java index f3668c09eba..420c7eae920 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java @@ -153,7 +153,7 @@ public abstract class Directory implements Closeable { * their own locking implementation. */ public LockFactory getLockFactory() { - return this.lockFactory; + return this.lockFactory; } /** diff --git a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java new file mode 100644 index 00000000000..3be6b3df8da --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java @@ -0,0 +1,129 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** A delegating Directory that records which files were + * written to and deleted. */ +public final class TrackingDirectoryWrapper extends Directory implements Closeable { + + private final Directory other; + private final Set createdFileNames = Collections.synchronizedSet(new HashSet()); + + public TrackingDirectoryWrapper(Directory other) { + this.other = other; + } + + @Override + public String[] listAll() throws IOException { + return other.listAll(); + } + + @Override + public boolean fileExists(String name) throws IOException { + return other.fileExists(name); + } + + @Override + public void deleteFile(String name) throws IOException { + createdFileNames.remove(name); + other.deleteFile(name); + } + + @Override + public long fileLength(String name) throws IOException { + return other.fileLength(name); + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + createdFileNames.add(name); + return other.createOutput(name, context); + } + + @Override + public void sync(Collection names) throws IOException { + other.sync(names); + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + return other.openInput(name, context); + } + + @Override + public Lock makeLock(String name) { + return other.makeLock(name); + } + + @Override + public void clearLock(String name) throws IOException { + other.clearLock(name); + } + + @Override + public void close() throws IOException { + other.close(); + } + + @Override + public void setLockFactory(LockFactory lockFactory) throws IOException { + other.setLockFactory(lockFactory); + } + + @Override + public LockFactory getLockFactory() { + return other.getLockFactory(); + } + + @Override + public String getLockID() { + return other.getLockID(); + } + + @Override + public String toString() { + return "TrackingDirectoryWrapper(" + other.toString() + ")"; + } + + @Override + public void copy(Directory to, String src, String dest, IOContext context) throws IOException { + // nocommit add dest to created files!? + other.copy(to, src, dest, context); + } + + @Override + public Directory.IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException { + return other.createSlicer(name, context); + } + + // nocommit no? + //@Override + //protected final void ensureOpen() throws + //AlreadyClosedException { + + public Set getCreatedFiles() { + return createdFileNames; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java index 06548f43b29..1f54d794dab 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java @@ -106,9 +106,6 @@ public class TestParallelReaderEmptyIndex extends LuceneTestCase { iw.addDocument(doc); iw.close(); - // nocommit - _TestUtil.checkIndex(rd1); - IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setMergePolicy(NoMergePolicy.COMPOUND_FILES); if (VERBOSE) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java index 6315413c34a..632b29acff8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWSegmentInfosWriter.java @@ -39,17 +39,20 @@ import org.apache.lucene.util.IOUtils; */ class PreFlexRWSegmentInfosWriter extends SegmentInfosWriter { + // NOTE: this is not "really" 3.x format, because we are + // writing each SI to its own file, vs 3.x where the list + // of segments and SI for each segment is written into a + // single segments_N file + /** Save a single segment's info. */ @Override - public void write(SegmentInfo si, FieldInfos fis) throws IOException { + public void write(Directory dir ,SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException { - // NOTE: this is NOT how 3.x is really written... String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfosFormat.SI_EXTENSION); - // nocommit what IOCtx boolean success = false; - IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(0, 0))); + IndexOutput output = dir.createOutput(fileName, ioContext); try { // we are about to write this SI in 3.x format, dropping all codec information, etc. // so it had better be a 3.x segment or you will get very confusing errors later.