mirror of https://github.com/apache/lucene.git
LUCENE-4055: SimpleText and Lucene40 SI impls no longer write 3x-only stuff (shared docs stores, norm gens); clean up some nocommits
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4055@1340691 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless
parent
b83dde502b
commit
5687147186
|
|
@ -47,5 +47,5 @@ public abstract class SegmentInfosWriter {
|
|||
* phase commit" operations as described above.
|
||||
* @throws IOException
|
||||
*/
|
||||
public abstract void write(SegmentInfo info, FieldInfos fis) throws IOException;
|
||||
public abstract void write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext) throws IOException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,9 +74,8 @@ public class Lucene3xSegmentInfosFormat extends SegmentInfosFormat {
|
|||
|
||||
@Override
|
||||
public void files(SegmentInfo info, Set<String> files) {
|
||||
// nocommit hacky!
|
||||
if (true || !info.getVersion().startsWith("3.")) {
|
||||
files.add(IndexFileNames.segmentFileName(info.name, "", SI_EXTENSION));
|
||||
}
|
||||
// nocommit must take care to filter this out if we are
|
||||
// "really" an old 3.x index
|
||||
files.add(IndexFileNames.segmentFileName(info.name, "", SI_EXTENSION));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader {
|
|||
infos.counter = input.readInt(); // read counter
|
||||
Lucene3xSegmentInfosReader reader = new Lucene3xSegmentInfosReader();
|
||||
for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
|
||||
SegmentInfo si = reader.readSegmentInfo(directory, format, input);
|
||||
SegmentInfo si = reader.readSegmentInfo(null, directory, format, input);
|
||||
if (si.getVersion() == null) {
|
||||
// Could be a 3.0 - try to open the doc stores - if it fails, it's a
|
||||
// 2.x segment, and an IndexFormatTooOldException will be thrown,
|
||||
|
|
@ -104,7 +104,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader {
|
|||
IndexInput input = directory.openInput(fileName, IOContext.READONCE);
|
||||
|
||||
try {
|
||||
SegmentInfo si = readSegmentInfo(directory, format, input);
|
||||
SegmentInfo si = readSegmentInfo(segmentName, directory, format, input);
|
||||
success = true;
|
||||
return si;
|
||||
} finally {
|
||||
|
|
@ -116,7 +116,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader {
|
|||
}
|
||||
}
|
||||
|
||||
private SegmentInfo readSegmentInfo(Directory dir, int format, IndexInput input) throws IOException {
|
||||
private SegmentInfo readSegmentInfo(String segmentName, Directory dir, int format, IndexInput input) throws IOException {
|
||||
// check that it is a format we can understand
|
||||
if (format > Lucene3xSegmentInfosFormat.FORMAT_MINIMUM) {
|
||||
throw new IndexFormatTooOldException(input, format,
|
||||
|
|
@ -132,7 +132,14 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader {
|
|||
} else {
|
||||
version = null;
|
||||
}
|
||||
|
||||
// NOTE: we ignore this and use the incoming arg
|
||||
// instead, if it's non-null:
|
||||
final String name = input.readString();
|
||||
if (segmentName == null) {
|
||||
segmentName = name;
|
||||
}
|
||||
|
||||
final int docCount = input.readInt();
|
||||
final long delGen = input.readLong();
|
||||
final int docStoreOffset = input.readInt();
|
||||
|
|
@ -149,7 +156,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader {
|
|||
// pre-4.0 indexes write a byte if there is a single norms file
|
||||
byte b = input.readByte();
|
||||
|
||||
System.out.println("version=" + version + " name=" + name + " docCount=" + docCount + " delGen=" + delGen + " dso=" + docStoreOffset + " dss=" + docStoreSegment + " dssCFs=" + docStoreIsCompoundFile + " b=" + b + " format=" + format);
|
||||
//System.out.println("version=" + version + " name=" + name + " docCount=" + docCount + " delGen=" + delGen + " dso=" + docStoreOffset + " dss=" + docStoreSegment + " dssCFs=" + docStoreIsCompoundFile + " b=" + b + " format=" + format);
|
||||
|
||||
assert 1 == b : "expected 1 but was: "+ b + " format: " + format;
|
||||
final int numNormGen = input.readInt();
|
||||
|
|
@ -178,7 +185,7 @@ public class Lucene3xSegmentInfosReader extends SegmentInfosReader {
|
|||
// nocommit we can use hasProx/hasVectors from the 3.x
|
||||
// si... if we can pass this to the other components...?
|
||||
|
||||
SegmentInfo info = new SegmentInfo(dir, version, name, docCount, docStoreOffset,
|
||||
SegmentInfo info = new SegmentInfo(dir, version, segmentName, docCount, docStoreOffset,
|
||||
docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
|
||||
delCount, null, diagnostics);
|
||||
info.setDelGen(delGen);
|
||||
|
|
|
|||
|
|
@ -48,37 +48,20 @@ public class Lucene40SegmentInfosReader extends SegmentInfosReader {
|
|||
try {
|
||||
final String version = input.readString();
|
||||
final int docCount = input.readInt();
|
||||
// this is still written in 4.0 if we open a 3.x and upgrade the SI
|
||||
final int docStoreOffset = input.readInt();
|
||||
final String docStoreSegment;
|
||||
final boolean docStoreIsCompoundFile;
|
||||
if (docStoreOffset != -1) {
|
||||
docStoreSegment = input.readString();
|
||||
docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
|
||||
} else {
|
||||
docStoreSegment = segment;
|
||||
docStoreIsCompoundFile = false;
|
||||
}
|
||||
final int numNormGen = input.readInt();
|
||||
final Map<Integer,Long> normGen;
|
||||
if (numNormGen == SegmentInfo.NO) {
|
||||
normGen = null;
|
||||
} else {
|
||||
normGen = new HashMap<Integer, Long>();
|
||||
for(int j=0;j<numNormGen;j++) {
|
||||
normGen.put(input.readInt(), input.readLong());
|
||||
}
|
||||
}
|
||||
final int docStoreOffset = -1;
|
||||
final String docStoreSegment = segment;
|
||||
final boolean docStoreIsCompoundFile = false;
|
||||
final Map<Integer,Long> normGen = null;
|
||||
final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
|
||||
|
||||
final int delCount = input.readInt();
|
||||
assert delCount <= docCount;
|
||||
final Map<String,String> diagnostics = input.readStringStringMap();
|
||||
|
||||
final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, docStoreOffset,
|
||||
docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
|
||||
0, null, diagnostics);
|
||||
success = true;
|
||||
return new SegmentInfo(dir, version, segment, docCount, docStoreOffset,
|
||||
docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
|
||||
delCount, null, diagnostics);
|
||||
|
||||
return si;
|
||||
|
||||
} finally {
|
||||
if (!success) {
|
||||
IOUtils.closeWhileHandlingException(input);
|
||||
|
|
|
|||
|
|
@ -43,38 +43,21 @@ public class Lucene40SegmentInfosWriter extends SegmentInfosWriter {
|
|||
|
||||
/** Save a single segment's info. */
|
||||
@Override
|
||||
public void write(SegmentInfo si, FieldInfos fis) throws IOException {
|
||||
public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
|
||||
assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name;
|
||||
final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene40SegmentInfosFormat.SI_EXTENSION);
|
||||
// nocommit what ioctxt to pass? cannot call .sizeInBytes()!
|
||||
final IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(si.docCount, 0)));
|
||||
final IndexOutput output = dir.createOutput(fileName, ioContext);
|
||||
|
||||
boolean success = false;
|
||||
try {
|
||||
// Write the Lucene version that created this segment, since 3.1
|
||||
output.writeString(si.getVersion());
|
||||
output.writeInt(si.docCount);
|
||||
// we still need to write this in 4.0 since we can open a 3.x with shared docStores
|
||||
output.writeInt(si.getDocStoreOffset());
|
||||
if (si.getDocStoreOffset() != -1) {
|
||||
output.writeString(si.getDocStoreSegment());
|
||||
output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0));
|
||||
}
|
||||
|
||||
// nocommit remove (4.0 doesn't write normGen)...
|
||||
Map<Integer,Long> normGen = si.getNormGen();
|
||||
if (normGen == null) {
|
||||
output.writeInt(SegmentInfo.NO);
|
||||
} else {
|
||||
output.writeInt(normGen.size());
|
||||
for (Entry<Integer,Long> entry : normGen.entrySet()) {
|
||||
output.writeInt(entry.getKey());
|
||||
output.writeLong(entry.getValue());
|
||||
}
|
||||
}
|
||||
assert si.getDocStoreOffset() == -1;
|
||||
assert si.getNormGen() == null;
|
||||
|
||||
output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
|
||||
output.writeInt(si.getDelCount());
|
||||
output.writeStringStringMap(si.getDiagnostics());
|
||||
|
||||
success = true;
|
||||
|
|
|
|||
|
|
@ -46,43 +46,6 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfosWriter.*
|
|||
*/
|
||||
public class SimpleTextSegmentInfosReader extends SegmentInfosReader {
|
||||
|
||||
/*
|
||||
final BytesRef scratch = new BytesRef();
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, VERSION);
|
||||
infos.version = Long.parseLong(readString(VERSION.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, COUNTER);
|
||||
infos.counter = Integer.parseInt(readString(COUNTER.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, NUM_USERDATA);
|
||||
int numUserData = Integer.parseInt(readString(NUM_USERDATA.length, scratch));
|
||||
infos.userData = new HashMap<String,String>();
|
||||
|
||||
for (int i = 0; i < numUserData; i++) {
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, USERDATA_KEY);
|
||||
String key = readString(USERDATA_KEY.length, scratch);
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, USERDATA_VALUE);
|
||||
String value = readString(USERDATA_VALUE.length, scratch);
|
||||
infos.userData.put(key, value);
|
||||
}
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, NUM_SEGMENTS);
|
||||
int numSegments = Integer.parseInt(readString(NUM_SEGMENTS.length, scratch));
|
||||
|
||||
for (int i = 0; i < numSegments; i++) {
|
||||
infos.add(readSegmentInfo(directory, input, scratch));
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
@Override
|
||||
public SegmentInfo read(Directory directory, String segmentName) throws IOException {
|
||||
BytesRef scratch = new BytesRef();
|
||||
|
|
@ -98,50 +61,10 @@ public class SimpleTextSegmentInfosReader extends SegmentInfosReader {
|
|||
assert StringHelper.startsWith(scratch, SI_DOCCOUNT);
|
||||
final int docCount = Integer.parseInt(readString(SI_DOCCOUNT.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_DELCOUNT);
|
||||
final int delCount = Integer.parseInt(readString(SI_DELCOUNT.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_USECOMPOUND);
|
||||
final boolean isCompoundFile = Boolean.parseBoolean(readString(SI_USECOMPOUND.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_DSOFFSET);
|
||||
final int dsOffset = Integer.parseInt(readString(SI_DSOFFSET.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_DSSEGMENT);
|
||||
final String dsSegment = readString(SI_DSSEGMENT.length, scratch);
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_DSCOMPOUND);
|
||||
final boolean dsCompoundFile = Boolean.parseBoolean(readString(SI_DSCOMPOUND.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_DELGEN);
|
||||
final long delGen = Long.parseLong(readString(SI_DELGEN.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_NUM_NORMGEN);
|
||||
final int numNormGen = Integer.parseInt(readString(SI_NUM_NORMGEN.length, scratch));
|
||||
final Map<Integer,Long> normGen;
|
||||
if (numNormGen == 0) {
|
||||
normGen = null;
|
||||
} else {
|
||||
normGen = new HashMap<Integer,Long>();
|
||||
for (int i = 0; i < numNormGen; i++) {
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_NORMGEN_KEY);
|
||||
int key = Integer.parseInt(readString(SI_NORMGEN_KEY.length, scratch));
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_NORMGEN_VALUE);
|
||||
long value = Long.parseLong(readString(SI_NORMGEN_VALUE.length, scratch));
|
||||
normGen.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
SimpleTextUtil.readLine(input, scratch);
|
||||
assert StringHelper.startsWith(scratch, SI_NUM_DIAG);
|
||||
int numDiag = Integer.parseInt(readString(SI_NUM_DIAG.length, scratch));
|
||||
|
|
@ -159,9 +82,9 @@ public class SimpleTextSegmentInfosReader extends SegmentInfosReader {
|
|||
}
|
||||
|
||||
success = true;
|
||||
return new SegmentInfo(directory, version, segmentName, docCount, dsOffset,
|
||||
dsSegment, dsCompoundFile, normGen, isCompoundFile,
|
||||
delCount, null, diagnostics);
|
||||
return new SegmentInfo(directory, version, segmentName, docCount, -1,
|
||||
segmentName, false, null, isCompoundFile,
|
||||
0, null, diagnostics);
|
||||
} finally {
|
||||
if (!success) {
|
||||
IOUtils.closeWhileHandlingException(input);
|
||||
|
|
|
|||
|
|
@ -42,92 +42,20 @@ import org.apache.lucene.util.IOUtils;
|
|||
*/
|
||||
public class SimpleTextSegmentInfosWriter extends SegmentInfosWriter {
|
||||
|
||||
final static BytesRef VERSION = new BytesRef("version ");
|
||||
final static BytesRef COUNTER = new BytesRef("counter ");
|
||||
final static BytesRef NUM_USERDATA = new BytesRef("user data entries ");
|
||||
final static BytesRef USERDATA_KEY = new BytesRef(" key ");
|
||||
final static BytesRef USERDATA_VALUE = new BytesRef(" value ");
|
||||
final static BytesRef NUM_SEGMENTS = new BytesRef("number of segments ");
|
||||
final static BytesRef SI_VERSION = new BytesRef(" version ");
|
||||
final static BytesRef SI_DOCCOUNT = new BytesRef(" number of documents ");
|
||||
final static BytesRef SI_DELCOUNT = new BytesRef(" number of deletions ");
|
||||
final static BytesRef SI_USECOMPOUND = new BytesRef(" uses compound file ");
|
||||
final static BytesRef SI_DSOFFSET = new BytesRef(" docstore offset ");
|
||||
final static BytesRef SI_DSSEGMENT = new BytesRef(" docstore segment ");
|
||||
final static BytesRef SI_DSCOMPOUND = new BytesRef(" docstore is compound file ");
|
||||
final static BytesRef SI_DELGEN = new BytesRef(" deletion generation ");
|
||||
final static BytesRef SI_NUM_NORMGEN = new BytesRef(" norms generations ");
|
||||
final static BytesRef SI_NORMGEN_KEY = new BytesRef(" key ");
|
||||
final static BytesRef SI_NORMGEN_VALUE = new BytesRef(" value ");
|
||||
final static BytesRef SI_NUM_DIAG = new BytesRef(" diagnostics ");
|
||||
final static BytesRef SI_DIAG_KEY = new BytesRef(" key ");
|
||||
final static BytesRef SI_DIAG_VALUE = new BytesRef(" value ");
|
||||
|
||||
/*
|
||||
BytesRef scratch = new BytesRef();
|
||||
IndexOutput out = new ChecksumIndexOutput(dir.createOutput(segmentsFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))));
|
||||
boolean success = false;
|
||||
try {
|
||||
// required preamble:
|
||||
out.writeInt(SegmentInfos.FORMAT_CURRENT); // write FORMAT
|
||||
out.writeString(codecID); // write codecID
|
||||
// end preamble
|
||||
|
||||
// version
|
||||
SimpleTextUtil.write(out, VERSION);
|
||||
SimpleTextUtil.write(out, Long.toString(infos.version), scratch);
|
||||
SimpleTextUtil.writeNewline(out);
|
||||
|
||||
// counter
|
||||
SimpleTextUtil.write(out, COUNTER);
|
||||
SimpleTextUtil.write(out, Integer.toString(infos.counter), scratch);
|
||||
SimpleTextUtil.writeNewline(out);
|
||||
|
||||
// user data
|
||||
int numUserDataEntries = infos.getUserData() == null ? 0 : infos.getUserData().size();
|
||||
SimpleTextUtil.write(out, NUM_USERDATA);
|
||||
SimpleTextUtil.write(out, Integer.toString(numUserDataEntries), scratch);
|
||||
SimpleTextUtil.writeNewline(out);
|
||||
|
||||
if (numUserDataEntries > 0) {
|
||||
for (Map.Entry<String,String> userEntry : infos.getUserData().entrySet()) {
|
||||
SimpleTextUtil.write(out, USERDATA_KEY);
|
||||
SimpleTextUtil.write(out, userEntry.getKey(), scratch);
|
||||
SimpleTextUtil.writeNewline(out);
|
||||
|
||||
SimpleTextUtil.write(out, USERDATA_VALUE);
|
||||
SimpleTextUtil.write(out, userEntry.getValue(), scratch);
|
||||
SimpleTextUtil.writeNewline(out);
|
||||
}
|
||||
}
|
||||
|
||||
// infos size
|
||||
SimpleTextUtil.write(out, NUM_SEGMENTS);
|
||||
SimpleTextUtil.write(out, Integer.toString(infos.size()), scratch);
|
||||
SimpleTextUtil.writeNewline(out);
|
||||
|
||||
for (SegmentInfo si : infos) {
|
||||
writeInfo(out, si);
|
||||
}
|
||||
|
||||
success = true;
|
||||
return out;
|
||||
} finally {
|
||||
if (!success) {
|
||||
IOUtils.closeWhileHandlingException(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
@Override
|
||||
public void write(SegmentInfo si, FieldInfos fis) throws IOException {
|
||||
public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
|
||||
assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name;
|
||||
|
||||
String fileName = IndexFileNames.segmentFileName(si.name, "", SimpleTextSegmentInfosFormat.SI_EXTENSION);
|
||||
// nocommit what IOCtx
|
||||
boolean success = false;
|
||||
IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(0, 0)));
|
||||
IndexOutput output = dir.createOutput(fileName, ioContext);
|
||||
|
||||
try {
|
||||
BytesRef scratch = new BytesRef();
|
||||
|
|
@ -140,49 +68,10 @@ public class SimpleTextSegmentInfosWriter extends SegmentInfosWriter {
|
|||
SimpleTextUtil.write(output, Integer.toString(si.docCount), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_DELCOUNT);
|
||||
SimpleTextUtil.write(output, Integer.toString(si.getDelCount()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_USECOMPOUND);
|
||||
SimpleTextUtil.write(output, Boolean.toString(si.getUseCompoundFile()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_DSOFFSET);
|
||||
SimpleTextUtil.write(output, Integer.toString(si.getDocStoreOffset()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_DSSEGMENT);
|
||||
SimpleTextUtil.write(output, si.getDocStoreSegment(), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_DSCOMPOUND);
|
||||
SimpleTextUtil.write(output, Boolean.toString(si.getDocStoreIsCompoundFile()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_DELGEN);
|
||||
SimpleTextUtil.write(output, Long.toString(si.getDelGen()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
Map<Integer,Long> normGen = si.getNormGen();
|
||||
int numNormGen = normGen == null ? 0 : normGen.size();
|
||||
SimpleTextUtil.write(output, SI_NUM_NORMGEN);
|
||||
SimpleTextUtil.write(output, Integer.toString(numNormGen), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
// nocommit no more:
|
||||
if (numNormGen > 0) {
|
||||
for (Entry<Integer,Long> entry : normGen.entrySet()) {
|
||||
SimpleTextUtil.write(output, SI_NORMGEN_KEY);
|
||||
SimpleTextUtil.write(output, Integer.toString(entry.getKey()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
|
||||
SimpleTextUtil.write(output, SI_NORMGEN_VALUE);
|
||||
SimpleTextUtil.write(output, Long.toString(entry.getValue()), scratch);
|
||||
SimpleTextUtil.writeNewline(output);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String,String> diagnostics = si.getDiagnostics();
|
||||
int numDiagnostics = diagnostics == null ? 0 : diagnostics.size();
|
||||
SimpleTextUtil.write(output, SI_NUM_DIAG);
|
||||
|
|
|
|||
|
|
@ -478,7 +478,7 @@ class DocumentsWriterPerThread {
|
|||
consumer.flush(flushState);
|
||||
pendingDeletes.terms.clear();
|
||||
final SegmentInfo newSegment = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, segment, flushState.numDocs,
|
||||
0, segment, false, null, false, 0,
|
||||
-1, segment, false, null, false, 0,
|
||||
flushState.codec,
|
||||
null);
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfosFormat;
|
||||
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
|
||||
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
|
||||
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
||||
|
|
@ -46,6 +47,7 @@ import org.apache.lucene.store.IOContext;
|
|||
import org.apache.lucene.store.Lock;
|
||||
import org.apache.lucene.store.LockObtainFailedException;
|
||||
import org.apache.lucene.store.MergeInfo;
|
||||
import org.apache.lucene.store.TrackingDirectoryWrapper;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.Constants;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
|
@ -2052,7 +2054,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
// creating CFS so that 1) .si isn't slurped into CFS,
|
||||
// and 2) .si reflects useCompoundFile=true change
|
||||
// above:
|
||||
codec.segmentInfosFormat().getSegmentInfosWriter().write(newSegment, flushedSegment.fieldInfos);
|
||||
codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, newSegment, flushedSegment.fieldInfos, context);
|
||||
newSegment.clearFilesCache();
|
||||
|
||||
// nocommit ideally we would freeze merge.info here!!
|
||||
|
|
@ -2338,7 +2340,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
// creating CFS so that 1) .si isn't slurped into CFS,
|
||||
// and 2) .si reflects useCompoundFile=true change
|
||||
// above:
|
||||
codec.segmentInfosFormat().getSegmentInfosWriter().write(info, mergeState.fieldInfos);
|
||||
codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, info, mergeState.fieldInfos, context);
|
||||
info.clearFilesCache();
|
||||
|
||||
// Register the new segment
|
||||
|
|
@ -2376,23 +2378,40 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
}
|
||||
|
||||
Set<String> codecDocStoreFiles = new HashSet<String>();
|
||||
if (info.getDocStoreOffset() != -1) {
|
||||
// only violate the codec this way if its preflex
|
||||
final boolean hasSharedDocStore = info.getDocStoreOffset() != -1;
|
||||
final String segmentInfoFileName3X = IndexFileNames.segmentFileName(info.name,
|
||||
"",
|
||||
Lucene3xSegmentInfosFormat.SI_EXTENSION);
|
||||
|
||||
if (hasSharedDocStore) {
|
||||
// only violate the codec this way if its preflex &
|
||||
// shares doc stores
|
||||
info.getCodec().storedFieldsFormat().files(info, codecDocStoreFiles);
|
||||
info.getCodec().termVectorsFormat().files(info, codecDocStoreFiles);
|
||||
}
|
||||
|
||||
//System.out.println("copy seg=" + info.name + " version=" + info.getVersion());
|
||||
|
||||
// Copy the segment files
|
||||
for (String file: info.files()) {
|
||||
|
||||
// nocommit messy: insteda we should pull .files()
|
||||
// from the codec's SIFormat and check if it's in
|
||||
// there...
|
||||
if (file.endsWith(".si")) {
|
||||
continue;
|
||||
}
|
||||
// Same SI as before but we change directory, name and docStoreSegment:
|
||||
SegmentInfo newInfo = new SegmentInfo(directory, info.getVersion(), segName, info.docCount, info.getDocStoreOffset(),
|
||||
newDsName, info.getDocStoreIsCompoundFile(), info.getNormGen(), info.getUseCompoundFile(),
|
||||
info.getDelCount(), info.getCodec(), info.getDiagnostics());
|
||||
newInfo.setDelGen(info.getDelGen());
|
||||
|
||||
// We must rewrite the SI file because it references
|
||||
// segment name (its own name, if its 3.x, and doc
|
||||
// store segment name):
|
||||
TrackingDirectoryWrapper dirWrapper = new TrackingDirectoryWrapper(directory);
|
||||
try {
|
||||
newInfo.getCodec().segmentInfosFormat().getSegmentInfosWriter().write(dirWrapper, newInfo, null, context);
|
||||
} catch (UnsupportedOperationException uoe) {
|
||||
// OK: 3x codec cannot write a new SI file;
|
||||
// SegmentInfos will write this on commit
|
||||
}
|
||||
final Set<String> siFileNames = dirWrapper.getCreatedFiles();
|
||||
|
||||
// Copy the segment's files
|
||||
for (String file: info.files()) {
|
||||
|
||||
final String newFileName;
|
||||
if (codecDocStoreFiles.contains(file)) {
|
||||
|
|
@ -2405,28 +2424,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
newFileName = segName + IndexFileNames.stripSegmentName(file);
|
||||
}
|
||||
|
||||
if (siFileNames != null && siFileNames.contains(newFileName)) {
|
||||
// We already rewwrote this above
|
||||
continue;
|
||||
}
|
||||
|
||||
assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
|
||||
assert !copiedFiles.contains(file): "file \"" + file + "\" is being copied more than once";
|
||||
copiedFiles.add(file);
|
||||
//System.out.println("COPY " + file + " -> " + newFileName);
|
||||
info.dir.copy(directory, file, newFileName, context);
|
||||
}
|
||||
|
||||
// Same SI as before but we change directory and name:
|
||||
SegmentInfo newInfo = new SegmentInfo(directory, info.getVersion(), segName, info.docCount, info.getDocStoreOffset(),
|
||||
newDsName, info.getDocStoreIsCompoundFile(), info.getNormGen(), info.getUseCompoundFile(),
|
||||
info.getDelCount(), info.getCodec(), info.getDiagnostics());
|
||||
newInfo.setDelGen(info.getDelGen());
|
||||
|
||||
// nocommit need to pass real FIS...
|
||||
// nocommit maybe we don't pass FIS......?
|
||||
// nocommit messy....
|
||||
//if (!newInfo.getCodec().getName().equals("Lucene3x")) {
|
||||
if (!newInfo.getVersion().startsWith("3.")) {
|
||||
//System.out.println(" now write si for seg=" + newInfo.name + " codec=" + newInfo.getCodec());
|
||||
newInfo.getCodec().segmentInfosFormat().getSegmentInfosWriter().write(newInfo, null);
|
||||
}
|
||||
|
||||
return newInfo;
|
||||
}
|
||||
|
||||
|
|
@ -3511,7 +3519,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
synchronized (this) { // Guard segmentInfos
|
||||
useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, merge.info);
|
||||
}
|
||||
|
||||
|
||||
if (useCompoundFile) {
|
||||
success = false;
|
||||
final String compoundFileName = IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
|
||||
|
|
@ -3520,7 +3528,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
if (infoStream.isEnabled("IW")) {
|
||||
infoStream.message("IW", "create compound file " + compoundFileName);
|
||||
}
|
||||
createCompoundFile(directory, compoundFileName, checkAbort, merge.info, new IOContext(merge.getMergeInfo()));
|
||||
createCompoundFile(directory, compoundFileName, checkAbort, merge.info, context);
|
||||
success = true;
|
||||
} catch (IOException ioe) {
|
||||
synchronized(this) {
|
||||
|
|
@ -3575,7 +3583,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
|
|||
// creating CFS so that 1) .si isn't slurped into CFS,
|
||||
// and 2) .si reflects useCompoundFile=true change
|
||||
// above:
|
||||
codec.segmentInfosFormat().getSegmentInfosWriter().write(merge.info, mergeState.fieldInfos);
|
||||
codec.segmentInfosFormat().getSegmentInfosWriter().write(directory, merge.info, mergeState.fieldInfos, context);
|
||||
merge.info.clearFilesCache();
|
||||
|
||||
// nocommit ideally we would freeze merge.info here!!
|
||||
|
|
|
|||
|
|
@ -284,6 +284,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
|
|||
info.setCodec(codec);
|
||||
info.setDelGen(input.readLong());
|
||||
info.setDelCount(input.readInt());
|
||||
assert info.getDelCount() <= info.docCount;
|
||||
add(info);
|
||||
}
|
||||
userData = input.readStringStringMap();
|
||||
|
|
@ -297,9 +298,6 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
|
|||
}
|
||||
}
|
||||
|
||||
// nocommit all 3.x indices have checksum right...????
|
||||
// ie we added it during 2.x? i think so!
|
||||
|
||||
final long checksumNow = input.getChecksum();
|
||||
final long checksumThen = input.readLong();
|
||||
if (checksumNow != checksumThen) {
|
||||
|
|
@ -376,8 +374,6 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
|
|||
if (!directory.fileExists(fileName)) {
|
||||
//System.out.println("write 3x info seg=" + si.name + " version=" + si.getVersion() + " codec=" + si.getCodec().getName());
|
||||
write3xInfo(si);
|
||||
// nocommit do this after, on success...
|
||||
//si.setVersion("4.0");
|
||||
si.clearFilesCache();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ public abstract class Directory implements Closeable {
|
|||
* their own locking implementation.
|
||||
*/
|
||||
public LockFactory getLockFactory() {
|
||||
return this.lockFactory;
|
||||
return this.lockFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
package org.apache.lucene.store;
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/** A delegating Directory that records which files were
|
||||
* written to and deleted. */
|
||||
public final class TrackingDirectoryWrapper extends Directory implements Closeable {
|
||||
|
||||
private final Directory other;
|
||||
private final Set<String> createdFileNames = Collections.synchronizedSet(new HashSet<String>());
|
||||
|
||||
public TrackingDirectoryWrapper(Directory other) {
|
||||
this.other = other;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] listAll() throws IOException {
|
||||
return other.listAll();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean fileExists(String name) throws IOException {
|
||||
return other.fileExists(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFile(String name) throws IOException {
|
||||
createdFileNames.remove(name);
|
||||
other.deleteFile(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long fileLength(String name) throws IOException {
|
||||
return other.fileLength(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexOutput createOutput(String name, IOContext context) throws IOException {
|
||||
createdFileNames.add(name);
|
||||
return other.createOutput(name, context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void sync(Collection<String> names) throws IOException {
|
||||
other.sync(names);
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexInput openInput(String name, IOContext context) throws IOException {
|
||||
return other.openInput(name, context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Lock makeLock(String name) {
|
||||
return other.makeLock(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clearLock(String name) throws IOException {
|
||||
other.clearLock(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
other.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLockFactory(LockFactory lockFactory) throws IOException {
|
||||
other.setLockFactory(lockFactory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public LockFactory getLockFactory() {
|
||||
return other.getLockFactory();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLockID() {
|
||||
return other.getLockID();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TrackingDirectoryWrapper(" + other.toString() + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
|
||||
// nocommit add dest to created files!?
|
||||
other.copy(to, src, dest, context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Directory.IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException {
|
||||
return other.createSlicer(name, context);
|
||||
}
|
||||
|
||||
// nocommit no?
|
||||
//@Override
|
||||
//protected final void ensureOpen() throws
|
||||
//AlreadyClosedException {
|
||||
|
||||
public Set<String> getCreatedFiles() {
|
||||
return createdFileNames;
|
||||
}
|
||||
}
|
||||
|
|
@ -106,9 +106,6 @@ public class TestParallelReaderEmptyIndex extends LuceneTestCase {
|
|||
iw.addDocument(doc);
|
||||
iw.close();
|
||||
|
||||
// nocommit
|
||||
_TestUtil.checkIndex(rd1);
|
||||
|
||||
IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
|
||||
.setMergePolicy(NoMergePolicy.COMPOUND_FILES);
|
||||
if (VERBOSE) {
|
||||
|
|
|
|||
|
|
@ -39,17 +39,20 @@ import org.apache.lucene.util.IOUtils;
|
|||
*/
|
||||
class PreFlexRWSegmentInfosWriter extends SegmentInfosWriter {
|
||||
|
||||
// NOTE: this is not "really" 3.x format, because we are
|
||||
// writing each SI to its own file, vs 3.x where the list
|
||||
// of segments and SI for each segment is written into a
|
||||
// single segments_N file
|
||||
|
||||
/** Save a single segment's info. */
|
||||
@Override
|
||||
public void write(SegmentInfo si, FieldInfos fis) throws IOException {
|
||||
public void write(Directory dir ,SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
|
||||
|
||||
// NOTE: this is NOT how 3.x is really written...
|
||||
String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfosFormat.SI_EXTENSION);
|
||||
|
||||
// nocommit what IOCtx
|
||||
boolean success = false;
|
||||
|
||||
IndexOutput output = si.dir.createOutput(fileName, new IOContext(new FlushInfo(0, 0)));
|
||||
IndexOutput output = dir.createOutput(fileName, ioContext);
|
||||
try {
|
||||
// we are about to write this SI in 3.x format, dropping all codec information, etc.
|
||||
// so it had better be a 3.x segment or you will get very confusing errors later.
|
||||
|
|
|
|||
Loading…
Reference in New Issue