Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5969: file mismatch detection for 5.0 fnm 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5969@1627544 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-09-25 13:55:26 +00:00
parent d1a67913d4
commit e5b18421d8
27 changed files with 69 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@ -49,8 +50,8 @@ class Lucene40FieldInfosReader extends FieldInfosReader {
}
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
IndexInput input = directory.openInput(fileName, iocontext);
boolean success = false;

2
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
View File

@ -73,7 +73,7 @@ public class Lucene40SegmentInfoReader extends SegmentInfoReader {
CodecUtil.checkEOF(input);
final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, null);
si.setFiles(files);
success = true;

5
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@ -49,8 +50,8 @@ final class Lucene42FieldInfosReader extends FieldInfosReader {
}
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene42FieldInfosFormat.EXTENSION);
IndexInput input = directory.openInput(fileName, iocontext);
boolean success = false;

5
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -47,8 +48,8 @@ final class Lucene46FieldInfosReader extends FieldInfosReader {
}
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
int codecVersion = CodecUtil.checkHeader(input, Lucene46FieldInfosFormat.CODEC_NAME,
Lucene46FieldInfosFormat.FORMAT_START,

5
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -43,8 +44,8 @@ final class Lucene46FieldInfosWriter extends FieldInfosWriter {
}
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
CodecUtil.writeHeader(output, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_CURRENT);
output.writeVInt(infos.size());

5
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@ -45,11 +46,11 @@ public class Lucene40FieldInfosWriter extends FieldInfosWriter {
}
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
if (!segmentSuffix.isEmpty()) {
throw new UnsupportedOperationException("4.0 does not support fieldinfo updates");
}
final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
IndexOutput output = directory.createOutput(fileName, context);
boolean success = false;
try {

2
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWSegmentInfoFormat.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.codecs.lucene40;
import org.apache.lucene.codecs.SegmentInfoWriter;
/** read-write version of 4.6 segmentinfos for testing */
/** read-write version of 4.0 segmentinfos for testing */
public class Lucene40RWSegmentInfoFormat extends Lucene40SegmentInfoFormat {
@Override

5
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@ -45,11 +46,11 @@ public final class Lucene42FieldInfosWriter extends FieldInfosWriter {
}
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
if (!segmentSuffix.isEmpty()) {
throw new UnsupportedOperationException("4.2 does not support fieldinfo updates");
}
final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, "", Lucene42FieldInfosFormat.EXTENSION);
IndexOutput output = directory.createOutput(fileName, context);
boolean success = false;
try {

5
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -47,8 +48,8 @@ import static org.apache.lucene.codecs.simpletext.SimpleTextFieldInfosWriter.*;
public class SimpleTextFieldInfosReader extends FieldInfosReader {
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
BytesRefBuilder scratch = new BytesRefBuilder();

5
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@ -61,8 +62,8 @@ public class SimpleTextFieldInfosWriter extends FieldInfosWriter {
final static BytesRef ATT_VALUE = new BytesRef(" value ");
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
IndexOutput out = directory.createOutput(fileName, context);
BytesRefBuilder scratch = new BytesRefBuilder();
boolean success = false;

3
lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -35,5 +36,5 @@ public abstract class FieldInfosReader {
/** Read the {@link FieldInfos} previously written with {@link
* FieldInfosWriter}. */
public abstract FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException;
public abstract FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException;
}

3
lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -35,5 +36,5 @@ public abstract class FieldInfosWriter {
/** Writes the provided {@link FieldInfos} to the
* directory. */
public abstract void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException;
public abstract void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException;
}

3
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.DataOutput;
/**
@ -36,6 +37,7 @@ import org.apache.lucene.store.DataOutput;
* <p>Data types:
* <ul>
* <li>Header --&gt; {@link CodecUtil#checkHeader CodecHeader}</li>
* <li>SegmentID --&gt; {@link DataOutput#writeString String}</li>
* <li>FieldsCount --&gt; {@link DataOutput#writeVInt VInt}</li>
* <li>FieldName --&gt; {@link DataOutput#writeString String}</li>
* <li>FieldBits, DocValuesBits --&gt; {@link DataOutput#writeByte Byte}</li>
@ -47,6 +49,7 @@ import org.apache.lucene.store.DataOutput;
* </p>
* Field Descriptions:
* <ul>
* <li>SegmentID: {@link SegmentInfo#getId()} this file belongs to</li>
* <li>FieldsCount: the number of fields in this file.</li>
* <li>FieldName: name of the field as a UTF-8 String.</li>
* <li>FieldNumber: the field's number. Note that unlike previous versions of

9
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -47,8 +48,8 @@ final class Lucene50FieldInfosReader extends FieldInfosReader {
}
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
Throwable priorE = null;
FieldInfo infos[] = null;
@ -56,6 +57,10 @@ final class Lucene50FieldInfosReader extends FieldInfosReader {
CodecUtil.checkHeader(input, Lucene50FieldInfosFormat.CODEC_NAME,
Lucene50FieldInfosFormat.FORMAT_START,
Lucene50FieldInfosFormat.FORMAT_CURRENT);
String id = input.readString();
if (!id.equals(segmentInfo.getId())) {
throw new CorruptIndexException("file mismatch, expected segment id=" + segmentInfo.getId() + ", got=" + id, input);
}
final int size = input.readVInt(); //read in the size
infos = new FieldInfo[size];

6
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosWriter.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -43,10 +44,11 @@ final class Lucene50FieldInfosWriter extends FieldInfosWriter {
}
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
CodecUtil.writeHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT);
output.writeString(segmentInfo.getId());
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
fi.checkConsistency();

2
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -119,7 +119,7 @@ final class DefaultIndexingChain extends DocConsumer {
// FreqProxTermsWriter does this with
// FieldInfo.storePayload.
FieldInfosWriter infosWriter = docWriter.codec.fieldInfosFormat().getFieldInfosWriter();
infosWriter.write(state.directory, state.segmentInfo.name, "", state.fieldInfos, IOContext.DEFAULT);
infosWriter.write(state.directory, state.segmentInfo, "", state.fieldInfos, IOContext.DEFAULT);
}
/** Writes all buffered doc values (called from {@link #flush}). */

2
lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
View File

@ -451,7 +451,7 @@ class ReadersAndUpdates {
final IOContext infosContext = new IOContext(new FlushInfo(info.info.getDocCount(), estInfosSize));
// separately also track which files were created for this gen
final TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
infosFormat.getFieldInfosWriter().write(trackingDir, info.info.name, segmentSuffix, fieldInfos, infosContext);
infosFormat.getFieldInfosWriter().write(trackingDir, info.info, segmentSuffix, fieldInfos, infosContext);
info.advanceFieldInfosGen();
return trackingDir.getCreatedFiles();
}

9
lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
View File

@ -82,15 +82,6 @@ public final class SegmentInfo {
return diagnostics;
}
/**
* Construct a new complete SegmentInfo instance from
* input, with a newly generated random id.
*/
public SegmentInfo(Directory dir, Version version, String name, int docCount,
boolean isCompoundFile, Codec codec, Map<String,String> diagnostics) {
this(dir, version, name, docCount, isCompoundFile, codec, diagnostics, null);
}
/**
* Construct a new complete SegmentInfo instance from input.
* <p>Note: this is public only to allow access from

2
lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -146,7 +146,7 @@ final class SegmentMerger {
// write the merged infos
FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
fieldInfosWriter.write(directory, mergeState.segmentInfo.name, "", mergeState.fieldInfos, context);
fieldInfosWriter.write(directory, mergeState.segmentInfo, "", mergeState.fieldInfos, context);
return mergeState;
}

2
lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -217,7 +217,7 @@ public final class SegmentReader extends LeafReader implements Accountable {
final String segmentSuffix = info.getFieldInfosGen() == -1 ? "" : Long.toString(info.getFieldInfosGen(), Character.MAX_RADIX);
Codec codec = info.info.getCodec();
FieldInfosFormat fisFormat = codec.fieldInfosFormat();
return fisFormat.getFieldInfosReader().read(dir, info.info.name, segmentSuffix, IOContext.READONCE);
return fisFormat.getFieldInfosReader().read(dir, info.info, segmentSuffix, IOContext.READONCE);
} finally {
if (closeDir) {
dir.close();

7
lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
View File

@ -42,6 +42,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
import org.junit.BeforeClass;
@ -248,7 +249,7 @@ public class TestCodecs extends LuceneTestCase {
final Directory dir = newDirectory();
this.write(fieldInfos, dir, fields);
Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null);
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId());
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random())));
@ -305,7 +306,7 @@ public class TestCodecs extends LuceneTestCase {
this.write(fieldInfos, dir, fields);
Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null);
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId());
if (VERBOSE) {
System.out.println("TEST: now read postings");
@ -800,7 +801,7 @@ public class TestCodecs extends LuceneTestCase {
private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
final Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null);
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId());
final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));
Arrays.sort(fields);

16
lucene/core/src/test/org/apache/lucene/index/TestDoc.java
View File

@ -43,6 +43,7 @@ import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
@ -218,7 +219,7 @@ public class TestDoc extends LuceneTestCase {
final Codec codec = Codec.getDefault();
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
final SegmentInfo si = new SegmentInfo(si1.info.dir, Version.LATEST, merged, -1, false, codec, null);
final SegmentInfo si = new SegmentInfo(si1.info.dir, Version.LATEST, merged, -1, false, codec, null, StringHelper.randomId());
SegmentMerger merger = new SegmentMerger(Arrays.<LeafReader>asList(r1, r2),
si, InfoStream.getDefault(), trackingDir,
@ -226,21 +227,18 @@ public class TestDoc extends LuceneTestCase {
MergeState mergeState = merger.merge();
r1.close();
r2.close();
final SegmentInfo info = new SegmentInfo(si1.info.dir, Version.LATEST, merged,
si1.info.getDocCount() + si2.info.getDocCount(),
false, codec, null);
info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
r2.close();;
si.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
if (useCompoundFile) {
Collection<String> filesToDelete = IndexWriter.createCompoundFile(InfoStream.getDefault(), dir, MergeState.CheckAbort.NONE, info, newIOContext(random()));
info.setUseCompoundFile(true);
Collection<String> filesToDelete = IndexWriter.createCompoundFile(InfoStream.getDefault(), dir, MergeState.CheckAbort.NONE, si, newIOContext(random()));
si.setUseCompoundFile(true);
for (final String fileToDelete : filesToDelete) {
si1.info.dir.deleteFile(fileToDelete);
}
}
return new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
return new SegmentCommitInfo(si, 0, -1L, -1L, -1L);
}

115
lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
View File

@ -1,115 +0,0 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.LuceneTestCase;
//import org.cnlp.utils.properties.ResourceBundleHelper;
public class TestFieldInfos extends LuceneTestCase {
private Document testDoc = new Document();
@Override
public void setUp() throws Exception {
super.setUp();
DocHelper.setupDoc(testDoc);
}
public FieldInfos createAndWriteFieldInfos(Directory dir, String filename) throws IOException{
//Positive test of FieldInfos
assertTrue(testDoc != null);
FieldInfos.Builder builder = new FieldInfos.Builder();
for (IndexableField field : testDoc.getFields()) {
builder.addOrUpdate(field.name(), field.fieldType());
}
FieldInfos fieldInfos = builder.finish();
//Since the complement is stored as well in the fields map
assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
IndexOutput output = dir.createOutput(filename, newIOContext(random()));
assertTrue(output != null);
//Use a RAMOutputStream
FieldInfosWriter writer = Codec.getDefault().fieldInfosFormat().getFieldInfosWriter();
writer.write(dir, filename, "", fieldInfos, IOContext.DEFAULT);
output.close();
return fieldInfos;
}
public FieldInfos readFieldInfos(Directory dir, String filename) throws IOException {
FieldInfosReader reader = Codec.getDefault().fieldInfosFormat().getFieldInfosReader();
return reader.read(dir, filename, "", IOContext.DEFAULT);
}
public void test() throws IOException {
String name = "testFile";
Directory dir = newDirectory();
FieldInfos fieldInfos = createAndWriteFieldInfos(dir, name);
FieldInfos readIn = readFieldInfos(dir, name);
assertTrue(fieldInfos.size() == readIn.size());
FieldInfo info = readIn.fieldInfo("textField1");
assertTrue(info != null);
assertTrue(info.hasVectors() == false);
assertTrue(info.omitsNorms() == false);
info = readIn.fieldInfo("textField2");
assertTrue(info != null);
assertTrue(info.omitsNorms() == false);
info = readIn.fieldInfo("textField3");
assertTrue(info != null);
assertTrue(info.hasVectors() == false);
assertTrue(info.omitsNorms() == true);
info = readIn.fieldInfo("omitNorms");
assertTrue(info != null);
assertTrue(info.hasVectors() == false);
assertTrue(info.omitsNorms() == true);
dir.close();
}
public void testReadOnly() throws IOException {
String name = "testFile";
Directory dir = newDirectory();
FieldInfos fieldInfos = createAndWriteFieldInfos(dir, name);
FieldInfos readOnly = readFieldInfos(dir, name);
assertReadOnly(readOnly, fieldInfos);
dir.close();
}
private void assertReadOnly(FieldInfos readOnly, FieldInfos modifiable) {
assertEquals(modifiable.size(), readOnly.size());
// assert we can iterate
for (FieldInfo fi : readOnly) {
assertEquals(fi.name, modifiable.fieldInfo(fi.number).name);
}
}
}

6
lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
@ -78,7 +79,7 @@ public class TestSegmentMerger extends LuceneTestCase {
public void testMerge() throws IOException {
final Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(mergedDir, Version.LATEST, mergedSegment, -1, false, codec, null);
final SegmentInfo si = new SegmentInfo(mergedDir, Version.LATEST, mergedSegment, -1, false, codec, null, StringHelper.randomId());
SegmentMerger merger = new SegmentMerger(Arrays.<LeafReader>asList(reader1, reader2),
si, InfoStream.getDefault(), mergedDir,
@ -88,8 +89,7 @@ public class TestSegmentMerger extends LuceneTestCase {
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = new SegmentReader(new SegmentCommitInfo(
new SegmentInfo(mergedDir, Version.LATEST, mergedSegment, docsMerged,
false, codec, null),
mergeState.segmentInfo,
0, -1L, -1L, -1L),
newIOContext(random()));
assertTrue(mergedReader != null);

3
lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
View File

@ -28,6 +28,7 @@ import java.util.List;
import java.util.Locale;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.StringHelper;
/**
* Command-line tool that enables listing segments in an
@ -137,7 +138,7 @@ public class IndexSplitter {
SegmentInfo info = infoPerCommit.info;
// Same info just changing the dir:
SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.getDocCount(),
info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics());
info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId());
destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(),
infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(),
infoPerCommit.getDocValuesGen()));

5
lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyFieldInfosFormat.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -59,11 +60,11 @@ class CrankyFieldInfosFormat extends FieldInfosFormat {
}
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
if (random.nextInt(100) == 0) {
throw new IOException("Fake IOException from FieldInfosWriter.write()");
}
delegate.write(directory, segmentName, segmentSuffix, infos, context);
delegate.write(directory, segmentInfo, segmentSuffix, infos, context);
}
}
}

3
lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
View File

@ -57,6 +57,7 @@ import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
@ -675,7 +676,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
// randomly index at lower IndexOption
private FieldsProducer buildIndex(Directory dir, IndexOptions maxAllowed, boolean allowPayloads, boolean alwaysTestMax) throws IOException {
Codec codec = getCodec();
SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, "_0", maxDoc, false, codec, null);
SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, "_0", maxDoc, false, codec, null, StringHelper.randomId());
int maxIndexOption = Arrays.asList(IndexOptions.values()).indexOf(maxAllowed);
if (VERBOSE) {