diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java index 370ddc1d2c7..784eb8e34c8 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java @@ -42,7 +42,8 @@ public class AppendingTermsDictReader extends StandardTermsDictReader { @Override protected void readHeader(IndexInput in) throws IOException { - CodecUtil.checkHeader(in, AppendingTermsDictWriter.CODEC_NAME, StandardTermsDictWriter.VERSION_CURRENT); + CodecUtil.checkHeader(in, AppendingTermsDictWriter.CODEC_NAME, + StandardTermsDictWriter.VERSION_START, StandardTermsDictWriter.VERSION_CURRENT); } @Override diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java index e61fe8c667e..3faa5d733bc 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java @@ -37,7 +37,8 @@ public class AppendingTermsIndexReader extends SimpleStandardTermsIndexReader { @Override protected void readHeader(IndexInput input) throws IOException { - CodecUtil.checkHeader(input, AppendingTermsIndexWriter.CODEC_NAME, AppendingTermsIndexWriter.VERSION_START); + CodecUtil.checkHeader(input, AppendingTermsIndexWriter.CODEC_NAME, + AppendingTermsIndexWriter.VERSION_START, AppendingTermsIndexWriter.VERSION_START); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 1dabe02c94f..7c17357a41c 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -24,6 +24,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.AbstractField; // for javadocs import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -343,12 +344,15 @@ public class CheckIndex { String sFormat = ""; boolean skip = false; - if (format == SegmentInfos.FORMAT_DIAGNOSTICS) + if (format == DefaultSegmentInfosWriter.FORMAT_DIAGNOSTICS) { sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]"; - else if (format == SegmentInfos.FORMAT_4_0) - sFormat = "FORMAT_FLEX_POSTINGS [Lucene 4.0]"; - else if (format < SegmentInfos.CURRENT_FORMAT) { - sFormat = "int=" + format + " [newer version of Lucene than this tool]"; + } else if (format == DefaultSegmentInfosWriter.FORMAT_4_0) { + sFormat = "FORMAT_4_0 [Lucene 4.0]"; + } else if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) { + sFormat = "int=" + format + " [newer version of Lucene than this tool supports]"; + skip = true; + } else if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) { + sFormat = "int=" + format + " [older version of Lucene than this tool supports]"; skip = true; } diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index bf065b2a81d..d4eb996e967 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -39,7 +39,10 @@ public final class FieldInfos { // First used in 2.9; prior to 2.9 there was no format header public static final int FORMAT_START = -2; - static final int CURRENT_FORMAT = FORMAT_START; + // whenever you add a new format, make it 1 smaller (negative version logic)! + static final int FORMAT_CURRENT = FORMAT_START; + + static final int FORMAT_MINIMUM = FORMAT_START; static final byte IS_INDEXED = 0x1; static final byte STORE_TERMVECTOR = 0x2; @@ -286,7 +289,7 @@ public final class FieldInfos { } public void write(IndexOutput output) throws IOException { - output.writeVInt(CURRENT_FORMAT); + output.writeVInt(FORMAT_CURRENT); output.writeVInt(size()); for (int i = 0; i < size(); i++) { FieldInfo fi = fieldInfo(i); @@ -307,8 +310,11 @@ public final class FieldInfos { private void read(IndexInput input, String fileName) throws IOException { format = input.readVInt(); - if (format > FORMAT_START) { - throw new CorruptIndexException("unrecognized format " + format + " in file \"" + fileName + "\""); + if (format > FORMAT_MINIMUM) { + throw new IndexFormatTooOldException(fileName, format, FORMAT_MINIMUM, FORMAT_CURRENT); + } + if (format < FORMAT_CURRENT) { + throw new IndexFormatTooNewException(fileName, format, FORMAT_MINIMUM, FORMAT_CURRENT); } final int size = input.readVInt(); //read in the size diff --git a/lucene/src/java/org/apache/lucene/index/FieldsReader.java b/lucene/src/java/org/apache/lucene/index/FieldsReader.java index 6211e335303..ee5882464ed 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/FieldsReader.java @@ -100,13 +100,15 @@ final class FieldsReader implements Cloneable { fieldInfos = fn; cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), readBufferSize); - cloneableIndexStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), readBufferSize); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + cloneableIndexStream = d.openInput(indexStreamFN, readBufferSize); format = cloneableIndexStream.readInt(); + if (format < FieldsWriter.FORMAT_MINIMUM) + throw new IndexFormatTooOldException(indexStreamFN, format, FieldsWriter.FORMAT_MINIMUM, FieldsWriter.FORMAT_CURRENT); if (format > FieldsWriter.FORMAT_CURRENT) - throw new CorruptIndexException("Incompatible format version: " + format + " expected " - + FieldsWriter.FORMAT_CURRENT + " or lower"); + throw new IndexFormatTooNewException(indexStreamFN, format, FieldsWriter.FORMAT_MINIMUM, FieldsWriter.FORMAT_CURRENT); fieldsStream = (IndexInput) cloneableFieldsStream.clone(); @@ -185,11 +187,9 @@ final class FieldsReader implements Cloneable { } boolean canReadRawDocs() { - // Disable reading raw docs in 2.x format, because of the removal of compressed - // fields in 3.0. We don't want rawDocs() to decode field bits to figure out - // if a field was compressed, hence we enforce ordinary (non-raw) stored field merges - // for <3.0 indexes. - return format >= FieldsWriter.FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; + // Since we currently only support >3.0 format anymore, always return true! + // I leave this method in because it may help for later format changes. + return true; } final Document doc(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException { @@ -306,7 +306,6 @@ final class FieldsReader implements Cloneable { private void addField(Document doc, FieldInfo fi, boolean binary, boolean tokenize) throws CorruptIndexException, IOException { - //we have a binary stored field, and it may be compressed if (binary) { int toRead = fieldsStream.readVInt(); final byte[] b = new byte[toRead]; diff --git a/lucene/src/java/org/apache/lucene/index/FieldsWriter.java b/lucene/src/java/org/apache/lucene/index/FieldsWriter.java index dcc6c8df5b9..2b29270f73d 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FieldsWriter.java @@ -39,6 +39,9 @@ final class FieldsWriter // switch to a new format! static final int FORMAT_CURRENT = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; + // when removing support for old versions, leave the last supported version here + static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; + private FieldInfos fieldInfos; private IndexOutput fieldsStream; diff --git a/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java b/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java new file mode 100644 index 00000000000..7448d88f758 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +/** + * This exception is thrown when Lucene detects + * an index that is newer than this Lucene version. + */ +public class IndexFormatTooNewException extends CorruptIndexException { + + public IndexFormatTooNewException(String filename, int version, int minVersion, int maxVersion) { + super("Format version is not supported" + (filename!=null ? (" in file '" + filename + "'") : "") + + ": " + version + " (needs to be between " + minVersion + " and " + maxVersion + ")"); + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java b/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java new file mode 100644 index 00000000000..9be38a91e2a --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +/** + * This exception is thrown when Lucene detects + * an index that is too old for this Lucene version + */ +public class IndexFormatTooOldException extends CorruptIndexException { + + public IndexFormatTooOldException(String filename, int version, int minVersion, int maxVersion) { + super("Format version is not supported" + (filename!=null ? (" in file '" + filename + "'") : "") + + ": " + version + " (needs to be between " + minVersion + " and " + maxVersion + + "). This version of Lucene only supports indexes created with release 3.0 and later."); + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java index f5f46133022..46864813b51 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java @@ -22,6 +22,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -154,7 +155,7 @@ public final class SegmentInfo { docStoreSegment = name; docStoreIsCompoundFile = false; } - if (format > SegmentInfos.FORMAT_4_0) { + if (format > DefaultSegmentInfosWriter.FORMAT_4_0) { // pre-4.0 indexes write a byte if there is a single norms file byte b = input.readByte(); assert 1 == b; @@ -177,7 +178,7 @@ public final class SegmentInfo { // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); - if (format <= SegmentInfos.FORMAT_4_0) + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) codecName = input.readString(); else codecName = "PreFlex"; diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java index efcba816a4e..ecd4631c817 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java @@ -53,20 +53,10 @@ public final class SegmentInfos extends Vector { * be removed, however the numbers should continue to decrease. */ - /** Used for the segments.gen file only! */ + /** Used for the segments.gen file only! + * Whenever you add a new format, make it 1 smaller (negative version logic)! */ public static final int FORMAT_SEGMENTS_GEN_CURRENT = -2; - - /** This format adds optional per-segment String - * diagnostics storage, and switches userData to Map */ - public static final int FORMAT_DIAGNOSTICS = -9; - - /** Each segment records whether its postings are written - * in the new flex format */ - public static final int FORMAT_4_0 = -10; - - /* This must always point to the most recent file format. */ - public static final int CURRENT_FORMAT = FORMAT_4_0; - + public int counter = 0; // used to name new segments /** @@ -556,9 +546,16 @@ public final class SegmentInfos extends Vector { genB = gen0; break; } + } else { + /* TODO: Investigate this! + throw new IndexFormatTooNewException("segments.gen version number invalid: " + version + + " (must be " + FORMAT_SEGMENTS_GEN_CURRENT + ")"); + */ } } catch (IOException err2) { - // will retry + // rethrow any format exception + if (err2 instanceof CorruptIndexException) throw err2; + // else will retry } finally { genInput.close(); } diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java b/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java index c0c18d95e71..63201238e53 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java @@ -35,7 +35,11 @@ class TermVectorsReader implements Cloneable { static final int FORMAT_UTF8_LENGTH_IN_BYTES = 4; // NOTE: always change this if you switch to a new format! + // whenever you add a new format, make it 1 larger (positive version logic)! static final int FORMAT_CURRENT = FORMAT_UTF8_LENGTH_IN_BYTES; + + // when removing support for old versions, leave the last supported version here + static final int FORMAT_MINIMUM = FORMAT_UTF8_LENGTH_IN_BYTES; //The size in bytes that the FORMAT_VERSION will take up at the beginning of each file static final int FORMAT_SIZE = 4; @@ -75,11 +79,13 @@ class TermVectorsReader implements Cloneable { String idxName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION); if (d.fileExists(idxName)) { tvx = d.openInput(idxName, readBufferSize); - format = checkValidFormat(tvx); - tvd = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), readBufferSize); - final int tvdFormat = checkValidFormat(tvd); - tvf = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION), readBufferSize); - final int tvfFormat = checkValidFormat(tvf); + format = checkValidFormat(tvx, idxName); + String fn = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); + tvd = d.openInput(fn, readBufferSize); + final int tvdFormat = checkValidFormat(tvd, fn); + fn = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); + tvf = d.openInput(fn, readBufferSize); + final int tvfFormat = checkValidFormat(tvf, fn); assert format == tvdFormat; assert format == tvfFormat; @@ -183,13 +189,13 @@ class TermVectorsReader implements Cloneable { } } - private int checkValidFormat(IndexInput in) throws CorruptIndexException, IOException + private int checkValidFormat(IndexInput in, String fn) throws CorruptIndexException, IOException { int format = in.readInt(); - if (format > FORMAT_CURRENT) { - throw new CorruptIndexException("Incompatible format version: " + format + " expected " - + FORMAT_CURRENT + " or less"); - } + if (format < FORMAT_MINIMUM) + throw new IndexFormatTooOldException(fn, format, FORMAT_MINIMUM, FORMAT_CURRENT); + if (format > FORMAT_CURRENT) + throw new IndexFormatTooNewException(fn, format, FORMAT_MINIMUM, FORMAT_CURRENT); return format; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java index bb13615d3da..099e58762dc 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java @@ -20,6 +20,8 @@ package org.apache.lucene.index.codecs; import java.io.IOException; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexInput; @@ -41,8 +43,12 @@ public class DefaultSegmentInfosReader extends SegmentInfosReader { int format = input.readInt(); // check that it is a format we can understand - if (format < SegmentInfos.CURRENT_FORMAT) - throw new CorruptIndexException("Unknown (newer than us?) format version: " + format); + if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) + throw new IndexFormatTooOldException(segmentsFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) + throw new IndexFormatTooNewException(segmentsFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); infos.version = input.readLong(); // read version infos.counter = input.readInt(); // read counter diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java index ee71c93aa75..9f5fb58419f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java @@ -31,11 +31,26 @@ import org.apache.lucene.store.IndexOutput; */ public class DefaultSegmentInfosWriter extends SegmentInfosWriter { + /** This format adds optional per-segment String + * diagnostics storage, and switches userData to Map */ + public static final int FORMAT_DIAGNOSTICS = -9; + + /** Each segment records whether its postings are written + * in the new flex format */ + public static final int FORMAT_4_0 = -10; + + /** This must always point to the most recent file format. + * whenever you add a new format, make it 1 smaller (negative version logic)! */ + public static final int FORMAT_CURRENT = FORMAT_4_0; + + /** This must always point to the first supported file format. */ + public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; + @Override public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos) throws IOException { IndexOutput out = createOutput(dir, segmentFileName); - out.writeInt(SegmentInfos.CURRENT_FORMAT); // write FORMAT + out.writeInt(FORMAT_CURRENT); // write FORMAT out.writeLong(++infos.version); // every write changes // the index out.writeInt(infos.counter); // write counter diff --git a/lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java b/lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java index cb137ab6af1..d03fc49fbae 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java @@ -37,7 +37,8 @@ public class SimpleIntBlockIndexInput extends FixedIntBlockIndexInput { public SimpleIntBlockIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException { IndexInput in = dir.openInput(fileName, readBufferSize); - CodecUtil.checkHeader(in, SimpleIntBlockIndexOutput.CODEC, SimpleIntBlockIndexOutput.VERSION_START); + CodecUtil.checkHeader(in, SimpleIntBlockIndexOutput.CODEC, + SimpleIntBlockIndexOutput.VERSION_START, SimpleIntBlockIndexOutput.VERSION_START); init(in); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java index 85ba8849038..3c085aa0dda 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java @@ -22,6 +22,8 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Term; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.IndexFormatTooNewException; /** * @deprecated No longer used with flex indexing, except for @@ -40,7 +42,11 @@ public final class SegmentTermEnum implements Cloneable { public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; // NOTE: always change this if you switch to a new format! + // whenever you add a new format, make it 1 smaller (negative version logic)! public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; + + // when removing support for old versions, levae the last supported version here + public static final int FORMAT_MINIMUM = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; private TermBuffer termBuffer = new TermBuffer(); private TermBuffer prevBuffer = new TermBuffer(); @@ -78,8 +84,10 @@ public final class SegmentTermEnum implements Cloneable { format = firstInt; // check that it is a format we can understand - if (format < FORMAT_CURRENT) - throw new CorruptIndexException("Unknown format version:" + format + " expected " + FORMAT_CURRENT + " or higher"); + if (format > FORMAT_MINIMUM) + throw new IndexFormatTooOldException(null, format, FORMAT_MINIMUM, FORMAT_CURRENT); + if (format < FORMAT_CURRENT) + throw new IndexFormatTooNewException(null, format, FORMAT_MINIMUM, FORMAT_CURRENT); size = input.readLong(); // read the size diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java index c3ac53bdf6e..8422b7bf37c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java @@ -51,7 +51,8 @@ public class PulsingPostingsReaderImpl extends StandardPostingsReader { @Override public void init(IndexInput termsIn) throws IOException { - CodecUtil.checkHeader(termsIn, PulsingPostingsWriterImpl.CODEC, PulsingPostingsWriterImpl.VERSION_START); + CodecUtil.checkHeader(termsIn, PulsingPostingsWriterImpl.CODEC, + PulsingPostingsWriterImpl.VERSION_START, PulsingPostingsWriterImpl.VERSION_START); maxPulsingDocFreq = termsIn.readVInt(); wrappedPostingsReader.init(termsIn); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java index d70ffaf0ac4..8755362bcde 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java @@ -95,7 +95,8 @@ public class SepPostingsReaderImpl extends StandardPostingsReader { @Override public void init(IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, SepPostingsWriterImpl.CODEC, SepPostingsWriterImpl.VERSION_START); + CodecUtil.checkHeader(termsIn, SepPostingsWriterImpl.CODEC, + SepPostingsWriterImpl.VERSION_START, SepPostingsWriterImpl.VERSION_START); skipInterval = termsIn.readInt(); maxSkipLevels = termsIn.readInt(); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java index 482036b8876..4a639e3334a 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java @@ -36,7 +36,8 @@ public class SingleIntIndexInput extends IntIndexInput { public SingleIntIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException { in = dir.openInput(fileName, readBufferSize); - CodecUtil.checkHeader(in, SingleIntIndexOutput.CODEC, SingleIntIndexOutput.VERSION_START); + CodecUtil.checkHeader(in, SingleIntIndexOutput.CODEC, + SingleIntIndexOutput.VERSION_START, SingleIntIndexOutput.VERSION_START); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java index 86426bb1513..e409e4c2bb5 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java @@ -146,7 +146,8 @@ public class SimpleStandardTermsIndexReader extends StandardTermsIndexReader { } protected void readHeader(IndexInput input) throws IOException { - CodecUtil.checkHeader(input, SimpleStandardTermsIndexWriter.CODEC_NAME, SimpleStandardTermsIndexWriter.VERSION_START); + CodecUtil.checkHeader(input, SimpleStandardTermsIndexWriter.CODEC_NAME, + SimpleStandardTermsIndexWriter.VERSION_START, SimpleStandardTermsIndexWriter.VERSION_START); dirOffset = input.readLong(); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java index 4fb72eb6b54..2c53dc3bad2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java @@ -73,7 +73,8 @@ public class StandardPostingsReaderImpl extends StandardPostingsReader { public void init(IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, StandardPostingsWriterImpl.CODEC, StandardPostingsWriterImpl.VERSION_START); + CodecUtil.checkHeader(termsIn, StandardPostingsWriterImpl.CODEC, + StandardPostingsWriterImpl.VERSION_START, StandardPostingsWriterImpl.VERSION_START); skipInterval = termsIn.readInt(); maxSkipLevels = termsIn.readInt(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java index 3f6114ed0ea..2abbe55985f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java @@ -153,7 +153,8 @@ public class StandardTermsDictReader extends FieldsProducer { } protected void readHeader(IndexInput input) throws IOException { - CodecUtil.checkHeader(in, StandardTermsDictWriter.CODEC_NAME, StandardTermsDictWriter.VERSION_CURRENT); + CodecUtil.checkHeader(in, StandardTermsDictWriter.CODEC_NAME, + StandardTermsDictWriter.VERSION_START, StandardTermsDictWriter.VERSION_CURRENT); dirOffset = in.readLong(); } diff --git a/lucene/src/java/org/apache/lucene/util/CodecUtil.java b/lucene/src/java/org/apache/lucene/util/CodecUtil.java index 3c5e3e63534..5e4c1d87b88 100644 --- a/lucene/src/java/org/apache/lucene/util/CodecUtil.java +++ b/lucene/src/java/org/apache/lucene/util/CodecUtil.java @@ -21,6 +21,8 @@ package org.apache.lucene.util; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; import java.io.IOException; @@ -48,7 +50,7 @@ public final class CodecUtil { return 9+codec.length(); } - public static int checkHeader(IndexInput in, String codec, int maxVersion) + public static int checkHeader(IndexInput in, String codec, int minVersion, int maxVersion) throws IOException { // Safety to guard against reading a bogus string: @@ -63,8 +65,11 @@ public final class CodecUtil { } final int actualVersion = in.readInt(); + if (actualVersion < minVersion) { + throw new IndexFormatTooOldException(null, actualVersion, minVersion, maxVersion); + } if (actualVersion > maxVersion) { - throw new CorruptIndexException("version " + actualVersion + " is too new (expected <= version " + maxVersion + ")"); + throw new IndexFormatTooNewException(null, actualVersion, minVersion, maxVersion); } return actualVersion; diff --git a/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java b/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java index ac02e59a47c..5fa78c4fa9c 100644 --- a/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java +++ b/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java @@ -157,7 +157,7 @@ public class PackedInts { * @lucene.internal */ public static Reader getReader(IndexInput in) throws IOException { - CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START); + CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); @@ -188,7 +188,7 @@ public class PackedInts { * @lucene.internal */ public static ReaderIterator getReaderIterator(IndexInput in) throws IOException { - CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START); + CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); diff --git a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 56982554520..63c4e42e74d 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -23,6 +23,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.util.Arrays; import java.util.Enumeration; import java.util.List; @@ -127,6 +129,74 @@ public class TestBackwardsCompatibility extends LuceneTestCase { "31.nocfs", }; + final String[] unsupportedNames = {"19.cfs", + "19.nocfs", + "20.cfs", + "20.nocfs", + "21.cfs", + "21.nocfs", + "22.cfs", + "22.nocfs", + "23.cfs", + "23.nocfs", + "24.cfs", + "24.nocfs", + "29.cfs", + "29.nocfs", + }; + + /** This test checks that *only* IndexFormatTooOldExceptions are throws when you open and operate on too old indexes! */ + public void testUnsupportedOldIndexes() throws Exception { + for(int i=0;i3.0 stored field change + fail("IndexReader.open should not pass for "+unsupportedNames[i]); + } catch (IndexFormatTooOldException e) { + // pass + } finally { + if (reader != null) reader.close(); + reader = null; + } + + try { + writer = new IndexWriter(dir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer()) + .setMergeScheduler(new SerialMergeScheduler()) // no threads! + ); + writer.optimize(); + reader = writer.getReader(); + reader.document(0); // to catch also 2.9->3.0 stored field change + fail("IndexWriter creation should not pass for "+unsupportedNames[i]); + } catch (IndexFormatTooOldException e) { + // pass + } finally { + if (reader != null) reader.close(); + reader = null; + if (writer != null) writer.close(); + writer = null; + } + + ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); + CheckIndex checker = new CheckIndex(dir); + checker.setInfoStream(new PrintStream(bos)); + CheckIndex.Status indexStatus = checker.checkIndex(); + assertFalse(indexStatus.clean); + assertTrue(bos.toString().contains(IndexFormatTooOldException.class.getName())); + + dir.close(); + rmDir(unsupportedNames[i]); + } + } + public void testOptimizeOldIndex() throws Exception { for(int i=0;i