Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CheckIndex failed to say it was checking points

This commit is contained in:
Mike McCandless 2016-03-09 11:03:19 -05:00
parent f1ad769666
commit dacbf333e4
7 changed files with 48 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -1683,9 +1683,14 @@ public final class CheckIndex implements Closeable {
* @lucene.experimental
*/
public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
if (infoStream != null) {
infoStream.print(" test: points..............");
}
long startNS = System.nanoTime();
FieldInfos fieldInfos = reader.getFieldInfos();
Status.PointsStatus status = new Status.PointsStatus();
try {
if (fieldInfos.hasPointValues()) {
PointsReader values = reader.getPointsReader();
if (values == null) {
@ -1840,6 +1845,8 @@ public final class CheckIndex implements Closeable {
}
}
}
msg(infoStream, String.format(Locale.ROOT, "OK [%d fields, %d points] [took %.3f sec]", status.totalValueFields, status.totalValuePoints, nsToSec(System.nanoTime()-startNS)));
} catch (Throwable e) {
if (failFast) {
IOUtils.reThrow(e);

3
lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
View File

@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
@ -132,7 +131,7 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase {
// CheckIndex should also fail:
try {
TestUtil.checkIndex(dirCopy, true, true);
TestUtil.checkIndex(dirCopy, true, true, null);
fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
} catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
// expected

4
lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.EOFException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import org.apache.lucene.analysis.MockAnalyzer;
@ -28,7 +27,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
@ -116,7 +114,7 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
// CheckIndex should also fail:
try {
TestUtil.checkIndex(dirCopy, true, true);
TestUtil.checkIndex(dirCopy, true, true, null);
fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
} catch (CorruptIndexException | EOFException e) {
// expected

29
lucene/core/src/test/org/apache/lucene/index/TestPointValues.java
View File

@ -17,6 +17,7 @@
package org.apache.lucene.index;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
@ -40,6 +41,7 @@ import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -628,4 +630,31 @@ public class TestPointValues extends LuceneTestCase {
w.close();
dir.close();
}
public void testCheckIndexIncludesPoints() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
Document doc = new Document();
doc.add(new IntPoint("int1", 17));
w.addDocument(doc);
doc = new Document();
doc.add(new IntPoint("int1", 44));
doc.add(new IntPoint("int2", -17));
w.addDocument(doc);
w.close();
ByteArrayOutputStream output = new ByteArrayOutputStream();
CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, output);
assertEquals(1, status.segmentInfos.size());
CheckIndex.Status.SegmentInfoStatus segStatus = status.segmentInfos.get(0);
// total 3 point values were index:
assertEquals(3, segStatus.pointsStatus.totalValuePoints);
// ... across 2 fields:
assertEquals(2, segStatus.pointsStatus.totalValueFields);
// Make sure CheckIndex in fact declares that it is testing points!
assertTrue(output.toString(IOUtils.UTF_8).contains("test: points..."));
dir.close();
}
}

3
lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
View File

@ -28,7 +28,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
@ -118,7 +117,7 @@ public class TestSwappedIndexFiles extends LuceneTestCase {
// CheckIndex should also fail:
try {
TestUtil.checkIndex(dirCopy, true, true);
TestUtil.checkIndex(dirCopy, true, true, null);
fail("wrong file " + victim + " not detected");
} catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
// expected

2
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -850,7 +850,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true);
TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, null);
}
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles

14
lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
View File

@ -279,28 +279,30 @@ public final class TestUtil {
}
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException {
return checkIndex(dir, crossCheckTermVectors, false);
return checkIndex(dir, crossCheckTermVectors, false, null);
}
/** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to
* look for any other corruption. */
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast, ByteArrayOutputStream output) throws IOException {
if (output == null) {
output = new ByteArrayOutputStream(1024);
}
// TODO: actually use the dir's locking, unless test uses a special method?
// some tests e.g. exception tests become much more complicated if they have to close the writer
try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) {
checker.setCrossCheckTermVectors(crossCheckTermVectors);
checker.setFailFast(failFast);
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
checker.setInfoStream(new PrintStream(output, false, IOUtils.UTF_8), false);
CheckIndex.Status indexStatus = checker.checkIndex(null);
if (indexStatus == null || indexStatus.clean == false) {
System.out.println("CheckIndex failed");
System.out.println(bos.toString(IOUtils.UTF_8));
System.out.println(output.toString(IOUtils.UTF_8));
throw new RuntimeException("CheckIndex failed");
} else {
if (LuceneTestCase.INFOSTREAM) {
System.out.println(bos.toString(IOUtils.UTF_8));
System.out.println(output.toString(IOUtils.UTF_8));
}
return indexStatus;
}