mirror of https://github.com/apache/lucene.git
CheckIndex failed to say it was checking points
This commit is contained in:
parent
ba0f63c0c2
commit
b9e204c07d
|
@ -1683,9 +1683,14 @@ public final class CheckIndex implements Closeable {
|
||||||
* @lucene.experimental
|
* @lucene.experimental
|
||||||
*/
|
*/
|
||||||
public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
|
public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
|
||||||
|
if (infoStream != null) {
|
||||||
|
infoStream.print(" test: points..............");
|
||||||
|
}
|
||||||
|
long startNS = System.nanoTime();
|
||||||
FieldInfos fieldInfos = reader.getFieldInfos();
|
FieldInfos fieldInfos = reader.getFieldInfos();
|
||||||
Status.PointsStatus status = new Status.PointsStatus();
|
Status.PointsStatus status = new Status.PointsStatus();
|
||||||
try {
|
try {
|
||||||
|
|
||||||
if (fieldInfos.hasPointValues()) {
|
if (fieldInfos.hasPointValues()) {
|
||||||
PointsReader values = reader.getPointsReader();
|
PointsReader values = reader.getPointsReader();
|
||||||
if (values == null) {
|
if (values == null) {
|
||||||
|
@ -1840,6 +1845,8 @@ public final class CheckIndex implements Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
msg(infoStream, String.format(Locale.ROOT, "OK [%d fields, %d points] [took %.3f sec]", status.totalValueFields, status.totalValuePoints, nsToSec(System.nanoTime()-startNS)));
|
||||||
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
if (failFast) {
|
if (failFast) {
|
||||||
IOUtils.reThrow(e);
|
IOUtils.reThrow(e);
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.IOContext;
|
import org.apache.lucene.store.IOContext;
|
||||||
import org.apache.lucene.store.IndexInput;
|
import org.apache.lucene.store.IndexInput;
|
||||||
import org.apache.lucene.store.IndexOutput;
|
import org.apache.lucene.store.IndexOutput;
|
||||||
import org.apache.lucene.store.MockDirectoryWrapper;
|
|
||||||
import org.apache.lucene.util.LineFileDocs;
|
import org.apache.lucene.util.LineFileDocs;
|
||||||
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
|
@ -132,7 +131,7 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase {
|
||||||
|
|
||||||
// CheckIndex should also fail:
|
// CheckIndex should also fail:
|
||||||
try {
|
try {
|
||||||
TestUtil.checkIndex(dirCopy, true, true);
|
TestUtil.checkIndex(dirCopy, true, true, null);
|
||||||
fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
|
fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
|
||||||
} catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
|
} catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
|
||||||
// expected
|
// expected
|
||||||
|
|
|
@ -19,7 +19,6 @@ package org.apache.lucene.index;
|
||||||
|
|
||||||
import java.io.EOFException;
|
import java.io.EOFException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
|
||||||
import org.apache.lucene.analysis.MockAnalyzer;
|
import org.apache.lucene.analysis.MockAnalyzer;
|
||||||
|
@ -28,7 +27,6 @@ import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.IOContext;
|
import org.apache.lucene.store.IOContext;
|
||||||
import org.apache.lucene.store.IndexInput;
|
import org.apache.lucene.store.IndexInput;
|
||||||
import org.apache.lucene.store.IndexOutput;
|
import org.apache.lucene.store.IndexOutput;
|
||||||
import org.apache.lucene.store.MockDirectoryWrapper;
|
|
||||||
import org.apache.lucene.util.LineFileDocs;
|
import org.apache.lucene.util.LineFileDocs;
|
||||||
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
|
@ -116,7 +114,7 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
|
||||||
|
|
||||||
// CheckIndex should also fail:
|
// CheckIndex should also fail:
|
||||||
try {
|
try {
|
||||||
TestUtil.checkIndex(dirCopy, true, true);
|
TestUtil.checkIndex(dirCopy, true, true, null);
|
||||||
fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
|
fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
|
||||||
} catch (CorruptIndexException | EOFException e) {
|
} catch (CorruptIndexException | EOFException e) {
|
||||||
// expected
|
// expected
|
||||||
|
|
|
@ -17,6 +17,7 @@
|
||||||
package org.apache.lucene.index;
|
package org.apache.lucene.index;
|
||||||
|
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.apache.lucene.analysis.MockAnalyzer;
|
import org.apache.lucene.analysis.MockAnalyzer;
|
||||||
|
@ -40,6 +41,7 @@ import org.apache.lucene.index.PointValues.Relation;
|
||||||
import org.apache.lucene.index.PointValues;
|
import org.apache.lucene.index.PointValues;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.FSDirectory;
|
import org.apache.lucene.store.FSDirectory;
|
||||||
|
import org.apache.lucene.store.RAMDirectory;
|
||||||
import org.apache.lucene.util.IOUtils;
|
import org.apache.lucene.util.IOUtils;
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
import org.apache.lucene.util.TestUtil;
|
import org.apache.lucene.util.TestUtil;
|
||||||
|
@ -628,4 +630,31 @@ public class TestPointValues extends LuceneTestCase {
|
||||||
w.close();
|
w.close();
|
||||||
dir.close();
|
dir.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testCheckIndexIncludesPoints() throws Exception {
|
||||||
|
Directory dir = new RAMDirectory();
|
||||||
|
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
|
||||||
|
Document doc = new Document();
|
||||||
|
doc.add(new IntPoint("int1", 17));
|
||||||
|
w.addDocument(doc);
|
||||||
|
|
||||||
|
doc = new Document();
|
||||||
|
doc.add(new IntPoint("int1", 44));
|
||||||
|
doc.add(new IntPoint("int2", -17));
|
||||||
|
w.addDocument(doc);
|
||||||
|
w.close();
|
||||||
|
|
||||||
|
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||||
|
CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, output);
|
||||||
|
assertEquals(1, status.segmentInfos.size());
|
||||||
|
CheckIndex.Status.SegmentInfoStatus segStatus = status.segmentInfos.get(0);
|
||||||
|
// total 3 point values were index:
|
||||||
|
assertEquals(3, segStatus.pointsStatus.totalValuePoints);
|
||||||
|
// ... across 2 fields:
|
||||||
|
assertEquals(2, segStatus.pointsStatus.totalValueFields);
|
||||||
|
|
||||||
|
// Make sure CheckIndex in fact declares that it is testing points!
|
||||||
|
assertTrue(output.toString(IOUtils.UTF_8).contains("test: points..."));
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.store.BaseDirectoryWrapper;
|
import org.apache.lucene.store.BaseDirectoryWrapper;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.IOContext;
|
import org.apache.lucene.store.IOContext;
|
||||||
import org.apache.lucene.store.MockDirectoryWrapper;
|
|
||||||
import org.apache.lucene.util.LineFileDocs;
|
import org.apache.lucene.util.LineFileDocs;
|
||||||
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
|
@ -118,7 +117,7 @@ public class TestSwappedIndexFiles extends LuceneTestCase {
|
||||||
|
|
||||||
// CheckIndex should also fail:
|
// CheckIndex should also fail:
|
||||||
try {
|
try {
|
||||||
TestUtil.checkIndex(dirCopy, true, true);
|
TestUtil.checkIndex(dirCopy, true, true, null);
|
||||||
fail("wrong file " + victim + " not detected");
|
fail("wrong file " + victim + " not detected");
|
||||||
} catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
|
} catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
|
||||||
// expected
|
// expected
|
||||||
|
|
|
@ -850,7 +850,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
|
||||||
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
|
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
|
||||||
}
|
}
|
||||||
|
|
||||||
TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true);
|
TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
|
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
|
||||||
|
|
|
@ -279,28 +279,30 @@ public final class TestUtil {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException {
|
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException {
|
||||||
return checkIndex(dir, crossCheckTermVectors, false);
|
return checkIndex(dir, crossCheckTermVectors, false, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to
|
/** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to
|
||||||
* look for any other corruption. */
|
* look for any other corruption. */
|
||||||
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException {
|
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast, ByteArrayOutputStream output) throws IOException {
|
||||||
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
|
if (output == null) {
|
||||||
|
output = new ByteArrayOutputStream(1024);
|
||||||
|
}
|
||||||
// TODO: actually use the dir's locking, unless test uses a special method?
|
// TODO: actually use the dir's locking, unless test uses a special method?
|
||||||
// some tests e.g. exception tests become much more complicated if they have to close the writer
|
// some tests e.g. exception tests become much more complicated if they have to close the writer
|
||||||
try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) {
|
try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) {
|
||||||
checker.setCrossCheckTermVectors(crossCheckTermVectors);
|
checker.setCrossCheckTermVectors(crossCheckTermVectors);
|
||||||
checker.setFailFast(failFast);
|
checker.setFailFast(failFast);
|
||||||
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
|
checker.setInfoStream(new PrintStream(output, false, IOUtils.UTF_8), false);
|
||||||
CheckIndex.Status indexStatus = checker.checkIndex(null);
|
CheckIndex.Status indexStatus = checker.checkIndex(null);
|
||||||
|
|
||||||
if (indexStatus == null || indexStatus.clean == false) {
|
if (indexStatus == null || indexStatus.clean == false) {
|
||||||
System.out.println("CheckIndex failed");
|
System.out.println("CheckIndex failed");
|
||||||
System.out.println(bos.toString(IOUtils.UTF_8));
|
System.out.println(output.toString(IOUtils.UTF_8));
|
||||||
throw new RuntimeException("CheckIndex failed");
|
throw new RuntimeException("CheckIndex failed");
|
||||||
} else {
|
} else {
|
||||||
if (LuceneTestCase.INFOSTREAM) {
|
if (LuceneTestCase.INFOSTREAM) {
|
||||||
System.out.println(bos.toString(IOUtils.UTF_8));
|
System.out.println(output.toString(IOUtils.UTF_8));
|
||||||
}
|
}
|
||||||
return indexStatus;
|
return indexStatus;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue