From b9e204c07d6499256086422abd769235e4e1cda5 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 9 Mar 2016 11:03:19 -0500 Subject: [PATCH] CheckIndex failed to say it was checking points --- .../org/apache/lucene/index/CheckIndex.java | 7 +++++ .../index/TestAllFilesCheckIndexHeader.java | 3 +- .../index/TestAllFilesDetectTruncation.java | 4 +-- .../apache/lucene/index/TestPointValues.java | 29 +++++++++++++++++++ .../lucene/index/TestSwappedIndexFiles.java | 3 +- .../lucene/store/MockDirectoryWrapper.java | 2 +- .../java/org/apache/lucene/util/TestUtil.java | 14 +++++---- 7 files changed, 48 insertions(+), 14 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index 3c437c1c508..89b36efa2d3 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -1683,9 +1683,14 @@ public final class CheckIndex implements Closeable { * @lucene.experimental */ public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { + if (infoStream != null) { + infoStream.print(" test: points.............."); + } + long startNS = System.nanoTime(); FieldInfos fieldInfos = reader.getFieldInfos(); Status.PointsStatus status = new Status.PointsStatus(); try { + if (fieldInfos.hasPointValues()) { PointsReader values = reader.getPointsReader(); if (values == null) { @@ -1840,6 +1845,8 @@ public final class CheckIndex implements Closeable { } } } + msg(infoStream, String.format(Locale.ROOT, "OK [%d fields, %d points] [took %.3f sec]", status.totalValueFields, status.totalValuePoints, nsToSec(System.nanoTime()-startNS))); + } catch (Throwable e) { if (failFast) { IOUtils.reThrow(e); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java index 68b7cc23ea5..f6c1486bc43 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -132,7 +131,7 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase { // CheckIndex should also fail: try { - TestUtil.checkIndex(dirCopy, true, true); + TestUtil.checkIndex(dirCopy, true, true, null); fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim); } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { // expected diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java index 16caae3baa5..c751417cd74 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java @@ -19,7 +19,6 @@ package org.apache.lucene.index; import java.io.EOFException; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import org.apache.lucene.analysis.MockAnalyzer; @@ -28,7 +27,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -116,7 +114,7 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase { // CheckIndex should also fail: try { - TestUtil.checkIndex(dirCopy, true, true); + TestUtil.checkIndex(dirCopy, true, true, null); fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim); } catch (CorruptIndexException | EOFException e) { // expected diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 55d47944d30..9b18f0298ee 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import java.io.ByteArrayOutputStream; import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; @@ -40,6 +41,7 @@ import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.PointValues; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -628,4 +630,31 @@ public class TestPointValues extends LuceneTestCase { w.close(); dir.close(); } + + public void testCheckIndexIncludesPoints() throws Exception { + Directory dir = new RAMDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); + Document doc = new Document(); + doc.add(new IntPoint("int1", 17)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new IntPoint("int1", 44)); + doc.add(new IntPoint("int2", -17)); + w.addDocument(doc); + w.close(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, output); + assertEquals(1, status.segmentInfos.size()); + CheckIndex.Status.SegmentInfoStatus segStatus = status.segmentInfos.get(0); + // total 3 point values were index: + assertEquals(3, segStatus.pointsStatus.totalValuePoints); + // ... across 2 fields: + assertEquals(2, segStatus.pointsStatus.totalValueFields); + + // Make sure CheckIndex in fact declares that it is testing points! + assertTrue(output.toString(IOUtils.UTF_8).contains("test: points...")); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java index a284fdd9d10..d2205020d99 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java @@ -28,7 +28,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -118,7 +117,7 @@ public class TestSwappedIndexFiles extends LuceneTestCase { // CheckIndex should also fail: try { - TestUtil.checkIndex(dirCopy, true, true); + TestUtil.checkIndex(dirCopy, true, true, null); fail("wrong file " + victim + " not detected"); } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { // expected diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 7fe7c3b0d5a..c2544b4ffba 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -850,7 +850,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); } - TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true); + TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, null); } // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index 5e328ba1366..d772ae321d3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -279,28 +279,30 @@ public final class TestUtil { } public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException { - return checkIndex(dir, crossCheckTermVectors, false); + return checkIndex(dir, crossCheckTermVectors, false, null); } /** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to * look for any other corruption. */ - public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); + public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast, ByteArrayOutputStream output) throws IOException { + if (output == null) { + output = new ByteArrayOutputStream(1024); + } // TODO: actually use the dir's locking, unless test uses a special method? // some tests e.g. exception tests become much more complicated if they have to close the writer try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) { checker.setCrossCheckTermVectors(crossCheckTermVectors); checker.setFailFast(failFast); - checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false); + checker.setInfoStream(new PrintStream(output, false, IOUtils.UTF_8), false); CheckIndex.Status indexStatus = checker.checkIndex(null); if (indexStatus == null || indexStatus.clean == false) { System.out.println("CheckIndex failed"); - System.out.println(bos.toString(IOUtils.UTF_8)); + System.out.println(output.toString(IOUtils.UTF_8)); throw new RuntimeException("CheckIndex failed"); } else { if (LuceneTestCase.INFOSTREAM) { - System.out.println(bos.toString(IOUtils.UTF_8)); + System.out.println(output.toString(IOUtils.UTF_8)); } return indexStatus; }