diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index e4c01252f54..b0a53f9e2ce 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -65,6 +65,8 @@ API Changes IndexSearcher#search(Query, CollectorManager) for TopFieldCollectorManager and TopScoreDocCollectorManager. (Zach Chen, Adrien Grand, Michael McCandless, Greg Miller, Luca Cavanna) +* GITHUB#11023: Adding -level param to CheckIndex, making the old -fast param the default behaviour. (Jakub Slowinski) + New Features --------------------- diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md index 158ce93c9a7..97fd943926f 100644 --- a/lucene/MIGRATE.md +++ b/lucene/MIGRATE.md @@ -101,6 +101,13 @@ The deprecated getter for the `Executor` that was optionally provided to the `In has been removed. Users that want to execute concurrent tasks should rely instead on the `TaskExecutor` that the searcher holds, retrieved via `IndexSearcher#getTaskExecutor`. +### CheckIndex params -slow and -fast are deprecated, replaced by -level X (GITHUB#11023) + +The `CheckIndex` former `-fast` behaviour of performing checksum checks only, is now the default. +Added a new parameter: `-level X`, to set the detail level of the index check. The higher the value, the more checks are performed. +Sample `-level` usage: `1` (Default) - Checksum checks only, `2` - all level 1 checks as well as logical integrity checks, `3` - all +level 2 checks as well as slow checks. + ## Migration from Lucene 9.0 to Lucene 9.1 ### Test framework package migration and module (LUCENE-10301) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestManyPointsInOldIndex.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestManyPointsInOldIndex.java index 0a7571bcc2f..7cfe57ddba6 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestManyPointsInOldIndex.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestManyPointsInOldIndex.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import org.apache.lucene.document.Document; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.Directory; @@ -70,7 +71,7 @@ public class TestManyPointsInOldIndex extends LuceneTestCase { dir.setCheckIndexOnClose(false); // ... because we check ourselves here: - TestUtil.checkIndex(dir, false, true, true, null); + TestUtil.checkIndex(dir, CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS, true, true, null); dir.close(); } } diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index 6f191b944c2..f15941c6e3a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -442,19 +442,20 @@ public final class CheckIndex implements Closeable { IOUtils.close(writeLock); } - private boolean doSlowChecks; + private int level; /** - * If true, additional slow checks are performed. This will likely drastically increase time it - * takes to run CheckIndex! + * Sets Level, the higher the value, the more additional checks are performed. This will likely + * drastically increase time it takes to run CheckIndex! See {@link Level} */ - public void setDoSlowChecks(boolean v) { - doSlowChecks = v; + public void setLevel(int v) { + Level.checkIfLevelInBounds(v); + level = v; } - /** See {@link #setDoSlowChecks}. */ - public boolean doSlowChecks() { - return doSlowChecks; + /** See {@link #setLevel}. */ + public int getLevel() { + return level; } private boolean failFast; @@ -474,21 +475,6 @@ public final class CheckIndex implements Closeable { private boolean verbose; - /** See {@link #getChecksumsOnly}. */ - public boolean getChecksumsOnly() { - return checksumsOnly; - } - - /** - * If true, only validate physical integrity for all files. Note that the returned nested status - * objects (e.g. storedFieldStatus) will be null. - */ - public void setChecksumsOnly(boolean v) { - checksumsOnly = v; - } - - private boolean checksumsOnly; - /** Set threadCount used for parallelizing index integrity checking. */ public void setThreadCount(int tc) { if (tc <= 0) { @@ -1065,8 +1051,7 @@ public final class CheckIndex implements Closeable { + (info.info.maxDoc() - reader.numDocs())); } } - - if (checksumsOnly == false) { + if (level >= Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS) { // Test Livedocs segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream, failFast); @@ -1077,15 +1062,14 @@ public final class CheckIndex implements Closeable { segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast); // Test the Term Index - segInfoStat.termIndexStatus = - testPostings(reader, infoStream, verbose, doSlowChecks, failFast); + segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, level, failFast); // Test Stored Fields segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast); // Test Term Vectors segInfoStat.termVectorStatus = - testTermVectors(reader, infoStream, verbose, doSlowChecks, failFast); + testTermVectors(reader, infoStream, verbose, level, failFast); // Test Docvalues segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast); @@ -1416,7 +1400,7 @@ public final class CheckIndex implements Closeable { boolean isVectors, PrintStream infoStream, boolean verbose, - boolean doSlowChecks) + int level) throws IOException { // TODO: we should probably return our own stats thing...?! long startNS; @@ -1999,14 +1983,13 @@ public final class CheckIndex implements Closeable { } // Checking score blocks is heavy, we only do it on long postings lists, on every 1024th - // term - // or if slow checks are enabled. - if (doSlowChecks + // term or if slow checks are enabled. + if (level >= Level.MIN_LEVEL_FOR_SLOW_CHECKS || docFreq > 1024 || (status.termCount + status.delTermCount) % 1024 == 0) { // First check max scores and block uptos // But only if slok checks are enabled since we visit all docs - if (doSlowChecks) { + if (level >= Level.MIN_LEVEL_FOR_SLOW_CHECKS) { int max = -1; int maxFreq = 0; ImpactsEnum impactsEnum = termsEnum.impacts(PostingsEnum.FREQS); @@ -2073,9 +2056,9 @@ public final class CheckIndex implements Closeable { Impacts impacts = impactsEnum.getImpacts(); checkImpacts(impacts, doc); maxFreq = Integer.MAX_VALUE; - for (int level = 0; level < impacts.numLevels(); ++level) { - if (impacts.getDocIdUpTo(level) >= max) { - List perLevelImpacts = impacts.getImpacts(level); + for (int impactsLevel = 0; impactsLevel < impacts.numLevels(); ++impactsLevel) { + if (impacts.getDocIdUpTo(impactsLevel) >= max) { + List perLevelImpacts = impacts.getImpacts(impactsLevel); maxFreq = perLevelImpacts.get(perLevelImpacts.size() - 1).freq; break; } @@ -2115,9 +2098,9 @@ public final class CheckIndex implements Closeable { Impacts impacts = impactsEnum.getImpacts(); checkImpacts(impacts, doc); maxFreq = Integer.MAX_VALUE; - for (int level = 0; level < impacts.numLevels(); ++level) { - if (impacts.getDocIdUpTo(level) >= max) { - List perLevelImpacts = impacts.getImpacts(level); + for (int impactsLevel = 0; impactsLevel < impacts.numLevels(); ++impactsLevel) { + if (impacts.getDocIdUpTo(impactsLevel) >= max) { + List perLevelImpacts = impacts.getImpacts(impactsLevel); maxFreq = perLevelImpacts.get(perLevelImpacts.size() - 1).freq; break; } @@ -2382,7 +2365,7 @@ public final class CheckIndex implements Closeable { static void checkImpacts(Impacts impacts, int lastTarget) { final int numLevels = impacts.numLevels(); if (numLevels < 1) { - throw new CheckIndexException("The number of levels must be >= 1, got " + numLevels); + throw new CheckIndexException("The number of impact levels must be >= 1, got " + numLevels); } int docIdUpTo0 = impacts.getDocIdUpTo(0); @@ -2394,17 +2377,17 @@ public final class CheckIndex implements Closeable { + lastTarget); } - for (int level = 1; level < numLevels; ++level) { - int docIdUpTo = impacts.getDocIdUpTo(level); - int previousDocIdUpTo = impacts.getDocIdUpTo(level - 1); + for (int impactsLevel = 1; impactsLevel < numLevels; ++impactsLevel) { + int docIdUpTo = impacts.getDocIdUpTo(impactsLevel); + int previousDocIdUpTo = impacts.getDocIdUpTo(impactsLevel - 1); if (docIdUpTo < previousDocIdUpTo) { throw new CheckIndexException( "Decreasing return for getDocIdUpTo: level " - + (level - 1) + + (impactsLevel - 1) + " returned " + previousDocIdUpTo + " but level " - + level + + impactsLevel + " returned " + docIdUpTo + " for target " @@ -2412,10 +2395,10 @@ public final class CheckIndex implements Closeable { } } - for (int level = 0; level < numLevels; ++level) { - List perLevelImpacts = impacts.getImpacts(level); + for (int impactsLevel = 0; impactsLevel < numLevels; ++impactsLevel) { + List perLevelImpacts = impacts.getImpacts(impactsLevel); if (perLevelImpacts.isEmpty()) { - throw new CheckIndexException("Got empty list of impacts on level " + level); + throw new CheckIndexException("Got empty list of impacts on level " + impactsLevel); } Impact first = perLevelImpacts.get(0); if (first.freq < 1) { @@ -2433,9 +2416,9 @@ public final class CheckIndex implements Closeable { "Impacts are not ordered or contain dups, got " + previous + " then " + impact); } } - if (level > 0) { - // Make sure that impacts at level N trigger better scores than an level N-1 - Iterator previousIt = impacts.getImpacts(level - 1).iterator(); + if (impactsLevel > 0) { + // Make sure that impacts at level N trigger better scores than an impactsLevel N-1 + Iterator previousIt = impacts.getImpacts(impactsLevel - 1).iterator(); previous = previousIt.next(); Iterator it = perLevelImpacts.iterator(); Impact impact = it.next(); @@ -2451,9 +2434,9 @@ public final class CheckIndex implements Closeable { "Found impact " + previous + " on level " - + (level - 1) + + (impactsLevel - 1) + " but no impact on level " - + level + + impactsLevel + " triggers a better score: " + perLevelImpacts); } @@ -2470,7 +2453,7 @@ public final class CheckIndex implements Closeable { */ public static Status.TermIndexStatus testPostings(CodecReader reader, PrintStream infoStream) throws IOException { - return testPostings(reader, infoStream, false, true, false); + return testPostings(reader, infoStream, false, Level.MIN_LEVEL_FOR_SLOW_CHECKS, false); } /** @@ -2479,15 +2462,11 @@ public final class CheckIndex implements Closeable { * @lucene.experimental */ public static Status.TermIndexStatus testPostings( - CodecReader reader, - PrintStream infoStream, - boolean verbose, - boolean doSlowChecks, - boolean failFast) + CodecReader reader, PrintStream infoStream, boolean verbose, int level, boolean failFast) throws IOException { - // TODO: we should go and verify term vectors match, if - // doSlowChecks is on... + // TODO: we should go and verify term vectors match, if the Level is high enough to + // include slow checks Status.TermIndexStatus status; final int maxDoc = reader.maxDoc(); @@ -2518,7 +2497,7 @@ public final class CheckIndex implements Closeable { false, infoStream, verbose, - doSlowChecks); + level); } catch (Throwable e) { if (failFast) { throw IOUtils.rethrowAlways(e); @@ -3661,7 +3640,7 @@ public final class CheckIndex implements Closeable { */ public static Status.TermVectorStatus testTermVectors(CodecReader reader, PrintStream infoStream) throws IOException { - return testTermVectors(reader, infoStream, false, false, false); + return testTermVectors(reader, infoStream, false, Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS, false); } /** @@ -3670,11 +3649,7 @@ public final class CheckIndex implements Closeable { * @lucene.experimental */ public static Status.TermVectorStatus testTermVectors( - CodecReader reader, - PrintStream infoStream, - boolean verbose, - boolean doSlowChecks, - boolean failFast) + CodecReader reader, PrintStream infoStream, boolean verbose, int level, boolean failFast) throws IOException { long startNS = System.nanoTime(); final Status.TermVectorStatus status = new Status.TermVectorStatus(); @@ -3687,14 +3662,14 @@ public final class CheckIndex implements Closeable { PostingsEnum postings = null; - // Only used if doSlowChecks is true: + // Only used if the Level is high enough to include slow checks: PostingsEnum postingsDocs = null; final Bits liveDocs = reader.getLiveDocs(); FieldsProducer postingsFields; // TODO: testTermsIndex - if (doSlowChecks) { + if (level >= Level.MIN_LEVEL_FOR_SLOW_CHECKS) { postingsFields = reader.getPostingsReader(); if (postingsFields != null) { postingsFields = postingsFields.getMergeInstance(); @@ -3718,8 +3693,7 @@ public final class CheckIndex implements Closeable { if (tfv != null) { // First run with no deletions: - checkFields( - tfv, null, 1, fieldInfos, null, false, true, infoStream, verbose, doSlowChecks); + checkFields(tfv, null, 1, fieldInfos, null, false, true, infoStream, verbose, level); // Only agg stats if the doc is live: final boolean doStats = liveDocs == null || liveDocs.get(j); @@ -3744,7 +3718,7 @@ public final class CheckIndex implements Closeable { + " but FieldInfo has storeTermVector=false"); } - if (doSlowChecks) { + if (level >= Level.MIN_LEVEL_FOR_SLOW_CHECKS) { Terms terms = tfv.terms(field); TermsEnum termsEnum = terms.iterator(); final boolean postingsHasFreq = @@ -4047,9 +4021,8 @@ public final class CheckIndex implements Closeable { /** Run-time configuration options for CheckIndex commands. */ public static class Options { boolean doExorcise = false; - boolean doSlowChecks = false; boolean verbose = false; - boolean doChecksumsOnly = false; + int level = Level.DEFAULT_VALUE; int threadCount; List onlySegments = new ArrayList<>(); String indexPath = null; @@ -4113,6 +4086,42 @@ public final class CheckIndex implements Closeable { } } + /** Class with static variables with information about CheckIndex's -level parameter. */ + public static class Level { + private Level() {} + + /** Minimum valid level. */ + public static final int MIN_VALUE = 1; + + /** Maximum valid level. */ + public static final int MAX_VALUE = 3; + + /** The default level if none is specified. */ + public static final int DEFAULT_VALUE = MIN_VALUE; + + /** Minimum level required to run checksum checks. */ + public static final int MIN_LEVEL_FOR_CHECKSUM_CHECKS = 1; + + /** Minimum level required to run integrity checks. */ + public static final int MIN_LEVEL_FOR_INTEGRITY_CHECKS = 2; + + /** Minimum level required to run slow checks. */ + public static final int MIN_LEVEL_FOR_SLOW_CHECKS = 3; + + /** Checks if given level value is within the allowed bounds else it raises an Exception. */ + public static void checkIfLevelInBounds(int levelVal) throws IllegalArgumentException { + if (levelVal < Level.MIN_VALUE || levelVal > Level.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "ERROR: given value: '%d' for -level option is out of bounds. Please use a value from '%d'->'%d'", + levelVal, + Level.MIN_VALUE, + Level.MAX_VALUE)); + } + } + } + /** * Parse command line args into fields * @@ -4127,15 +4136,29 @@ public final class CheckIndex implements Closeable { int i = 0; while (i < args.length) { String arg = args[i]; - if ("-fast".equals(arg)) { - opts.doChecksumsOnly = true; + if ("-level".equals(arg)) { + if (i == args.length - 1) { + throw new IllegalArgumentException("ERROR: missing value for -level option"); + } + i++; + int level = Integer.parseInt(args[i]); + Level.checkIfLevelInBounds(level); + opts.level = level; + } else if ("-fast".equals(arg)) { + // Deprecated. Remove in Lucene 11. + System.err.println( + "-fast is deprecated, use '-level 1' for explicitly verifying file checksums only. This is also now the default " + + "behaviour!"); + } else if ("-slow".equals(arg)) { + // Deprecated. Remove in Lucene 11. + System.err.println("-slow is deprecated, use '-level 3' instead for slow checks"); + opts.level = Level.MIN_LEVEL_FOR_SLOW_CHECKS; } else if ("-exorcise".equals(arg)) { opts.doExorcise = true; } else if ("-crossCheckTermVectors".equals(arg)) { - System.err.println("-crossCheckTermVectors is deprecated, use -slow instead"); - opts.doSlowChecks = true; - } else if ("-slow".equals(arg)) { - opts.doSlowChecks = true; + // Deprecated. Remove in Lucene 11. + System.err.println("-crossCheckTermVectors is deprecated, use '-level 3' instead"); + opts.level = Level.MAX_VALUE; } else if (arg.equals("-verbose")) { opts.verbose = true; } else if (arg.equals("-segment")) { @@ -4172,11 +4195,13 @@ public final class CheckIndex implements Closeable { if (opts.indexPath == null) { throw new IllegalArgumentException( "\nERROR: index path not specified" - + "\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-slow] [-segment X] [-segment Y] [-threadCount X] [-dir-impl X]\n" + + "\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-exorcise] [-level X] [-segment X] [-segment Y] [-threadCount X] [-dir-impl X]\n" + "\n" + " -exorcise: actually write a new segments_N file, removing any problematic segments\n" - + " -fast: just verify file checksums, omitting logical integrity checks\n" - + " -slow: do additional slow checks; THIS IS VERY SLOW!\n" + + " -level X: sets the detail level of the check. The higher the value, the more checks are done.\n" + + " 1 - (Default) Checksum checks only.\n" + + " 2 - All level 1 checks + logical integrity checks.\n" + + " 3 - All level 2 checks + slow checks.\n" + " -codec X: when exorcising, codec to write the new segments_N file with\n" + " -verbose: print additional details\n" + " -segment X: only check the specified segments. This can be specified multiple\n" @@ -4191,7 +4216,8 @@ public final class CheckIndex implements Closeable { + "If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" - + "\n" + + "CheckIndex only verifies file checksums as default.\n" + + "Use -level with value of '2' or higher if you also want to check segment file contents.\n\n" + "**WARNING**: -exorcise *LOSES DATA*. This should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" @@ -4213,10 +4239,6 @@ public final class CheckIndex implements Closeable { throw new IllegalArgumentException("ERROR: cannot specify both -exorcise and -segment"); } - if (opts.doChecksumsOnly && opts.doSlowChecks) { - throw new IllegalArgumentException("ERROR: cannot specify both -fast and -slow"); - } - return opts; } @@ -4227,8 +4249,7 @@ public final class CheckIndex implements Closeable { * @return 0 iff the index is clean, 1 otherwise */ public int doCheck(Options opts) throws IOException, InterruptedException { - setDoSlowChecks(opts.doSlowChecks); - setChecksumsOnly(opts.doChecksumsOnly); + setLevel(opts.level); setInfoStream(opts.out, opts.verbose); // user provided thread count via command line argument, overriding the default with user // provided value diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java index f08a9d9a6ad..6b4abc87a09 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java @@ -166,7 +166,11 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase { expectThrows(Exception.class, () -> DirectoryReader.open(dirCopy).close()); // CheckIndex should also fail: - expectThrows(Exception.class, () -> TestUtil.checkIndex(dirCopy, true, true, true, null)); + expectThrows( + Exception.class, + () -> + TestUtil.checkIndex( + dirCopy, CheckIndex.Level.MIN_LEVEL_FOR_SLOW_CHECKS, true, true, null)); } } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java index 752ce7a7320..16d73eea97c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java @@ -139,7 +139,9 @@ public class TestCheckIndex extends BaseTestCheckIndex { } ByteArrayOutputStream output = new ByteArrayOutputStream(); - CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, true, output); + CheckIndex.Status status = + TestUtil.checkIndex( + dir, CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS, true, true, output); assertEquals(1, status.segmentInfos.size()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 364f1cc3720..0390edb36cf 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -2714,7 +2714,7 @@ public class TestIndexWriter extends LuceneTestCase { // Make sure CheckIndex includes id output: ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(d); - checker.setDoSlowChecks(false); + checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS); checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false); CheckIndex.Status indexStatus = checker.checkIndex(null); String s = bos.toString(IOUtils.UTF_8); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 334db5dccc2..2f9fd499cfd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -731,7 +731,9 @@ public class TestPointValues extends LuceneTestCase { w.close(); ByteArrayOutputStream output = new ByteArrayOutputStream(); - CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, true, output); + CheckIndex.Status status = + TestUtil.checkIndex( + dir, CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS, true, true, output); assertEquals(1, status.segmentInfos.size()); CheckIndex.Status.SegmentInfoStatus segStatus = status.segmentInfos.get(0); // total 3 point values were index: diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java index 8884295151a..8442a4a9ca7 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java @@ -121,7 +121,9 @@ public class TestSwappedIndexFiles extends LuceneTestCase { EOFException.class, IndexFormatTooOldException.class, CheckIndex.CheckIndexException.class), - () -> TestUtil.checkIndex(dirCopy, true, true, true, null)); + () -> + TestUtil.checkIndex( + dirCopy, CheckIndex.Level.MIN_LEVEL_FOR_SLOW_CHECKS, true, true, null)); } } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java index 8f5f8a84eb1..d513c6361be 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java @@ -1270,7 +1270,9 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe } ByteArrayOutputStream output = new ByteArrayOutputStream(); - CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, true, output); + CheckIndex.Status status = + TestUtil.checkIndex( + dir, CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS, true, true, output); assertEquals(1, status.segmentInfos.size()); CheckIndex.Status.SegmentInfoStatus segStatus = status.segmentInfos.get(0); // total 3 vector values were indexed: diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseTestCheckIndex.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseTestCheckIndex.java index afd3b10362c..1573aa63728 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseTestCheckIndex.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseTestCheckIndex.java @@ -59,6 +59,7 @@ public class BaseTestCheckIndex extends LuceneTestCase { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(dir); checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8)); + checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS); if (VERBOSE) checker.setInfoStream(System.out); CheckIndex.Status indexStatus = checker.checkIndex(); if (indexStatus.clean == false) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryWrapper.java index a440d2712f9..319722b5252 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryWrapper.java @@ -17,6 +17,7 @@ package org.apache.lucene.tests.store; import java.io.IOException; +import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; @@ -29,7 +30,7 @@ import org.apache.lucene.tests.util.TestUtil; public abstract class BaseDirectoryWrapper extends FilterDirectory { private boolean checkIndexOnClose = true; - private boolean doSlowChecksOnClose = true; + private int levelForCheckOnClose = CheckIndex.Level.MIN_LEVEL_FOR_SLOW_CHECKS; protected volatile boolean isOpen = true; protected BaseDirectoryWrapper(Directory delegate) { @@ -41,7 +42,7 @@ public abstract class BaseDirectoryWrapper extends FilterDirectory { if (isOpen) { isOpen = false; if (checkIndexOnClose && DirectoryReader.indexExists(this)) { - TestUtil.checkIndex(this, doSlowChecksOnClose); + TestUtil.checkIndex(this, levelForCheckOnClose); } } super.close(); @@ -61,10 +62,15 @@ public abstract class BaseDirectoryWrapper extends FilterDirectory { } public void setCrossCheckTermVectorsOnClose(boolean value) { - this.doSlowChecksOnClose = value; + // If true, we are enabling slow checks. + if (value == true) { + this.levelForCheckOnClose = CheckIndex.Level.MIN_LEVEL_FOR_SLOW_CHECKS; + } else { + this.levelForCheckOnClose = CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS; + } } - public boolean getCrossCheckTermVectorsOnClose() { - return doSlowChecksOnClose; + public int getLevelForCheckOnClose() { + return levelForCheckOnClose; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java index 62ffa70f237..0411a2c183b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java @@ -906,7 +906,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { // TestUtil#checkIndex checks segment concurrently using another thread, but making // call back to synchronized methods such as MockDirectoryWrapper#fileLength. // Hence passing concurrent = false to this method to turn off concurrent checks. - TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, false, null); + TestUtil.checkIndex(this, getLevelForCheckOnClose(), true, false, null); } // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java index 6d1d8573e5e..f86bedafef7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java @@ -306,12 +306,11 @@ public final class TestUtil { * thrown; else, true is returned. */ public static CheckIndex.Status checkIndex(Directory dir) throws IOException { - return checkIndex(dir, true); + return checkIndex(dir, CheckIndex.Level.MIN_LEVEL_FOR_SLOW_CHECKS); } - public static CheckIndex.Status checkIndex(Directory dir, boolean doSlowChecks) - throws IOException { - return checkIndex(dir, doSlowChecks, false, true, null); + public static CheckIndex.Status checkIndex(Directory dir, int level) throws IOException { + return checkIndex(dir, level, false, true, null); } /** @@ -319,11 +318,7 @@ public final class TestUtil { * moving on to other fields/segments to look for any other corruption. */ public static CheckIndex.Status checkIndex( - Directory dir, - boolean doSlowChecks, - boolean failFast, - boolean concurrent, - ByteArrayOutputStream output) + Directory dir, int level, boolean failFast, boolean concurrent, ByteArrayOutputStream output) throws IOException { if (output == null) { output = new ByteArrayOutputStream(1024); @@ -332,7 +327,7 @@ public final class TestUtil { // some tests e.g. exception tests become much more complicated if they have to close the writer try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) { - checker.setDoSlowChecks(doSlowChecks); + checker.setLevel(level); checker.setFailFast(failFast); checker.setInfoStream(new PrintStream(output, false, IOUtils.UTF_8), false); if (concurrent) { @@ -361,11 +356,11 @@ public final class TestUtil { */ public static void checkReader(IndexReader reader) throws IOException { for (LeafReaderContext context : reader.leaves()) { - checkReader(context.reader(), true); + checkReader(context.reader(), CheckIndex.Level.MIN_LEVEL_FOR_SLOW_CHECKS); } } - public static void checkReader(LeafReader reader, boolean doSlowChecks) throws IOException { + public static void checkReader(LeafReader reader, int level) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); PrintStream infoStream = new PrintStream(bos, false, IOUtils.UTF_8); @@ -379,9 +374,9 @@ public final class TestUtil { CheckIndex.testLiveDocs(codecReader, infoStream, true); CheckIndex.testFieldInfos(codecReader, infoStream, true); CheckIndex.testFieldNorms(codecReader, infoStream, true); - CheckIndex.testPostings(codecReader, infoStream, false, doSlowChecks, true); + CheckIndex.testPostings(codecReader, infoStream, false, level, true); CheckIndex.testStoredFields(codecReader, infoStream, true); - CheckIndex.testTermVectors(codecReader, infoStream, false, doSlowChecks, true); + CheckIndex.testTermVectors(codecReader, infoStream, false, level, true); CheckIndex.testDocValues(codecReader, infoStream, true); CheckIndex.testPoints(codecReader, infoStream, true);