Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1354: Provide programmatic access to CheckIndex 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@693508 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Grant Ingersoll 2008-09-09 16:34:18 +00:00
parent dadaebc290
commit 2bb9ab2246
5 changed files with 201 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -269,6 +269,8 @@ New features
18. LUCENE-1001: Provide access to Payloads via Spans. All existing Span Query implementations in Lucene implement. (Mark Miller, Grant Ingersoll) 18. LUCENE-1001: Provide access to Payloads via Spans. All existing Span Query implementations in Lucene implement. (Mark Miller, Grant Ingersoll)
19. LUCENE-1354: Provide programmatic access to CheckIndex (Grant Ingersoll, Mike McCandless)
Optimizations Optimizations
1. LUCENE-705: When building a compound file, use 1. LUCENE-705: When building a compound file, use

233
src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -42,7 +42,7 @@ import java.util.ArrayList;
public class CheckIndex { public class CheckIndex {
public static PrintStream out = System.out; public static PrintStream out = null;
private static class MySegmentTermDocs extends SegmentTermDocs { private static class MySegmentTermDocs extends SegmentTermDocs {
@ -63,21 +63,23 @@ public class CheckIndex {
} }
/** Returns true if index is clean, else false.*/ /** Returns true if index is clean, else false.*/
public static boolean check(Directory dir, boolean doFix) throws IOException { public static CheckIndexStatus check(Directory dir, boolean doFix) throws IOException {
return check(dir, doFix, null); return check(dir, doFix, null);
} }
/** Returns true if index is clean, else false.*/ /** Returns true if index is clean, else false.*/
public static boolean check(Directory dir, boolean doFix, List onlySegments) throws IOException { public static CheckIndexStatus check(Directory dir, boolean doFix, List onlySegments) throws IOException {
NumberFormat nf = NumberFormat.getInstance(); NumberFormat nf = NumberFormat.getInstance();
SegmentInfos sis = new SegmentInfos(); SegmentInfos sis = new SegmentInfos();
CheckIndexStatus result = new CheckIndexStatus();
result.dir = dir;
try { try {
sis.read(dir); sis.read(dir);
} catch (Throwable t) { } catch (Throwable t) {
out.println("ERROR: could not read any segments file in directory"); msg("ERROR: could not read any segments file in directory");
result.missingSegments = true;
t.printStackTrace(out); t.printStackTrace(out);
return false; return result;
} }
final int numSegments = sis.size(); final int numSegments = sis.size();
@ -86,17 +88,19 @@ public class CheckIndex {
try { try {
input = dir.openInput(segmentsFileName); input = dir.openInput(segmentsFileName);
} catch (Throwable t) { } catch (Throwable t) {
out.println("ERROR: could not open segments file in directory"); msg("ERROR: could not open segments file in directory");
t.printStackTrace(out); t.printStackTrace(out);
return false; result.cantOpenSegments = true;
return result;
} }
int format = 0; int format = 0;
try { try {
format = input.readInt(); format = input.readInt();
} catch (Throwable t) { } catch (Throwable t) {
out.println("ERROR: could not read segment file version in directory"); msg("ERROR: could not read segment file version in directory");
t.printStackTrace(out); t.printStackTrace(out);
return false; result.missingSegmentVersion = true;
return result;
} finally { } finally {
if (input != null) if (input != null)
input.close(); input.close();
@ -128,7 +132,10 @@ public class CheckIndex {
} }
} }
out.println("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat); msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);
result.segmentsFileName = segmentsFileName;
result.numSegments = numSegments;
result.segmentFormat = sFormat;
if (onlySegments != null) { if (onlySegments != null) {
out.print("\nChecking only these segments:"); out.print("\nChecking only these segments:");
@ -136,56 +143,80 @@ public class CheckIndex {
while (it.hasNext()) { while (it.hasNext()) {
out.print(" " + it.next()); out.print(" " + it.next());
} }
out.println(":"); result.segmentsChecked.addAll(onlySegments);
msg(":");
} }
if (skip) { if (skip) {
out.println("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting"); msg("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
return false; result.toolOutOfDate = true;
return result;
} }
SegmentInfos newSIS = (SegmentInfos) sis.clone();
newSIS.clear(); result.newSegments = (SegmentInfos) sis.clone();
boolean changed = false; result.newSegments.clear();
int totLoseDocCount = 0;
int numBadSegments = 0;
for(int i=0;i<numSegments;i++) { for(int i=0;i<numSegments;i++) {
final SegmentInfo info = sis.info(i); final SegmentInfo info = sis.info(i);
if (onlySegments != null && !onlySegments.contains(info.name)) if (onlySegments != null && !onlySegments.contains(info.name))
continue; continue;
out.println(" " + (1+i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount); CheckIndexStatus.SegmentInfoStatus segInfoStat = new CheckIndexStatus.SegmentInfoStatus();
result.segmentInfos.add(segInfoStat);
msg(" " + (1+i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
segInfoStat.name = info.name;
segInfoStat.docCount = info.docCount;
int toLoseDocCount = info.docCount; int toLoseDocCount = info.docCount;
SegmentReader reader = null; SegmentReader reader = null;
try { try {
out.println(" compound=" + info.getUseCompoundFile()); msg(" compound=" + info.getUseCompoundFile());
out.println(" hasProx=" + info.getHasProx()); segInfoStat.compound = info.getUseCompoundFile();
out.println(" numFiles=" + info.files().size()); msg(" hasProx=" + info.getHasProx());
out.println(" size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.))); segInfoStat.hasProx = info.getHasProx();
msg(" numFiles=" + info.files().size());
segInfoStat.numFiles = info.files().size();
msg(" size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.)));
segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
final int docStoreOffset = info.getDocStoreOffset(); final int docStoreOffset = info.getDocStoreOffset();
if (docStoreOffset != -1) { if (docStoreOffset != -1) {
out.println(" docStoreOffset=" + docStoreOffset); msg(" docStoreOffset=" + docStoreOffset);
out.println(" docStoreSegment=" + info.getDocStoreSegment()); segInfoStat.docStoreOffset = docStoreOffset;
out.println(" docStoreIsCompoundFile=" + info.getDocStoreIsCompoundFile()); msg(" docStoreSegment=" + info.getDocStoreSegment());
segInfoStat.docStoreSegment = info.getDocStoreSegment();
msg(" docStoreIsCompoundFile=" + info.getDocStoreIsCompoundFile());
segInfoStat.docStoreCompoundFile = info.getDocStoreIsCompoundFile();
} }
final String delFileName = info.getDelFileName(); final String delFileName = info.getDelFileName();
if (delFileName == null) if (delFileName == null){
out.println(" no deletions"); msg(" no deletions");
else segInfoStat.hasDeletions = false;
out.println(" has deletions [delFileName=" + delFileName + "]"); }
else{
msg(" has deletions [delFileName=" + delFileName + "]");
segInfoStat.hasDeletions = true;
segInfoStat.deletionsFileName = delFileName;
}
out.print(" test: open reader........."); out.print(" test: open reader.........");
reader = SegmentReader.get(info); reader = SegmentReader.get(info);
final int numDocs = reader.numDocs(); final int numDocs = reader.numDocs();
toLoseDocCount = numDocs; toLoseDocCount = numDocs;
if (reader.hasDeletions()) { if (reader.hasDeletions()) {
if (info.docCount - numDocs != info.getDelCount()) if (info.docCount - numDocs != info.getDelCount()){
throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs)); throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
out.println("OK [" + (info.docCount - numDocs) + " deleted docs]"); }
segInfoStat.numDeleted = info.docCount - numDocs;
msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
} else { } else {
if (info.getDelCount() != 0) if (info.getDelCount() != 0){
throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs)); throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
out.println("OK"); }
msg("OK");
} }
out.print(" test: fields, norms......."); out.print(" test: fields, norms.......");
@ -198,8 +229,8 @@ public class CheckIndex {
throw new RuntimeException("norms for field \"" + fieldName + "\" is length " + b.length + " != maxDoc " + info.docCount); throw new RuntimeException("norms for field \"" + fieldName + "\" is length " + b.length + " != maxDoc " + info.docCount);
} }
out.println("OK [" + fieldNames.size() + " fields]"); msg("OK [" + fieldNames.size() + " fields]");
segInfoStat.numFields = fieldNames.size();
out.print(" test: terms, freq, prox..."); out.print(" test: terms, freq, prox...");
final TermEnum termEnum = reader.terms(); final TermEnum termEnum = reader.terms();
final TermPositions termPositions = reader.termPositions(); final TermPositions termPositions = reader.termPositions();
@ -255,7 +286,7 @@ public class CheckIndex {
throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount); throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
} }
out.println("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]"); msg("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");
out.print(" test: stored fields......."); out.print(" test: stored fields.......");
int docCount = 0; int docCount = 0;
@ -270,7 +301,7 @@ public class CheckIndex {
if (docCount != reader.numDocs()) if (docCount != reader.numDocs())
throw new RuntimeException("docCount=" + docCount + " but saw " + docCount + " undeleted docs"); throw new RuntimeException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
out.println("OK [" + totFields + " total field count; avg " + nf.format((((float) totFields)/docCount)) + " fields per doc]"); msg("OK [" + totFields + " total field count; avg " + nf.format((((float) totFields)/docCount)) + " fields per doc]");
out.print(" test: term vectors........"); out.print(" test: term vectors........");
int totVectors = 0; int totVectors = 0;
@ -281,22 +312,21 @@ public class CheckIndex {
totVectors += tfv.length; totVectors += tfv.length;
} }
out.println("OK [" + totVectors + " total vector count; avg " + nf.format((((float) totVectors)/docCount)) + " term/freq vector fields per doc]"); msg("OK [" + totVectors + " total vector count; avg " + nf.format((((float) totVectors)/docCount)) + " term/freq vector fields per doc]");
out.println(""); msg("");
} catch (Throwable t) { } catch (Throwable t) {
out.println("FAILED"); msg("FAILED");
String comment; String comment;
if (doFix) if (doFix)
comment = "will remove reference to this segment (-fix is specified)"; comment = "will remove reference to this segment (-fix is specified)";
else else
comment = "would remove reference to this segment (-fix was not specified)"; comment = "would remove reference to this segment (-fix was not specified)";
out.println(" WARNING: " + comment + "; full exception:"); msg(" WARNING: " + comment + "; full exception:");
t.printStackTrace(out); t.printStackTrace(out);
out.println(""); msg("");
totLoseDocCount += toLoseDocCount; result.totLoseDocCount += toLoseDocCount;
numBadSegments++; result.numBadSegments++;
changed = true;
continue; continue;
} finally { } finally {
if (reader != null) if (reader != null)
@ -304,50 +334,25 @@ public class CheckIndex {
} }
// Keeper // Keeper
newSIS.add(info.clone()); result.newSegments.add(info.clone());
} }
if (!changed) { if (0 == result.numBadSegments) {
out.println("No problems were detected with this index.\n"); result.clean = true;
return true; msg("No problems were detected with this index.\n");
} else { } else
out.println("WARNING: " + numBadSegments + " broken segments detected"); msg("WARNING: " + result.numBadSegments + " broken segments (containing " + result.totLoseDocCount + " documents) detected");
if (doFix)
out.println("WARNING: " + totLoseDocCount + " documents will be lost");
else
out.println("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
out.println();
}
if (doFix) { return result;
out.println("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!"); }
for(int i=0;i<5;i++) {
try { /** Repairs the index using previously returned result from
Thread.sleep(1000); * {@link #check}. <b>WARNING</b>: this writes a new
} catch (InterruptedException ie) { * segments file into the index, effectively removing
Thread.currentThread().interrupt(); * all documents in broken segments from the index. BE
i--; * CAREFUL. */
continue; static public void fix(CheckIndexStatus result) throws IOException {
} result.newSegments.commit(result.dir);
out.println(" " + (5-i) + "...");
}
out.print("Writing...");
try {
newSIS.commit(dir);
} catch (Throwable t) {
out.println("FAILED; exiting");
t.printStackTrace(out);
return false;
}
out.println("OK");
out.println("Wrote new segments file \"" + newSIS.getCurrentSegmentFileName() + "\"");
} else {
out.println("NOTE: would write new segments file [-fix was not specified]");
}
out.println("");
return false;
} }
static boolean assertsOn; static boolean assertsOn;
@ -357,6 +362,12 @@ public class CheckIndex {
return true; return true;
} }
private static void msg(String msg) {
if (out != null) {
out.println(msg);
}
}
public static void main(String[] args) throws Throwable { public static void main(String[] args) throws Throwable {
boolean doFix = false; boolean doFix = false;
@ -369,14 +380,14 @@ public class CheckIndex {
i++; i++;
} else if (args[i].equals("-segment")) { } else if (args[i].equals("-segment")) {
if (i == args.length-1) { if (i == args.length-1) {
out.println("ERROR: missing name for -segment option"); msg("ERROR: missing name for -segment option");
System.exit(1); System.exit(1);
} }
onlySegments.add(args[i+1]); onlySegments.add(args[i+1]);
i += 2; i += 2;
} else { } else {
if (indexPath != null) { if (indexPath != null) {
out.println("ERROR: unexpected extra argument '" + args[i] + "'"); msg("ERROR: unexpected extra argument '" + args[i] + "'");
System.exit(1); System.exit(1);
} }
indexPath = args[i]; indexPath = args[i];
@ -385,8 +396,8 @@ public class CheckIndex {
} }
if (indexPath == null) { if (indexPath == null) {
out.println("\nERROR: index path not specified"); msg("\nERROR: index path not specified");
out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + msg("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" +
"\n" + "\n" +
" -fix: actually write a new segments_N file, removing any problematic segments\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" +
" -segment X: only check the specified segments. This can be specified multiple\n" + " -segment X: only check the specified segments. This can be specified multiple\n" +
@ -412,31 +423,55 @@ public class CheckIndex {
if (onlySegments.size() == 0) if (onlySegments.size() == 0)
onlySegments = null; onlySegments = null;
else if (doFix) { else if (doFix) {
out.println("ERROR: cannot specify both -fix and -segment"); msg("ERROR: cannot specify both -fix and -segment");
System.exit(1); System.exit(1);
} }
assert testAsserts(); assert testAsserts();
if (!assertsOn) if (!assertsOn)
out.println("\nNOTE: testing will be more thorough if you run java with '-ea:org.apache.lucene', so assertions are enabled"); msg("\nNOTE: testing will be more thorough if you run java with '-ea:org.apache.lucene', so assertions are enabled");
out.println("\nOpening index @ " + indexPath + "\n"); msg("\nOpening index @ " + indexPath + "\n");
Directory dir = null; Directory dir = null;
try { try {
dir = FSDirectory.getDirectory(indexPath); dir = FSDirectory.getDirectory(indexPath);
} catch (Throwable t) { } catch (Throwable t) {
out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting"); msg("ERROR: could not open directory \"" + indexPath + "\"; exiting");
t.printStackTrace(out); t.printStackTrace(out);
System.exit(1); System.exit(1);
} }
boolean isClean = check(dir, doFix, onlySegments); CheckIndexStatus result = check(dir, doFix, onlySegments);
if (!result.clean) {
if (!doFix){
msg("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
} else {
msg("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
msg("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for(int s=0;s<5;s++) {
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
s--;
continue;
}
msg(" " + (5-i) + "...");
}
msg("Writing...");
CheckIndex.fix(result);
}
msg("OK");
msg("Wrote new segments file \"" + result.newSegments.getCurrentSegmentFileName() + "\"");
}
msg("");
final int exitCode; final int exitCode;
if (isClean) if (result != null && result.clean == true)
exitCode = 0; exitCode = 0;
else else
exitCode = 1; exitCode = 1;
System.exit(exitCode); System.exit(exitCode);
} }
} }

57
src/java/org/apache/lucene/index/CheckIndexStatus.java Normal file
View File

@ -0,0 +1,57 @@
package org.apache.lucene.index;
import org.apache.lucene.store.Directory;
import java.util.List;
import java.util.ArrayList;
/**
*
*
**/
public class CheckIndexStatus {
public boolean clean;
public boolean missingSegments;
public boolean cantOpenSegments;
public boolean missingSegmentVersion;
public String segmentsFileName;
public int numSegments;
public String segmentFormat;
public List/*<String>*/ segmentsChecked = new ArrayList();
public boolean toolOutOfDate;
public List/*<SegmentInfoStatus*/ segmentInfos = new ArrayList();
public Directory dir;
public SegmentInfos newSegments;
public int totLoseDocCount;
public int numBadSegments;
public static class SegmentInfoStatus{
public String name;
public int docCount;
public boolean compound;
public int numFiles;
public double sizeMB;
public int docStoreOffset = -1;
public String docStoreSegment;
public boolean docStoreCompoundFile;
public boolean hasDeletions;
public String deletionsFileName;
public int numDeleted;
public boolean openReaderPassed;
int numFields;
public boolean hasProx;
}
}

5
src/test/org/apache/lucene/index/TestCheckIndex.java
View File

@ -49,13 +49,14 @@ public class TestCheckIndex extends LuceneTestCase {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex.out = new PrintStream(bos); CheckIndex.out = new PrintStream(bos);
if (!CheckIndex.check(dir, false, null)) { CheckIndexStatus indexStatus = CheckIndex.check(dir, false, null);
if (indexStatus.clean == false) {
System.out.println("CheckIndex failed"); System.out.println("CheckIndex failed");
System.out.println(bos.toString()); System.out.println(bos.toString());
fail(); fail();
} }
final List onlySegments = new ArrayList(); final List onlySegments = new ArrayList();
onlySegments.add("_0"); onlySegments.add("_0");
assertTrue(CheckIndex.check(dir, false, onlySegments)); assertTrue(CheckIndex.check(dir, false, onlySegments).clean == true);
} }
} }

6
src/test/org/apache/lucene/util/_TestUtil.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.CheckIndexStatus;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.PrintStream; import java.io.PrintStream;
@ -60,7 +61,10 @@ public class _TestUtil {
public static boolean checkIndex(Directory dir) throws IOException { public static boolean checkIndex(Directory dir) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex.out = new PrintStream(bos); CheckIndex.out = new PrintStream(bos);
if (!CheckIndex.check(dir, false, null)) {
//TODO: fix this
CheckIndexStatus indexStatus = CheckIndex.check(dir, false, null);
if (indexStatus == null || indexStatus.clean == false) {
System.out.println("CheckIndex failed"); System.out.println("CheckIndex failed");
System.out.println(bos.toString()); System.out.println(bos.toString());
throw new RuntimeException("CheckIndex failed"); throw new RuntimeException("CheckIndex failed");