From e7debdf9e3c8875db9632370387ac47176b9f6b3 Mon Sep 17 00:00:00 2001
From: Michael McCandless <mikemccand@apache.org>
Date: Mon, 22 Oct 2007 14:22:24 +0000
Subject: [PATCH] LUCENE-1020: created basic tool to check an index, print
 details of each segment, and fix the index by removing broken segments

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@587107 13f79535-47bb-0310-9956-ffa450edef68
---
 .../org/apache/lucene/index/CheckIndex.java   | 302 ++++++++++++++++++
 1 file changed, 302 insertions(+)
 create mode 100644 src/java/org/apache/lucene/index/CheckIndex.java

diff --git a/src/java/org/apache/lucene/index/CheckIndex.java b/src/java/org/apache/lucene/index/CheckIndex.java
new file mode 100644
index 00000000000..258ea65e5d8
--- /dev/null
+++ b/src/java/org/apache/lucene/index/CheckIndex.java
@@ -0,0 +1,302 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.document.Document;
+
+import java.text.NumberFormat;
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * Basic tool to check the health of an index and write a
+ * new segments file that removes reference to problematic
+ * segments.  There are many more checks that this tool
+ * could do but does not yet, eg: reconstructing a segments
+ * file by looking for all loadable segments (if no segments
+ * file is found), removing specifically specified segments,
+ * listing files that exist but are not referenced, etc.
+ */
+
+public class CheckIndex {
+
+  public static void main(String[] args) throws Throwable {
+
+    boolean doFix = false;
+    for(int i=0;i<args.length;i++)
+      if (args[i].equals("-fix")) {
+        doFix = true;
+        break;
+      }
+
+    if (args.length != (doFix ? 2:1)) {
+      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix]\n" +
+                         "\n" +
+                         "  -fix: actually write a new segments_N file, removing any problematic segments\n" +
+                         "\n" + 
+                         "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
+                         "documents (perhaps many) to be permanently removed from the index.  Always make\n" +
+                         "a backup copy of your index before running this!  Do not run this tool on an index\n" +
+                         "that is actively being written to.  You have been warned!\n" +
+                         "\n" +
+                         "Run without -fix, this tool will open the index, report version information\n" +
+                         "and report any exceptions it hits and what action it would take if -fix were\n" +
+                         "specified.  With -fix, this tool will remove any segments that have issues and\n" + 
+                         "write a new segments_N file.  This means all documents contained in the affected\n" +
+                         "segments will be removed.\n");
+      System.exit(1);
+    }
+
+    NumberFormat nf = NumberFormat.getInstance();
+    
+    SegmentInfos sis = new SegmentInfos();
+    final String dirName = args[0];
+    System.out.println("\nOpening index @ " + dirName + "\n");
+    Directory dir = null;
+    try {
+      dir = FSDirectory.getDirectory(dirName);
+    } catch (Throwable t) {
+      System.out.println("ERROR: could not open directory \"" + dirName + "\"; exiting");
+      t.printStackTrace(System.out);
+      System.exit(1);
+    } 
+    
+    try {
+      sis.read(dir);
+    } catch (Throwable t) {
+      System.out.println("ERROR: could not read any segments file in directory \"" + dirName + "\"; exiting");
+      t.printStackTrace(System.out);
+      System.exit(1);
+    }
+
+    final int numSegments = sis.size();
+    final String segmentsFileName = sis.getCurrentSegmentFileName();
+    IndexInput input = null;
+    try {
+      input = dir.openInput(segmentsFileName);
+    } catch (Throwable t) {
+      System.out.println("ERROR: could not open segments file in directory \"" + dirName + "\"; exiting");
+      t.printStackTrace(System.out);
+      System.exit(1);
+    }
+    int format = 0;
+    try {
+      format = input.readInt();
+    } catch (Throwable t) {
+      System.out.println("ERROR: could not read segment file version in directory \"" + dirName + "\"; exiting");
+      t.printStackTrace(System.out);
+      System.exit(1);
+    }
+
+    String sFormat = "";
+    boolean skip = false;
+
+    if (format == SegmentInfos.FORMAT)
+      sFormat = "FORMAT [Lucene Pre-2.1]";
+    if (format == SegmentInfos.FORMAT_LOCKLESS)
+      sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
+    else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
+      sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
+    else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
+      sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
+    else if (format < SegmentInfos.FORMAT_SHARED_DOC_STORE) {
+      sFormat = "int=" + format + " [newer version of Lucene than this tool]";
+      skip = true;
+    } else {
+      sFormat = format + " [Lucene 1.3 or prior]";
+    }
+
+    System.out.println("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);
+
+    if (skip) {
+      System.out.println("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
+      System.exit(1);
+    }
+
+    SegmentInfos newSIS = (SegmentInfos) sis.clone();
+    newSIS.clear();
+    boolean changed = false;
+    int totLoseDocCount = 0;
+    int numBadSegments = 0;
+    for(int i=0;i<numSegments;i++) {
+      final SegmentInfo info = sis.info(i);
+      System.out.println("  " + (1+i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
+      int toLoseDocCount = info.docCount;
+
+      try {
+        System.out.println("    compound=" + info.getUseCompoundFile());
+        System.out.println("    numFiles=" + info.files().size());
+        System.out.println("    size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.)));
+        final int docStoreOffset = info.getDocStoreOffset();
+        if (docStoreOffset != -1) {
+          System.out.println("    docStoreOffset=" + docStoreOffset);
+          System.out.println("    docStoreSegment=" + info.getDocStoreSegment());
+          System.out.println("    docStoreIsCompoundFile=" + info.getDocStoreIsCompoundFile());
+        }
+        final String delFileName = info.getDelFileName();
+        if (delFileName == null)
+          System.out.println("    no deletions");
+        else
+          System.out.println("    has deletions [delFileName=" + delFileName + "]");
+        System.out.print("    test: open reader.........");
+        SegmentReader reader = SegmentReader.get(info);
+        final int numDocs = reader.numDocs();
+        toLoseDocCount = numDocs;
+        if (reader.hasDeletions())
+          System.out.println("OK [" + (info.docCount - numDocs) + " deleted docs]");
+        else
+          System.out.println("OK");
+
+        System.out.print("    test: fields, norms.......");
+        Collection fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
+        Iterator it = fieldNames.iterator();
+        while(it.hasNext()) {
+          final String fieldName = (String) it.next();
+          byte[] b = reader.norms(fieldName);
+          if (b.length != info.docCount)
+            throw new RuntimeException("norms for field \"" + fieldName + "\" is length " + b.length + " != maxDoc " + info.docCount);
+
+        }
+        System.out.println("OK [" + fieldNames.size() + " fields]");
+
+        System.out.print("    test: terms, freq, prox...");
+        final TermEnum termEnum = reader.terms();
+        final TermPositions termPositions = reader.termPositions();
+        long termCount = 0;
+        long totFreq = 0;
+        long totPos = 0;
+        while(termEnum.next()) {
+          termCount++;
+          final Term term = termEnum.term();
+          final int docFreq = termEnum.docFreq();
+          termPositions.seek(term);
+          int lastDoc = -1;
+          int freq0 = 0;
+          totFreq += docFreq;
+          while(termPositions.next()) {
+            freq0++;
+            final int doc = termPositions.doc();
+            final int freq = termPositions.freq();
+            if (doc <= lastDoc)
+              throw new RuntimeException("term " + term + ": doc " + doc + " < lastDoc " + lastDoc);
+            lastDoc = doc;
+            if (freq <= 0)
+              throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+            
+            int lastPos = -1;
+            totPos += freq;
+            for(int j=0;j<freq;j++) {
+              final int pos = termPositions.nextPosition();
+              if (pos < 0)
+                throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
+              if (pos <= lastPos)
+                throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
+            }
+          }
+          if (freq0 != docFreq)
+            throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0);
+        }
+
+        System.out.println("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");
+
+        System.out.print("    test: stored fields.......");
+        int docCount = 0;
+        long totFields = 0;
+        for(int j=0;j<info.docCount;j++)
+          if (!reader.isDeleted(j)) {
+            docCount++;
+            Document doc = reader.document(j);
+            totFields += doc.getFields().size();
+          }
+
+        if (docCount != reader.numDocs())
+          throw new RuntimeException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
+
+        System.out.println("OK [" + totFields + " total field count; avg " + nf.format((((float) totFields)/docCount)) + " fields per doc]");
+
+        System.out.print("    test: term vectors........");
+        int totVectors = 0;
+        for(int j=0;j<info.docCount;j++)
+          if (!reader.isDeleted(j)) {
+            TermFreqVector[] tfv = reader.getTermFreqVectors(j);
+            if (tfv != null)
+              totVectors += tfv.length;
+          }
+
+        System.out.println("OK [" + totVectors + " total vector count; avg " + nf.format((((float) totVectors)/docCount)) + " term/freq vector fields per doc]");
+        System.out.println("");
+
+      } catch (Throwable t) {
+        System.out.println("FAILED");
+        String comment;
+        if (doFix)
+          comment = "will remove reference to this segment (-fix is specified)";
+        else
+          comment = "would remove reference to this segment (-fix was not specified)";
+        System.out.println("    WARNING: " + comment + "; full exception:");
+        t.printStackTrace(System.out);
+        System.out.println("");
+        totLoseDocCount += toLoseDocCount;
+        numBadSegments++;
+        changed = true;
+        continue;
+      }
+
+      // Keeper
+      newSIS.add(info.clone());
+    }
+
+    if (!changed) {
+      System.out.println("No problems were detected with this index.\n");
+      System.exit(0);
+    } else {
+      System.out.println("WARNING: " + numBadSegments + " broken segments detected");
+      if (doFix)
+        System.out.println("WARNING: " + totLoseDocCount + " documents will be lost");
+      else
+        System.out.println("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
+      System.out.println("");
+    }
+
+    if (doFix) {
+      System.out.println("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
+      for(int i=0;i<5;i++) {
+        Thread.sleep(1000);
+        System.out.println("  " + (5-i) + "...");
+      }
+      System.out.print("Writing...");
+      try {
+        newSIS.write(dir);
+      } catch (Throwable t) {
+        System.out.println("FAILED; exiting");
+        t.printStackTrace(System.out);
+        System.exit(1);
+      }
+      System.out.println("OK");
+      System.out.println("Wrote new segments file \"" + newSIS.getCurrentSegmentFileName() + "\"");
+    } else {
+      System.out.println("NOTE: would write new segments file [-fix was not specified]");
+    }
+    System.out.println("");
+
+    System.exit(0);
+  }
+}