LUCENE-3586: allow specifying -dir-impl FSDirectory subclass to CheckIndex, IndexUpgrader

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1213800 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -717,6 +717,10 @@ New Features
   have at least one or no value at all in a specific field. (Simon Willnauer,
   Uwe Schindler, Robert Muir)
 
+* LUCENE-3586: CheckIndex and IndexUpgrader allow you to specify the
+  specific FSDirectory implementation to use (with the new -dir-impl
+  command-line option).  (Luca Cavanna via Mike McCandless)
+  
 Bug fixes
 
 * LUCENE-3595: Fixed FieldCacheRangeFilter and FieldCacheTermsFilter

lucene/src/java/org/apache/lucene/index/CheckIndex.java

@@ -17,15 +17,6 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import org.apache.lucene.document.FieldType; // for javadocs
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.codecs.Codec;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
@@ -37,6 +28,16 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.document.FieldType; // for javadocs
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codec;
+
 import org.apache.lucene.index.codecs.BlockTreeTermsReader;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.values.IndexDocValues;
@@ -44,6 +45,7 @@ import org.apache.lucene.index.values.IndexDocValues.Source;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.StringHelper;
 
@@ -1408,41 +1410,48 @@ public class CheckIndex {
     boolean verbose = false;
     List<String> onlySegments = new ArrayList<String>();
     String indexPath = null;
+    String dirImpl = null;
     int i = 0;
     while(i < args.length) {
-      if (args[i].equals("-fix")) {
+      String arg = args[i];
+      if ("-fix".equals(arg)) {
         doFix = true;
-        i++;
+      } else if ("-codec".equals(arg)) {
-      } else if (args[i].equals("-codec")) {
         if (i == args.length-1) {
           System.out.println("ERROR: missing name for -codec option");
           System.exit(1);
         }
-        codec = Codec.forName(args[i+1]);
-        i+=2;
-      } else if (args[i].equals("-verbose")) {
-        verbose = true;
         i++;
-      } else if (args[i].equals("-segment")) {
+        codec = Codec.forName(args[i]);
+      } else if (arg.equals("-verbose")) {
+        verbose = true;
+      } else if (arg.equals("-segment")) {
         if (i == args.length-1) {
           System.out.println("ERROR: missing name for -segment option");
           System.exit(1);
         }
-        onlySegments.add(args[i+1]);
+        i++;
-        i += 2;
+        onlySegments.add(args[i]);
+      } else if ("-dir-impl".equals(arg)) {
+        if (i == args.length - 1) {
+          System.out.println("ERROR: missing value for -dir-impl option");
+          System.exit(1);
+        }
+        i++;
+        dirImpl = args[i];
       } else {
         if (indexPath != null) {
           System.out.println("ERROR: unexpected extra argument '" + args[i] + "'");
           System.exit(1);
         }
         indexPath = args[i];
-        i++;
       }
+      i++;
     }
 
     if (indexPath == null) {
       System.out.println("\nERROR: index path not specified");
-      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" +
+      System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y] [-dir-impl X]\n" +
                          "\n" +
                          "  -fix: actually write a new segments_N file, removing any problematic segments\n" +
                          "  -codec X: when fixing, codec to write the new segments_N file with\n" +
@@ -1450,7 +1459,8 @@ public class CheckIndex {
                          "  -segment X: only check the specified segments.  This can be specified multiple\n" + 
                          "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" +
                          "              You can't use this with the -fix option\n" +
-                         "\n" + 
+                         "  -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " +
+                         		"If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" +
                          "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
                          "documents (perhaps many) to be permanently removed from the index.  Always make\n" +
                          "a backup copy of your index before running this!  Do not run this tool on an index\n" +
@@ -1480,7 +1490,11 @@ public class CheckIndex {
     System.out.println("\nOpening index @ " + indexPath + "\n");
     Directory dir = null;
     try {
+      if (dirImpl == null) {
         dir = FSDirectory.open(new File(indexPath));
+      } else {
+        dir = CommandLineUtil.newFSDirectory(dirImpl, new File(indexPath));
+      }
     } catch (Throwable t) {
       System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting");
       t.printStackTrace(System.out);

lucene/src/java/org/apache/lucene/index/IndexReader.java

@@ -36,6 +36,7 @@ import org.apache.lucene.store.*;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.ReaderUtil;         // for javadocs
 
 /** IndexReader is an abstract class, providing an interface for accessing an
@@ -982,17 +983,28 @@ public abstract class IndexReader implements Cloneable,Closeable {
   public static void main(String [] args) {
     String filename = null;
     boolean extract = false;
+    String dirImpl = null;
 
-    for (int i = 0; i < args.length; ++i) {
+    int j = 0;
-      if (args[i].equals("-extract")) {
+    while(j < args.length) {
+      String arg = args[j];
+      if ("-extract".equals(arg)) {
         extract = true;
-      } else if (filename == null) {
+      } else if ("-dir-impl".equals(arg)) {
-        filename = args[i];
+        if (j == args.length - 1) {
+          System.out.println("ERROR: missing value for -dir-impl option");
+          System.exit(1);
         }
+        j++;
+        dirImpl = args[j];
+      } else if (filename == null) {
+        filename = arg;
+      }
+      j++;
     }
 
     if (filename == null) {
-      System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] <cfsfile>");
+      System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] [-dir-impl X] <cfsfile>");
       return;
     }
 
@@ -1004,7 +1016,12 @@ public abstract class IndexReader implements Cloneable,Closeable {
       File file = new File(filename);
       String dirname = file.getAbsoluteFile().getParent();
       filename = file.getName();
+      if (dirImpl == null) {
         dir = FSDirectory.open(new File(dirname));
+      } else {
+        dir = CommandLineUtil.newFSDirectory(dirImpl, new File(dirname));
+      }
+      
       cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false);
 
       String [] files = cfr.listAll();

lucene/src/java/org/apache/lucene/index/IndexUpgrader.java

@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.Version;
@@ -54,36 +55,56 @@ public final class IndexUpgrader {
   private static void printUsage() {
     System.err.println("Upgrades an index so all segments created with a previous Lucene version are rewritten.");
     System.err.println("Usage:");
-    System.err.println("  java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] indexDir");
+    System.err.println("  java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] [-dir-impl X] indexDir");
     System.err.println("This tool keeps only the last commit in an index; for this");
     System.err.println("reason, if the incoming index has more than one commit, the tool");
     System.err.println("refuses to run by default. Specify -delete-prior-commits to override");
     System.err.println("this, allowing the tool to delete all but the last commit.");
+    System.err.println("Specify a " + FSDirectory.class.getSimpleName() + 
+        " implementation through the -dir-impl option to force its use. If no package is specified the " 
+        + FSDirectory.class.getPackage().getName() + " package will be used.");
     System.err.println("WARNING: This tool may reorder document IDs!");
     System.exit(1);
   }
 
   @SuppressWarnings("deprecation")
   public static void main(String[] args) throws IOException {
-    String dir = null;
+    String path = null;
     boolean deletePriorCommits = false;
     PrintStream out = null;
-    for (String arg : args) {
+    String dirImpl = null;
+    int i = 0;
+    while (i<args.length) {
+      String arg = args[i];
       if ("-delete-prior-commits".equals(arg)) {
         deletePriorCommits = true;
       } else if ("-verbose".equals(arg)) {
         out = System.out;
-      } else if (dir == null) {
+      } else if (path == null) {
-        dir = arg;
+        path = arg;
-      } else {
+      } else if ("-dir-impl".equals(arg)) {
+        if (i == args.length - 1) {
+          System.out.println("ERROR: missing value for -dir-impl option");
+          System.exit(1);
+        }
+        i++;
+        dirImpl = args[i];
+      }else {
         printUsage();
       }
+      i++;
     }
-    if (dir == null) {
+    if (path == null) {
       printUsage();
     }
     
-    new IndexUpgrader(FSDirectory.open(new File(dir)), Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
+    Directory dir = null;
+    if (dirImpl == null) {
+      dir = FSDirectory.open(new File(path));
+    } else {
+      dir = CommandLineUtil.newFSDirectory(dirImpl, new File(path));
+    }
+    new IndexUpgrader(dir, Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
   }
   
   private final Directory dir;

lucene/src/java/org/apache/lucene/util/CommandLineUtil.java

@@ -0,0 +1,112 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+/**
+ * Class containing some useful methods used by command line tools 
+ *
+ */
+public final class CommandLineUtil {
+  
+  private CommandLineUtil() {
+    
+  }
+  
+  /**
+   * Creates a specific FSDirectory instance starting from its class name
+   * @param className The name of the FSDirectory class to load
+   * @param file The file to be used as parameter constructor
+   * @return the new FSDirectory instance
+   */
+  public static FSDirectory newFSDirectory(String clazzName, File file) {
+    try {
+      final Class<? extends FSDirectory> clazz = loadFSDirectoryClass(clazzName);
+      return newFSDirectory(clazz, file);
+    } catch (ClassNotFoundException e) {
+      throw new IllegalArgumentException(FSDirectory.class.getSimpleName()
+          + " implementation not found: " + clazzName, e);
+    } catch (ClassCastException e) {
+      throw new IllegalArgumentException(clazzName + " is not a " + FSDirectory.class.getSimpleName()
+          + " implementation", e);
+    } catch (NoSuchMethodException e) {
+      throw new IllegalArgumentException(clazzName + " constructor with "
+          + File.class.getSimpleName() + " as parameter not found", e);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Error creating " + clazzName + " instance", e);
+    }
+  }
+  
+  /**
+   * Loads a specific Directory implementation 
+   * @param className The name of the Directory class to load
+   * @return The Directory class loaded
+   * @throws ClassNotFoundException
+   */
+  public static Class<? extends Directory> loadDirectoryClass(String clazzName) 
+      throws ClassNotFoundException {
+    return Class.forName(adjustDirectoryClassName(clazzName)).asSubclass(Directory.class);
+  }
+  
+  /**
+   * Loads a specific FSDirectory implementation
+   * @param className The name of the FSDirectory class to load
+   * @return The FSDirectory class loaded
+   * @throws ClassNotFoundException
+   */
+  public static Class<? extends FSDirectory> loadFSDirectoryClass(String clazzName) 
+      throws ClassNotFoundException {
+    return Class.forName(adjustDirectoryClassName(clazzName)).asSubclass(FSDirectory.class);
+  }
+  
+  private static String adjustDirectoryClassName(String clazzName) {
+    if (clazzName == null || clazzName.trim().length() == 0) {
+      throw new IllegalArgumentException("The " + FSDirectory.class.getSimpleName()
+          + " implementation cannot be null or empty");
+    }
+    
+    if (clazzName.indexOf(".") == -1) {// if not fully qualified, assume .store
+      clazzName = Directory.class.getPackage().getName() + "." + clazzName;
+    }
+    return clazzName;
+  }
+  
+  /**
+   * Creates a new specific FSDirectory instance
+   * @param clazz The class of the object to be created
+   * @param file The file to be used as parameter constructor
+   * @return The new FSDirectory instance
+   * @throws NoSuchMethodException
+   * @throws InstantiationException
+   * @throws IllegalAccessException
+   * @throws InvocationTargetException
+   */
+  public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, File file) 
+      throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException {
+    // Assuming every FSDirectory has a ctor(File):
+    Constructor<? extends FSDirectory> ctor = clazz.getConstructor(File.class);
+    return ctor.newInstance(file);
+  }
+  
+}

lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java

@@ -26,7 +26,6 @@ import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
-import java.lang.reflect.Constructor;
 import java.util.*;
 import java.util.Map.Entry;
 import java.util.concurrent.ExecutorService;
@@ -1035,24 +1034,16 @@ public abstract class LuceneTestCase extends Assert {
       fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)];
     }
 
-    if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store
-      fsdirClass = "org.apache.lucene.store." + fsdirClass;
-    }
-
     Class<? extends FSDirectory> clazz;
     try {
       try {
-        clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class);
+        clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass);
       } catch (ClassCastException e) {
         // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random
         fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)];
-
+        clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass);
-        if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store
-          fsdirClass = "org.apache.lucene.store." + fsdirClass;
       }
       
-        clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class);
-      }
       MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f));
       if (lf != null) {
         dir.setLockFactory(lf);
@@ -1165,10 +1156,7 @@ public abstract class LuceneTestCase extends Assert {
       throws IOException {
     FSDirectory d = null;
     try {
-      // Assuming every FSDirectory has a ctor(File), but not all may take a
+      d = CommandLineUtil.newFSDirectory(clazz, file);
-      // LockFactory too, so setting it afterwards.
-      Constructor<? extends FSDirectory> ctor = clazz.getConstructor(File.class);
-      d = ctor.newInstance(file);
     } catch (Exception e) {
       d = FSDirectory.open(file);
     }
@@ -1186,12 +1174,12 @@ public abstract class LuceneTestCase extends Assert {
   }
   
   static Directory newDirectoryImpl(Random random, String clazzName) {
-    if (clazzName.equals("random"))
+    if (clazzName.equals("random")) {
       clazzName = randomDirectory(random);
-    if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store
+    }
-      clazzName = "org.apache.lucene.store." + clazzName;
+    
     try {
-      final Class<? extends Directory> clazz = Class.forName(clazzName).asSubclass(Directory.class);
+      final Class<? extends Directory> clazz = CommandLineUtil.loadDirectoryClass(clazzName);
       // If it is a FSDirectory type, try its ctor(File)
       if (FSDirectory.class.isAssignableFrom(clazz)) {
         final File dir = _TestUtil.getTempDir("index");