From d6a96a43d9891ffbd4210d41b5e8a50a55bf7916 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 13 Dec 2011 17:34:58 +0000 Subject: [PATCH] LUCENE-3586: allow specifying -dir-impl FSDirectory subclass to CheckIndex, IndexUpgrader git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1213800 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 4 + .../org/apache/lucene/index/CheckIndex.java | 60 ++++++---- .../org/apache/lucene/index/IndexReader.java | 27 ++++- .../apache/lucene/index/IndexUpgrader.java | 37 ++++-- .../apache/lucene/util/CommandLineUtil.java | 112 ++++++++++++++++++ .../apache/lucene/util/LuceneTestCase.java | 28 ++--- 6 files changed, 212 insertions(+), 56 deletions(-) create mode 100644 lucene/src/java/org/apache/lucene/util/CommandLineUtil.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 5c8727fcbea..a952c18a243 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -716,6 +716,10 @@ New Features * LUCENE-3593: Added a FieldValueFilter that accepts all documents that either have at least one or no value at all in a specific field. (Simon Willnauer, Uwe Schindler, Robert Muir) + +* LUCENE-3586: CheckIndex and IndexUpgrader allow you to specify the + specific FSDirectory implementation to use (with the new -dir-impl + command-line option). (Luca Cavanna via Mike McCandless) Bug fixes diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 462b78cf116..eda9b7c4536 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -17,15 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import org.apache.lucene.document.FieldType; // for javadocs -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.codecs.Codec; import java.io.File; import java.io.IOException; import java.io.PrintStream; @@ -37,6 +28,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.document.FieldType; // for javadocs +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.codecs.Codec; + import org.apache.lucene.index.codecs.BlockTreeTermsReader; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; @@ -44,6 +45,7 @@ import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CommandLineUtil; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.StringHelper; @@ -1408,41 +1410,48 @@ public class CheckIndex { boolean verbose = false; List onlySegments = new ArrayList(); String indexPath = null; + String dirImpl = null; int i = 0; while(i < args.length) { - if (args[i].equals("-fix")) { + String arg = args[i]; + if ("-fix".equals(arg)) { doFix = true; - i++; - } else if (args[i].equals("-codec")) { + } else if ("-codec".equals(arg)) { if (i == args.length-1) { System.out.println("ERROR: missing name for -codec option"); System.exit(1); } - codec = Codec.forName(args[i+1]); - i+=2; - } else if (args[i].equals("-verbose")) { - verbose = true; i++; - } else if (args[i].equals("-segment")) { + codec = Codec.forName(args[i]); + } else if (arg.equals("-verbose")) { + verbose = true; + } else if (arg.equals("-segment")) { if (i == args.length-1) { System.out.println("ERROR: missing name for -segment option"); System.exit(1); } - onlySegments.add(args[i+1]); - i += 2; + i++; + onlySegments.add(args[i]); + } else if ("-dir-impl".equals(arg)) { + if (i == args.length - 1) { + System.out.println("ERROR: missing value for -dir-impl option"); + System.exit(1); + } + i++; + dirImpl = args[i]; } else { if (indexPath != null) { System.out.println("ERROR: unexpected extra argument '" + args[i] + "'"); System.exit(1); } indexPath = args[i]; - i++; } + i++; } if (indexPath == null) { System.out.println("\nERROR: index path not specified"); - System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + + System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y] [-dir-impl X]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -codec X: when fixing, codec to write the new segments_N file with\n" + @@ -1450,7 +1459,8 @@ public class CheckIndex { " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + - "\n" + + " -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " + + "If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + @@ -1480,7 +1490,11 @@ public class CheckIndex { System.out.println("\nOpening index @ " + indexPath + "\n"); Directory dir = null; try { - dir = FSDirectory.open(new File(indexPath)); + if (dirImpl == null) { + dir = FSDirectory.open(new File(indexPath)); + } else { + dir = CommandLineUtil.newFSDirectory(dirImpl, new File(indexPath)); + } } catch (Throwable t) { System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting"); t.printStackTrace(System.out); diff --git a/lucene/src/java/org/apache/lucene/index/IndexReader.java b/lucene/src/java/org/apache/lucene/index/IndexReader.java index 15f0f2562f6..0ae804d766f 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexReader.java @@ -36,6 +36,7 @@ import org.apache.lucene.store.*; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CommandLineUtil; import org.apache.lucene.util.ReaderUtil; // for javadocs /** IndexReader is an abstract class, providing an interface for accessing an @@ -982,17 +983,28 @@ public abstract class IndexReader implements Cloneable,Closeable { public static void main(String [] args) { String filename = null; boolean extract = false; + String dirImpl = null; - for (int i = 0; i < args.length; ++i) { - if (args[i].equals("-extract")) { + int j = 0; + while(j < args.length) { + String arg = args[j]; + if ("-extract".equals(arg)) { extract = true; + } else if ("-dir-impl".equals(arg)) { + if (j == args.length - 1) { + System.out.println("ERROR: missing value for -dir-impl option"); + System.exit(1); + } + j++; + dirImpl = args[j]; } else if (filename == null) { - filename = args[i]; + filename = arg; } + j++; } if (filename == null) { - System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] "); + System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] [-dir-impl X] "); return; } @@ -1004,7 +1016,12 @@ public abstract class IndexReader implements Cloneable,Closeable { File file = new File(filename); String dirname = file.getAbsoluteFile().getParent(); filename = file.getName(); - dir = FSDirectory.open(new File(dirname)); + if (dirImpl == null) { + dir = FSDirectory.open(new File(dirname)); + } else { + dir = CommandLineUtil.newFSDirectory(dirImpl, new File(dirname)); + } + cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false); String [] files = cfr.listAll(); diff --git a/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java b/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java index 396ec5cf178..227d82f6c8e 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.CommandLineUtil; import org.apache.lucene.util.Constants; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; @@ -54,36 +55,56 @@ public final class IndexUpgrader { private static void printUsage() { System.err.println("Upgrades an index so all segments created with a previous Lucene version are rewritten."); System.err.println("Usage:"); - System.err.println(" java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] indexDir"); + System.err.println(" java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] [-dir-impl X] indexDir"); System.err.println("This tool keeps only the last commit in an index; for this"); System.err.println("reason, if the incoming index has more than one commit, the tool"); System.err.println("refuses to run by default. Specify -delete-prior-commits to override"); System.err.println("this, allowing the tool to delete all but the last commit."); + System.err.println("Specify a " + FSDirectory.class.getSimpleName() + + " implementation through the -dir-impl option to force its use. If no package is specified the " + + FSDirectory.class.getPackage().getName() + " package will be used."); System.err.println("WARNING: This tool may reorder document IDs!"); System.exit(1); } @SuppressWarnings("deprecation") public static void main(String[] args) throws IOException { - String dir = null; + String path = null; boolean deletePriorCommits = false; PrintStream out = null; - for (String arg : args) { + String dirImpl = null; + int i = 0; + while (i clazz = loadFSDirectoryClass(clazzName); + return newFSDirectory(clazz, file); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException(FSDirectory.class.getSimpleName() + + " implementation not found: " + clazzName, e); + } catch (ClassCastException e) { + throw new IllegalArgumentException(clazzName + " is not a " + FSDirectory.class.getSimpleName() + + " implementation", e); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException(clazzName + " constructor with " + + File.class.getSimpleName() + " as parameter not found", e); + } catch (Exception e) { + throw new IllegalArgumentException("Error creating " + clazzName + " instance", e); + } + } + + /** + * Loads a specific Directory implementation + * @param className The name of the Directory class to load + * @return The Directory class loaded + * @throws ClassNotFoundException + */ + public static Class loadDirectoryClass(String clazzName) + throws ClassNotFoundException { + return Class.forName(adjustDirectoryClassName(clazzName)).asSubclass(Directory.class); + } + + /** + * Loads a specific FSDirectory implementation + * @param className The name of the FSDirectory class to load + * @return The FSDirectory class loaded + * @throws ClassNotFoundException + */ + public static Class loadFSDirectoryClass(String clazzName) + throws ClassNotFoundException { + return Class.forName(adjustDirectoryClassName(clazzName)).asSubclass(FSDirectory.class); + } + + private static String adjustDirectoryClassName(String clazzName) { + if (clazzName == null || clazzName.trim().length() == 0) { + throw new IllegalArgumentException("The " + FSDirectory.class.getSimpleName() + + " implementation cannot be null or empty"); + } + + if (clazzName.indexOf(".") == -1) {// if not fully qualified, assume .store + clazzName = Directory.class.getPackage().getName() + "." + clazzName; + } + return clazzName; + } + + /** + * Creates a new specific FSDirectory instance + * @param clazz The class of the object to be created + * @param file The file to be used as parameter constructor + * @return The new FSDirectory instance + * @throws NoSuchMethodException + * @throws InstantiationException + * @throws IllegalAccessException + * @throws InvocationTargetException + */ + public static FSDirectory newFSDirectory(Class clazz, File file) + throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException { + // Assuming every FSDirectory has a ctor(File): + Constructor ctor = clazz.getConstructor(File.class); + return ctor.newInstance(file); + } + +} diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java index 9e093847e0c..56a40e568a6 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java @@ -26,7 +26,6 @@ import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import java.lang.reflect.Constructor; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutorService; @@ -1035,24 +1034,16 @@ public abstract class LuceneTestCase extends Assert { fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; } - if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store - fsdirClass = "org.apache.lucene.store." + fsdirClass; - } - Class clazz; try { try { - clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); + clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); } catch (ClassCastException e) { // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; - - if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store - fsdirClass = "org.apache.lucene.store." + fsdirClass; - } - - clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); + clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); } + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f)); if (lf != null) { dir.setLockFactory(lf); @@ -1165,10 +1156,7 @@ public abstract class LuceneTestCase extends Assert { throws IOException { FSDirectory d = null; try { - // Assuming every FSDirectory has a ctor(File), but not all may take a - // LockFactory too, so setting it afterwards. - Constructor ctor = clazz.getConstructor(File.class); - d = ctor.newInstance(file); + d = CommandLineUtil.newFSDirectory(clazz, file); } catch (Exception e) { d = FSDirectory.open(file); } @@ -1186,12 +1174,12 @@ public abstract class LuceneTestCase extends Assert { } static Directory newDirectoryImpl(Random random, String clazzName) { - if (clazzName.equals("random")) + if (clazzName.equals("random")) { clazzName = randomDirectory(random); - if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store - clazzName = "org.apache.lucene.store." + clazzName; + } + try { - final Class clazz = Class.forName(clazzName).asSubclass(Directory.class); + final Class clazz = CommandLineUtil.loadDirectoryClass(clazzName); // If it is a FSDirectory type, try its ctor(File) if (FSDirectory.class.isAssignableFrom(clazz)) { final File dir = _TestUtil.getTempDir("index");