Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3586: allow specifying -dir-impl FSDirectory subclass to CheckIndex, IndexUpgrader 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1213800 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2011-12-13 17:34:58 +00:00
parent 0d86474d3c
commit d6a96a43d9
6 changed files with 212 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -716,6 +716,10 @@ New Features
* LUCENE-3593: Added a FieldValueFilter that accepts all documents that either
have at least one or no value at all in a specific field. (Simon Willnauer,
Uwe Schindler, Robert Muir)
* LUCENE-3586: CheckIndex and IndexUpgrader allow you to specify the
specific FSDirectory implementation to use (with the new -dir-impl
command-line option). (Luca Cavanna via Mike McCandless)
Bug fixes

60
lucene/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -17,15 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import org.apache.lucene.document.FieldType; // for javadocs
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.codecs.Codec;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
@ -37,6 +28,16 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.document.FieldType; // for javadocs
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
@ -44,6 +45,7 @@ import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.StringHelper;
@ -1408,41 +1410,48 @@ public class CheckIndex {
boolean verbose = false;
List<String> onlySegments = new ArrayList<String>();
String indexPath = null;
String dirImpl = null;
int i = 0;
while(i < args.length) {
if (args[i].equals("-fix")) {
String arg = args[i];
if ("-fix".equals(arg)) {
doFix = true;
i++;
} else if (args[i].equals("-codec")) {
} else if ("-codec".equals(arg)) {
if (i == args.length-1) {
System.out.println("ERROR: missing name for -codec option");
System.exit(1);
}
codec = Codec.forName(args[i+1]);
i+=2;
} else if (args[i].equals("-verbose")) {
verbose = true;
i++;
} else if (args[i].equals("-segment")) {
codec = Codec.forName(args[i]);
} else if (arg.equals("-verbose")) {
verbose = true;
} else if (arg.equals("-segment")) {
if (i == args.length-1) {
System.out.println("ERROR: missing name for -segment option");
System.exit(1);
}
onlySegments.add(args[i+1]);
i += 2;
i++;
onlySegments.add(args[i]);
} else if ("-dir-impl".equals(arg)) {
if (i == args.length - 1) {
System.out.println("ERROR: missing value for -dir-impl option");
System.exit(1);
}
i++;
dirImpl = args[i];
} else {
if (indexPath != null) {
System.out.println("ERROR: unexpected extra argument '" + args[i] + "'");
System.exit(1);
}
indexPath = args[i];
i++;
}
i++;
}
if (indexPath == null) {
System.out.println("\nERROR: index path not specified");
System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" +
System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y] [-dir-impl X]\n" +
"\n" +
" -fix: actually write a new segments_N file, removing any problematic segments\n" +
" -codec X: when fixing, codec to write the new segments_N file with\n" +
@ -1450,7 +1459,8 @@ public class CheckIndex {
" -segment X: only check the specified segments. This can be specified multiple\n" +
" times, to check more than one segment, eg '-segment _2 -segment _a'.\n" +
" You can't use this with the -fix option\n" +
"\n" +
" -dir-impl X: use a specific " + FSDirectory.class.getSimpleName() + " implementation. " +
"If no package is specified the " + FSDirectory.class.getPackage().getName() + " package will be used.\n" +
"**WARNING**: -fix should only be used on an emergency basis as it will cause\n" +
"documents (perhaps many) to be permanently removed from the index. Always make\n" +
"a backup copy of your index before running this! Do not run this tool on an index\n" +
@ -1480,7 +1490,11 @@ public class CheckIndex {
System.out.println("\nOpening index @ " + indexPath + "\n");
Directory dir = null;
try {
dir = FSDirectory.open(new File(indexPath));
if (dirImpl == null) {
dir = FSDirectory.open(new File(indexPath));
} else {
dir = CommandLineUtil.newFSDirectory(dirImpl, new File(indexPath));
}
} catch (Throwable t) {
System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting");
t.printStackTrace(System.out);

27
lucene/src/java/org/apache/lucene/index/IndexReader.java
View File

@ -36,6 +36,7 @@ import org.apache.lucene.store.*;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.ReaderUtil; // for javadocs
/** IndexReader is an abstract class, providing an interface for accessing an
@ -982,17 +983,28 @@ public abstract class IndexReader implements Cloneable,Closeable {
public static void main(String [] args) {
String filename = null;
boolean extract = false;
String dirImpl = null;
for (int i = 0; i < args.length; ++i) {
if (args[i].equals("-extract")) {
int j = 0;
while(j < args.length) {
String arg = args[j];
if ("-extract".equals(arg)) {
extract = true;
} else if ("-dir-impl".equals(arg)) {
if (j == args.length - 1) {
System.out.println("ERROR: missing value for -dir-impl option");
System.exit(1);
}
j++;
dirImpl = args[j];
} else if (filename == null) {
filename = args[i];
filename = arg;
}
j++;
}
if (filename == null) {
System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] <cfsfile>");
System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] [-dir-impl X] <cfsfile>");
return;
}
@ -1004,7 +1016,12 @@ public abstract class IndexReader implements Cloneable,Closeable {
File file = new File(filename);
String dirname = file.getAbsoluteFile().getParent();
filename = file.getName();
dir = FSDirectory.open(new File(dirname));
if (dirImpl == null) {
dir = FSDirectory.open(new File(dirname));
} else {
dir = CommandLineUtil.newFSDirectory(dirImpl, new File(dirname));
}
cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false);
String [] files = cfr.listAll();

37
lucene/src/java/org/apache/lucene/index/IndexUpgrader.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.Version;
@ -54,36 +55,56 @@ public final class IndexUpgrader {
private static void printUsage() {
System.err.println("Upgrades an index so all segments created with a previous Lucene version are rewritten.");
System.err.println("Usage:");
System.err.println(" java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] indexDir");
System.err.println(" java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] [-dir-impl X] indexDir");
System.err.println("This tool keeps only the last commit in an index; for this");
System.err.println("reason, if the incoming index has more than one commit, the tool");
System.err.println("refuses to run by default. Specify -delete-prior-commits to override");
System.err.println("this, allowing the tool to delete all but the last commit.");
System.err.println("Specify a " + FSDirectory.class.getSimpleName() +
" implementation through the -dir-impl option to force its use. If no package is specified the "
+ FSDirectory.class.getPackage().getName() + " package will be used.");
System.err.println("WARNING: This tool may reorder document IDs!");
System.exit(1);
}
@SuppressWarnings("deprecation")
public static void main(String[] args) throws IOException {
String dir = null;
String path = null;
boolean deletePriorCommits = false;
PrintStream out = null;
for (String arg : args) {
String dirImpl = null;
int i = 0;
while (i<args.length) {
String arg = args[i];
if ("-delete-prior-commits".equals(arg)) {
deletePriorCommits = true;
} else if ("-verbose".equals(arg)) {
out = System.out;
} else if (dir == null) {
dir = arg;
} else {
} else if (path == null) {
path = arg;
} else if ("-dir-impl".equals(arg)) {
if (i == args.length - 1) {
System.out.println("ERROR: missing value for -dir-impl option");
System.exit(1);
}
i++;
dirImpl = args[i];
}else {
printUsage();
}
i++;
}
if (dir == null) {
if (path == null) {
printUsage();
}
new IndexUpgrader(FSDirectory.open(new File(dir)), Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
Directory dir = null;
if (dirImpl == null) {
dir = FSDirectory.open(new File(path));
} else {
dir = CommandLineUtil.newFSDirectory(dirImpl, new File(path));
}
new IndexUpgrader(dir, Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
}
private final Directory dir;

112
lucene/src/java/org/apache/lucene/util/CommandLineUtil.java Normal file
View File

@ -0,0 +1,112 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
/**
* Class containing some useful methods used by command line tools
*
*/
public final class CommandLineUtil {
private CommandLineUtil() {
}
/**
* Creates a specific FSDirectory instance starting from its class name
* @param className The name of the FSDirectory class to load
* @param file The file to be used as parameter constructor
* @return the new FSDirectory instance
*/
public static FSDirectory newFSDirectory(String clazzName, File file) {
try {
final Class<? extends FSDirectory> clazz = loadFSDirectoryClass(clazzName);
return newFSDirectory(clazz, file);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException(FSDirectory.class.getSimpleName()
+ " implementation not found: " + clazzName, e);
} catch (ClassCastException e) {
throw new IllegalArgumentException(clazzName + " is not a " + FSDirectory.class.getSimpleName()
+ " implementation", e);
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException(clazzName + " constructor with "
+ File.class.getSimpleName() + " as parameter not found", e);
} catch (Exception e) {
throw new IllegalArgumentException("Error creating " + clazzName + " instance", e);
}
}
/**
* Loads a specific Directory implementation
* @param className The name of the Directory class to load
* @return The Directory class loaded
* @throws ClassNotFoundException
*/
public static Class<? extends Directory> loadDirectoryClass(String clazzName)
throws ClassNotFoundException {
return Class.forName(adjustDirectoryClassName(clazzName)).asSubclass(Directory.class);
}
/**
* Loads a specific FSDirectory implementation
* @param className The name of the FSDirectory class to load
* @return The FSDirectory class loaded
* @throws ClassNotFoundException
*/
public static Class<? extends FSDirectory> loadFSDirectoryClass(String clazzName)
throws ClassNotFoundException {
return Class.forName(adjustDirectoryClassName(clazzName)).asSubclass(FSDirectory.class);
}
private static String adjustDirectoryClassName(String clazzName) {
if (clazzName == null || clazzName.trim().length() == 0) {
throw new IllegalArgumentException("The " + FSDirectory.class.getSimpleName()
+ " implementation cannot be null or empty");
}
if (clazzName.indexOf(".") == -1) {// if not fully qualified, assume .store
clazzName = Directory.class.getPackage().getName() + "." + clazzName;
}
return clazzName;
}
/**
* Creates a new specific FSDirectory instance
* @param clazz The class of the object to be created
* @param file The file to be used as parameter constructor
* @return The new FSDirectory instance
* @throws NoSuchMethodException
* @throws InstantiationException
* @throws IllegalAccessException
* @throws InvocationTargetException
*/
public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, File file)
throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException {
// Assuming every FSDirectory has a ctor(File):
Constructor<? extends FSDirectory> ctor = clazz.getConstructor(File.class);
return ctor.newInstance(file);
}
}

28
lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -26,7 +26,6 @@ import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.Constructor;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
@ -1035,24 +1034,16 @@ public abstract class LuceneTestCase extends Assert {
fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)];
}
if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store
fsdirClass = "org.apache.lucene.store." + fsdirClass;
}
Class<? extends FSDirectory> clazz;
try {
try {
clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class);
clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass);
} catch (ClassCastException e) {
// TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random
fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)];
if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store
fsdirClass = "org.apache.lucene.store." + fsdirClass;
}
clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class);
clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass);
}
MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f));
if (lf != null) {
dir.setLockFactory(lf);
@ -1165,10 +1156,7 @@ public abstract class LuceneTestCase extends Assert {
throws IOException {
FSDirectory d = null;
try {
// Assuming every FSDirectory has a ctor(File), but not all may take a
// LockFactory too, so setting it afterwards.
Constructor<? extends FSDirectory> ctor = clazz.getConstructor(File.class);
d = ctor.newInstance(file);
d = CommandLineUtil.newFSDirectory(clazz, file);
} catch (Exception e) {
d = FSDirectory.open(file);
}
@ -1186,12 +1174,12 @@ public abstract class LuceneTestCase extends Assert {
}
static Directory newDirectoryImpl(Random random, String clazzName) {
if (clazzName.equals("random"))
if (clazzName.equals("random")) {
clazzName = randomDirectory(random);
if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store
clazzName = "org.apache.lucene.store." + clazzName;
}
try {
final Class<? extends Directory> clazz = Class.forName(clazzName).asSubclass(Directory.class);
final Class<? extends Directory> clazz = CommandLineUtil.loadDirectoryClass(clazzName);
// If it is a FSDirectory type, try its ctor(File)
if (FSDirectory.class.isAssignableFrom(clazz)) {
final File dir = _TestUtil.getTempDir("index");