LUCENE-5953: Restructure Directory and LockFactory APIs

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1637665 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2014-11-09 12:18:03 +00:00
parent f7e6d2f177
commit c429f437f0
55 changed files with 628 additions and 1250 deletions

View File

@ -174,6 +174,13 @@ API Changes
* LUCENE-6021: FixedBitSet.nextSetBit now returns DocIdSetIterator.NO_MORE_DOCS * LUCENE-6021: FixedBitSet.nextSetBit now returns DocIdSetIterator.NO_MORE_DOCS
instead of -1 when there are no more bits which are set. (Adrien Grand) instead of -1 when there are no more bits which are set. (Adrien Grand)
* LUCENE-5953: Directory and LockFactory APIs were restructured: Locking is
now under the responsibility of the Directory implementation. LockFactory is
only used by subclasses of BaseDirectory to delegate locking to an impl
class. LockFactories are now singletons and are responsible to create a Lock
instance based on a Directory implementation passed to the factory method.
See MIGRATE.txt for more details. (Uwe Schindler, Robert Muir)
Bug Fixes Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary * LUCENE-5650: Enforce read-only access to any path outside the temporary

View File

@ -32,7 +32,6 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.MergeState.CheckAbort; import org.apache.lucene.index.MergeState.CheckAbort;
import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.BaseDirectory;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
@ -103,7 +102,7 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
endOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLEEND)); endOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLEEND));
} }
return new BaseDirectory() { return new Directory() {
private int getIndex(String name) throws IOException { private int getIndex(String name) throws IOException {
int index = Arrays.binarySearch(fileNames, name); int index = Arrays.binarySearch(fileNames, name);
@ -135,7 +134,6 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
@Override @Override
public void close() throws IOException { public void close() throws IOException {
isOpen = false;
in.close(); in.close();
} }
@ -155,9 +153,6 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
@Override @Override
public Lock makeLock(String name) { throw new UnsupportedOperationException(); } public Lock makeLock(String name) { throw new UnsupportedOperationException(); }
@Override
public void clearLock(String name) { throw new UnsupportedOperationException(); }
}; };
} }

View File

@ -21,7 +21,6 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.BaseDirectory;
import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
@ -43,7 +42,7 @@ import java.io.IOException;
* Directory methods that would normally modify data throw an exception. * Directory methods that would normally modify data throw an exception.
* @lucene.experimental * @lucene.experimental
*/ */
final class Lucene50CompoundReader extends BaseDirectory { final class Lucene50CompoundReader extends Directory {
/** Offset/Length for a slice inside of a compound file */ /** Offset/Length for a slice inside of a compound file */
public static final class FileEntry { public static final class FileEntry {
@ -84,7 +83,6 @@ final class Lucene50CompoundReader extends BaseDirectory {
IOUtils.closeWhileHandlingException(handle); IOUtils.closeWhileHandlingException(handle);
} }
} }
this.isOpen = true;
} }
/** Helper method that reads CFS entries from an input stream */ /** Helper method that reads CFS entries from an input stream */
@ -119,7 +117,6 @@ final class Lucene50CompoundReader extends BaseDirectory {
@Override @Override
public void close() throws IOException { public void close() throws IOException {
isOpen = false;
IOUtils.close(handle); IOUtils.close(handle);
} }
@ -186,11 +183,6 @@ final class Lucene50CompoundReader extends BaseDirectory {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public void clearLock(String name) throws IOException {
throw new UnsupportedOperationException();
}
@Override @Override
public String toString() { public String toString() {
return "CompoundFileDirectory(segment=\"" + segmentName + "\" in dir=" + directory + ")"; return "CompoundFileDirectory(segment=\"" + segmentName + "\" in dir=" + directory + ")";

View File

@ -17,10 +17,9 @@ package org.apache.lucene.store;
* limitations under the License. * limitations under the License.
*/ */
import java.io.IOException;
/** /**
* Base implementation for a concrete {@link Directory}. * Base implementation for a concrete {@link Directory} that uses a {@link LockFactory} for locking.
* @lucene.experimental * @lucene.experimental
*/ */
public abstract class BaseDirectory extends Directory { public abstract class BaseDirectory extends Directory {
@ -29,35 +28,20 @@ public abstract class BaseDirectory extends Directory {
/** Holds the LockFactory instance (implements locking for /** Holds the LockFactory instance (implements locking for
* this Directory instance). */ * this Directory instance). */
protected LockFactory lockFactory; protected final LockFactory lockFactory;
/** Sole constructor. */ /** Sole constructor. */
protected BaseDirectory() { protected BaseDirectory(LockFactory lockFactory) {
super(); super();
} if (lockFactory == null) {
throw new NullPointerException("LockFactory cannot be null, use an explicit instance!");
@Override
public Lock makeLock(String name) {
return lockFactory.makeLock(name);
}
@Override
public void clearLock(String name) throws IOException {
if (lockFactory != null) {
lockFactory.clearLock(name);
} }
}
@Override
public void setLockFactory(LockFactory lockFactory) throws IOException {
assert lockFactory != null;
this.lockFactory = lockFactory; this.lockFactory = lockFactory;
lockFactory.setLockPrefix(this.getLockID());
} }
@Override @Override
public LockFactory getLockFactory() { public final Lock makeLock(String name) {
return this.lockFactory; return lockFactory.makeLock(this, name);
} }
@Override @Override
@ -66,4 +50,9 @@ public abstract class BaseDirectory extends Directory {
throw new AlreadyClosedException("this Directory is closed"); throw new AlreadyClosedException("this Directory is closed");
} }
@Override
public String toString() {
return super.toString() + " lockFactory=" + lockFactory;
}
} }

View File

@ -37,8 +37,7 @@ import org.apache.lucene.util.IOUtils;
* </ul> * </ul>
* *
* Directory locking is implemented by an instance of {@link * Directory locking is implemented by an instance of {@link
* LockFactory}, and can be changed for each Directory * LockFactory}.
* instance using {@link #setLockFactory}.
* *
*/ */
public abstract class Directory implements Closeable { public abstract class Directory implements Closeable {
@ -116,53 +115,14 @@ public abstract class Directory implements Closeable {
*/ */
public abstract Lock makeLock(String name); public abstract Lock makeLock(String name);
/**
* Attempt to clear (forcefully unlock and remove) the
* specified lock. Only call this at a time when you are
* certain this lock is no longer in use.
* @param name name of the lock to be cleared.
*/
public abstract void clearLock(String name) throws IOException;
/** Closes the store. */ /** Closes the store. */
@Override @Override
public abstract void close() public abstract void close()
throws IOException; throws IOException;
/**
* Set the LockFactory that this Directory instance should
* use for its locking implementation. Each * instance of
* LockFactory should only be used for one directory (ie,
* do not share a single instance across multiple
* Directories).
*
* @param lockFactory instance of {@link LockFactory}.
*/
public abstract void setLockFactory(LockFactory lockFactory) throws IOException;
/**
* Get the LockFactory that this Directory instance is
* using for its locking implementation. Note that this
* may be null for Directory implementations that provide
* their own locking implementation.
*/
public abstract LockFactory getLockFactory();
/**
* Return a string identifier that uniquely differentiates
* this Directory instance from other Directory instances.
* This ID should be the same if two Directory instances
* (even in different JVMs and/or on different machines)
* are considered "the same index". This is how locking
* "scopes" to the right index.
*/
public String getLockID() {
return this.toString();
}
@Override @Override
public String toString() { public String toString() {
return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode()) + " lockFactory=" + getLockFactory(); return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode());
} }
/** /**

View File

@ -125,15 +125,9 @@ public abstract class FSDirectory extends BaseDirectory {
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
protected FSDirectory(Path path, LockFactory lockFactory) throws IOException { protected FSDirectory(Path path, LockFactory lockFactory) throws IOException {
// new ctors use always NativeFSLockFactory as default: super(lockFactory);
if (lockFactory == null) {
lockFactory = new NativeFSLockFactory();
}
Files.createDirectories(path); // create directory, if it doesnt exist Files.createDirectories(path); // create directory, if it doesnt exist
directory = path.toRealPath(); directory = path.toRealPath();
setLockFactory(lockFactory);
} }
/** Creates an FSDirectory instance, trying to pick the /** Creates an FSDirectory instance, trying to pick the
@ -157,7 +151,7 @@ public abstract class FSDirectory extends BaseDirectory {
* *
* <p>See <a href="#subclasses">above</a> */ * <p>See <a href="#subclasses">above</a> */
public static FSDirectory open(Path path) throws IOException { public static FSDirectory open(Path path) throws IOException {
return open(path, null); return open(path, FSLockFactory.getDefault());
} }
/** Just like {@link #open(Path)}, but allows you to /** Just like {@link #open(Path)}, but allows you to
@ -172,26 +166,6 @@ public abstract class FSDirectory extends BaseDirectory {
} }
} }
@Override
public void setLockFactory(LockFactory lockFactory) throws IOException {
super.setLockFactory(lockFactory);
// for filesystem based LockFactory, delete the lockPrefix, if the locks are placed
// in index dir. If no index dir is given, set ourselves
if (lockFactory instanceof FSLockFactory) {
final FSLockFactory lf = (FSLockFactory) lockFactory;
final Path dir = lf.getLockDir();
// if the lock factory has no lockDir set, use the this directory as lockDir
if (dir == null) {
lf.setLockDir(directory);
lf.setLockPrefix(null);
} else if (dir.toRealPath().equals(directory)) {
lf.setLockPrefix(null);
}
}
}
/** Lists all files (not subdirectories) in the /** Lists all files (not subdirectories) in the
* directory. * directory.
* *
@ -280,19 +254,6 @@ public abstract class FSDirectory extends BaseDirectory {
IOUtils.fsync(directory, true); IOUtils.fsync(directory, true);
} }
@Override
public String getLockID() {
ensureOpen();
String dirName = directory.toString(); // name to be hashed
int digest = 0;
for(int charIDX=0;charIDX<dirName.length();charIDX++) {
final char ch = dirName.charAt(charIDX);
digest = 31 * digest + ch;
}
return "lucene-" + Integer.toHexString(digest);
}
/** Closes the store to future operations. */ /** Closes the store to future operations. */
@Override @Override
public synchronized void close() { public synchronized void close() {
@ -308,7 +269,7 @@ public abstract class FSDirectory extends BaseDirectory {
/** For debug output. */ /** For debug output. */
@Override @Override
public String toString() { public String toString() {
return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + getLockFactory(); return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
} }
final class FSIndexOutput extends OutputStreamIndexOutput { final class FSIndexOutput extends OutputStreamIndexOutput {

View File

@ -17,47 +17,29 @@ package org.apache.lucene.store;
* limitations under the License. * limitations under the License.
*/ */
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
/** /**
* Base class for file system based locking implementation. * Base class for file system based locking implementation.
* This class is explicitly checking that the passed {@link Directory}
* is an {@link FSDirectory}.
*/ */
public abstract class FSLockFactory extends LockFactory { public abstract class FSLockFactory extends LockFactory {
/**
* Directory for the lock files.
*/
protected Path lockDir = null;
/**
* Set the lock directory. This method can be only called
* once to initialize the lock directory. It is used by {@link FSDirectory}
* to set the lock directory to itself.
* Subclasses can also use this method to set the directory
* in the constructor.
*/
protected final void setLockDir(Path lockDir) throws IOException {
if (this.lockDir != null)
throw new IllegalStateException("You can set the lock directory for this factory only once.");
if (lockDir != null) {
Files.createDirectories(lockDir);
}
this.lockDir = lockDir;
}
/** /** Returns the default locking implementation for this platform.
* Retrieve the lock directory. * This method currently returns always {@link NativeFSLockFactory}.
*/ */
public Path getLockDir() { public static final FSLockFactory getDefault() {
return lockDir; return NativeFSLockFactory.INSTANCE;
} }
@Override @Override
public String toString() { public final Lock makeLock(Directory dir, String lockName) {
return this.getClass().getSimpleName() + "@" + lockDir; if (!(dir instanceof FSDirectory)) {
throw new UnsupportedOperationException(getClass().getSimpleName() + " can only be used with FSDirectory subclasses, got: " + dir);
}
return makeFSLock((FSDirectory) dir, lockName);
} }
/** Implement this method to create a lock for a FSDirectory instance. */
protected abstract Lock makeFSLock(FSDirectory dir, String lockName);
} }

View File

@ -20,13 +20,14 @@ package org.apache.lucene.store;
import java.io.IOException; import java.io.IOException;
import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.NoSuchFileException; import java.nio.file.NoSuchFileException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.HashSet; import java.util.HashSet;
import org.apache.lucene.util.IOUtils;
/** /**
* Expert: A Directory instance that switches files between * Expert: A Directory instance that switches files between
@ -38,10 +39,15 @@ import java.util.HashSet;
* to this class, and must allow multiple threads to call * to this class, and must allow multiple threads to call
* contains at once.</p> * contains at once.</p>
* *
* <p>Locks with a name having the specified extensions are
* delegated to the primary directory; others are delegated
* to the secondary directory. Ideally, both Directory
* instances should use the same lock factory.</p>
*
* @lucene.experimental * @lucene.experimental
*/ */
public class FileSwitchDirectory extends BaseDirectory { public class FileSwitchDirectory extends Directory {
private final Directory secondaryDir; private final Directory secondaryDir;
private final Directory primaryDir; private final Directory primaryDir;
private final Set<String> primaryExtensions; private final Set<String> primaryExtensions;
@ -52,7 +58,6 @@ public class FileSwitchDirectory extends BaseDirectory {
this.primaryDir = primaryDir; this.primaryDir = primaryDir;
this.secondaryDir = secondaryDir; this.secondaryDir = secondaryDir;
this.doClose = doClose; this.doClose = doClose;
this.lockFactory = primaryDir.getLockFactory();
} }
/** Return the primary directory */ /** Return the primary directory */
@ -65,14 +70,15 @@ public class FileSwitchDirectory extends BaseDirectory {
return secondaryDir; return secondaryDir;
} }
@Override
public Lock makeLock(String name) {
return getDirectory(name).makeLock(name);
}
@Override @Override
public void close() throws IOException { public void close() throws IOException {
if (doClose) { if (doClose) {
try { IOUtils.close(primaryDir, secondaryDir);
secondaryDir.close();
} finally {
primaryDir.close();
}
doClose = false; doClose = false;
} }
} }

View File

@ -85,31 +85,11 @@ public class FilterDirectory extends Directory {
return in.makeLock(name); return in.makeLock(name);
} }
@Override
public void clearLock(String name) throws IOException {
in.clearLock(name);
}
@Override @Override
public void close() throws IOException { public void close() throws IOException {
in.close(); in.close();
} }
@Override
public void setLockFactory(LockFactory lockFactory) throws IOException {
in.setLockFactory(lockFactory);
}
@Override
public String getLockID() {
return in.getLockID();
}
@Override
public LockFactory getLockFactory() {
return in.getLockFactory();
}
@Override @Override
public String toString() { public String toString() {
return getClass().getSimpleName() + "(" + in.toString() + ")"; return getClass().getSimpleName() + "(" + in.toString() + ")";

View File

@ -17,7 +17,6 @@ package org.apache.lucene.store;
* limitations under the License. * limitations under the License.
*/ */
import java.io.IOException;
/** /**
* <p>Base class for Locking implementation. {@link Directory} uses * <p>Base class for Locking implementation. {@link Directory} uses
@ -46,40 +45,10 @@ import java.io.IOException;
public abstract class LockFactory { public abstract class LockFactory {
protected String lockPrefix = null;
/**
* Set the prefix in use for all locks created in this
* LockFactory. This is normally called once, when a
* Directory gets this LockFactory instance. However, you
* can also call this (after this instance is assigned to
* a Directory) to override the prefix in use. This
* is helpful if you're running Lucene on machines that
* have different mount points for the same shared
* directory.
*/
public void setLockPrefix(String lockPrefix) {
this.lockPrefix = lockPrefix;
}
/**
* Get the prefix in use for all locks created in this LockFactory.
*/
public String getLockPrefix() {
return this.lockPrefix;
}
/** /**
* Return a new Lock instance identified by lockName. * Return a new Lock instance identified by lockName.
* @param lockName name of the lock to be created. * @param lockName name of the lock to be created.
*/ */
public abstract Lock makeLock(String lockName); public abstract Lock makeLock(Directory dir, String lockName);
/**
* Attempt to clear (forcefully unlock and remove) the
* specified lock. Only call this at a time when you are
* certain this lock is no longer in use.
* @param lockName name of the lock to be cleared.
*/
abstract public void clearLock(String lockName) throws IOException;
} }

View File

@ -45,8 +45,8 @@ public class LockStressTest {
" myID = int from 0 .. 255 (should be unique for test process)\n" + " myID = int from 0 .. 255 (should be unique for test process)\n" +
" verifierHost = hostname that LockVerifyServer is listening on\n" + " verifierHost = hostname that LockVerifyServer is listening on\n" +
" verifierPort = port that LockVerifyServer is listening on\n" + " verifierPort = port that LockVerifyServer is listening on\n" +
" lockFactoryClassName = primary LockFactory class that we will use\n" + " lockFactoryClassName = primary FSLockFactory class that we will use\n" +
" lockDirName = path to the lock directory (only set for Simple/NativeFSLockFactory\n" + " lockDirName = path to the lock directory\n" +
" sleepTimeMS = milliseconds to pause betweeen each lock obtain/release\n" + " sleepTimeMS = milliseconds to pause betweeen each lock obtain/release\n" +
" count = number of locking tries\n" + " count = number of locking tries\n" +
"\n" + "\n" +
@ -69,11 +69,13 @@ public class LockStressTest {
final String verifierHost = args[arg++]; final String verifierHost = args[arg++];
final int verifierPort = Integer.parseInt(args[arg++]); final int verifierPort = Integer.parseInt(args[arg++]);
final String lockFactoryClassName = args[arg++]; final String lockFactoryClassName = args[arg++];
final String lockDirName = args[arg++]; final Path lockDirPath = Paths.get(args[arg++]);
final int sleepTimeMS = Integer.parseInt(args[arg++]); final int sleepTimeMS = Integer.parseInt(args[arg++]);
final int count = Integer.parseInt(args[arg++]); final int count = Integer.parseInt(args[arg++]);
final LockFactory lockFactory = getNewLockFactory(lockFactoryClassName, lockDirName); final LockFactory lockFactory = getNewLockFactory(lockFactoryClassName);
// we test the lock factory directly, so we don't need it on the directory itsself (the directory is just for testing)
final FSDirectory lockDir = new SimpleFSDirectory(lockDirPath, NoLockFactory.INSTANCE);
final InetSocketAddress addr = new InetSocketAddress(verifierHost, verifierPort); final InetSocketAddress addr = new InetSocketAddress(verifierHost, verifierPort);
System.out.println("Connecting to server " + addr + System.out.println("Connecting to server " + addr +
" and registering as client " + myID + "..."); " and registering as client " + myID + "...");
@ -86,7 +88,7 @@ public class LockStressTest {
out.write(myID); out.write(myID);
out.flush(); out.flush();
LockFactory verifyLF = new VerifyingLockFactory(lockFactory, in, out); LockFactory verifyLF = new VerifyingLockFactory(lockFactory, in, out);
Lock l = verifyLF.makeLock("test.lock"); Lock l = verifyLF.makeLock(lockDir, "test.lock");
final Random rnd = new Random(); final Random rnd = new Random();
// wait for starting gun // wait for starting gun
@ -103,9 +105,9 @@ public class LockStressTest {
if (obtained) { if (obtained) {
if (rnd.nextInt(10) == 0) { if (rnd.nextInt(10) == 0) {
if (rnd.nextBoolean()) { if (rnd.nextBoolean()) {
verifyLF = new VerifyingLockFactory(getNewLockFactory(lockFactoryClassName, lockDirName), in, out); verifyLF = new VerifyingLockFactory(getNewLockFactory(lockFactoryClassName), in, out);
} }
final Lock secondLock = verifyLF.makeLock("test.lock"); final Lock secondLock = verifyLF.makeLock(lockDir, "test.lock");
if (secondLock.obtain()) { if (secondLock.obtain()) {
throw new IOException("Double Obtain"); throw new IOException("Double Obtain");
} }
@ -126,20 +128,21 @@ public class LockStressTest {
} }
private static LockFactory getNewLockFactory(String lockFactoryClassName, String lockDirName) throws IOException { private static FSLockFactory getNewLockFactory(String lockFactoryClassName) throws IOException {
LockFactory lockFactory; // try to get static INSTANCE field of class
try { try {
lockFactory = Class.forName(lockFactoryClassName).asSubclass(LockFactory.class).newInstance(); return (FSLockFactory) Class.forName(lockFactoryClassName).getField("INSTANCE").get(null);
} catch (ReflectiveOperationException e) {
// fall-through
}
// try to create a new instance
try {
return Class.forName(lockFactoryClassName).asSubclass(FSLockFactory.class).newInstance();
} catch (IllegalAccessException | InstantiationException | ClassCastException | ClassNotFoundException e) { } catch (IllegalAccessException | InstantiationException | ClassCastException | ClassNotFoundException e) {
throw new IOException("Cannot instantiate lock factory " + lockFactoryClassName); // fall-through
} }
Path lockDir = Paths.get(lockDirName); throw new IOException("Cannot get lock factory singleton of " + lockFactoryClassName);
if (lockFactory instanceof FSLockFactory) {
((FSLockFactory) lockFactory).setLockDir(lockDir);
}
lockFactory.setLockPrefix("test");
return lockFactory;
} }
} }

View File

@ -24,7 +24,6 @@ import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode; import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.security.AccessController; import java.security.AccessController;
import java.security.PrivilegedExceptionAction; import java.security.PrivilegedExceptionAction;
import java.security.PrivilegedActionException; import java.security.PrivilegedActionException;
@ -85,27 +84,37 @@ public class MMapDirectory extends FSDirectory {
* Default max chunk size. * Default max chunk size.
* @see #MMapDirectory(Path, LockFactory, int) * @see #MMapDirectory(Path, LockFactory, int)
*/ */
public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? (1 << 30) : (1 << 28); public static final int DEFAULT_MAX_CHUNK_SIZE = Constants.JRE_IS_64BIT ? (1 << 30) : (1 << 28);
final int chunkSizePower; final int chunkSizePower;
/** Create a new MMapDirectory for the named location. /** Create a new MMapDirectory for the named location.
* *
* @param path the path of the directory * @param path the path of the directory
* @param lockFactory the lock factory to use, or null for the default * @param lockFactory the lock factory to use
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public MMapDirectory(Path path, LockFactory lockFactory) throws IOException { public MMapDirectory(Path path, LockFactory lockFactory) throws IOException {
this(path, lockFactory, DEFAULT_MAX_BUFF); this(path, lockFactory, DEFAULT_MAX_CHUNK_SIZE);
} }
/** Create a new MMapDirectory for the named location and {@link NativeFSLockFactory}. /** Create a new MMapDirectory for the named location and {@link FSLockFactory#getDefault()}.
* *
* @param path the path of the directory * @param path the path of the directory
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public MMapDirectory(Path path) throws IOException { public MMapDirectory(Path path) throws IOException {
this(path, null); this(path, FSLockFactory.getDefault());
}
/** Create a new MMapDirectory for the named location and {@link FSLockFactory#getDefault()}.
*
* @param path the path of the directory
* @param maxChunkSize maximum chunk size (default is 1 GiBytes for
* 64 bit JVMs and 256 MiBytes for 32 bit JVMs) used for memory mapping.
* @throws IOException if there is a low-level I/O error
*/
public MMapDirectory(Path path, int maxChunkSize) throws IOException {
this(path, FSLockFactory.getDefault(), maxChunkSize);
} }
/** /**

View File

@ -56,21 +56,20 @@ public class NIOFSDirectory extends FSDirectory {
/** Create a new NIOFSDirectory for the named location. /** Create a new NIOFSDirectory for the named location.
* *
* @param path the path of the directory * @param path the path of the directory
* @param lockFactory the lock factory to use, or null for the default * @param lockFactory the lock factory to use
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public NIOFSDirectory(Path path, LockFactory lockFactory) throws IOException { public NIOFSDirectory(Path path, LockFactory lockFactory) throws IOException {
super(path, lockFactory); super(path, lockFactory);
} }
/** Create a new NIOFSDirectory for the named location and {@link NativeFSLockFactory}. /** Create a new NIOFSDirectory for the named location and {@link FSLockFactory#getDefault()}.
* *
* @param path the path of the directory * @param path the path of the directory
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public NIOFSDirectory(Path path) throws IOException { public NIOFSDirectory(Path path) throws IOException {
super(path, null); this(path, FSLockFactory.getDefault());
} }
/** Creates an IndexInput for the file with the given name. */ /** Creates an IndexInput for the file with the given name. */

View File

@ -23,7 +23,6 @@ import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
@ -66,167 +65,149 @@ import org.apache.lucene.util.IOUtils;
* not working properly in your environment, you can easily * not working properly in your environment, you can easily
* test it by using {@link VerifyingLockFactory}, {@link * test it by using {@link VerifyingLockFactory}, {@link
* LockVerifyServer} and {@link LockStressTest}.</p> * LockVerifyServer} and {@link LockStressTest}.</p>
*
* <p>This is a singleton, you have to use {@link #INSTANCE}.
* *
* @see LockFactory * @see LockFactory
*/ */
public class NativeFSLockFactory extends FSLockFactory { public final class NativeFSLockFactory extends FSLockFactory {
/** /**
* Create a NativeFSLockFactory instance, with null (unset) * Singleton instance
* lock directory. When you pass this factory to a {@link FSDirectory}
* subclass, the lock directory is automatically set to the
* directory itself. Be sure to create one instance for each directory
* your create!
*/ */
public NativeFSLockFactory() throws IOException { public static final NativeFSLockFactory INSTANCE = new NativeFSLockFactory();
this((Path) null);
}
/** private NativeFSLockFactory() {}
* Create a NativeFSLockFactory instance, storing lock
* files into the specified lockDir:
*
* @param lockDir where lock files are created.
*/
public NativeFSLockFactory(Path lockDir) throws IOException {
setLockDir(lockDir);
}
@Override @Override
public synchronized Lock makeLock(String lockName) { protected Lock makeFSLock(FSDirectory dir, String lockName) {
if (lockPrefix != null) return new NativeFSLock(dir.getDirectory(), lockName);
lockName = lockPrefix + "-" + lockName;
return new NativeFSLock(lockDir, lockName);
} }
static final class NativeFSLock extends Lock {
@Override private FileChannel channel;
public void clearLock(String lockName) throws IOException { private FileLock lock;
makeLock(lockName).close(); private Path path;
} private Path lockDir;
} private static final Set<String> LOCK_HELD = Collections.synchronizedSet(new HashSet<String>());
class NativeFSLock extends Lock {
private FileChannel channel;
private FileLock lock;
private Path path;
private Path lockDir;
private static final Set<String> LOCK_HELD = Collections.synchronizedSet(new HashSet<String>());
public NativeFSLock(Path lockDir, String lockFileName) { public NativeFSLock(Path lockDir, String lockFileName) {
this.lockDir = lockDir; this.lockDir = lockDir;
path = lockDir.resolve(lockFileName); path = lockDir.resolve(lockFileName);
}
@Override
public synchronized boolean obtain() throws IOException {
if (lock != null) {
// Our instance is already locked:
return false;
} }
// Ensure that lockDir exists and is a directory.
Files.createDirectories(lockDir); @Override
try { public synchronized boolean obtain() throws IOException {
Files.createFile(path);
} catch (IOException ignore) { if (lock != null) {
// we must create the file to have a truly canonical path. // Our instance is already locked:
// if its already created, we don't care. if it cant be created, it will fail below. return false;
} }
final Path canonicalPath = path.toRealPath();
// Make sure nobody else in-process has this lock held // Ensure that lockDir exists and is a directory.
// already, and, mark it held if not: Files.createDirectories(lockDir);
// This is a pretty crazy workaround for some documented
// but yet awkward JVM behavior:
//
// On some systems, closing a channel releases all locks held by the Java virtual machine on the underlying file
// regardless of whether the locks were acquired via that channel or via another channel open on the same file.
// It is strongly recommended that, within a program, a unique channel be used to acquire all locks on any given
// file.
//
// This essentially means if we close "A" channel for a given file all locks might be released... the odd part
// is that we can't re-obtain the lock in the same JVM but from a different process if that happens. Nevertheless
// this is super trappy. See LUCENE-5738
boolean obtained = false;
if (LOCK_HELD.add(canonicalPath.toString())) {
try { try {
channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE); Files.createFile(path);
} catch (IOException ignore) {
// we must create the file to have a truly canonical path.
// if its already created, we don't care. if it cant be created, it will fail below.
}
final Path canonicalPath = path.toRealPath();
// Make sure nobody else in-process has this lock held
// already, and, mark it held if not:
// This is a pretty crazy workaround for some documented
// but yet awkward JVM behavior:
//
// On some systems, closing a channel releases all locks held by the Java virtual machine on the underlying file
// regardless of whether the locks were acquired via that channel or via another channel open on the same file.
// It is strongly recommended that, within a program, a unique channel be used to acquire all locks on any given
// file.
//
// This essentially means if we close "A" channel for a given file all locks might be released... the odd part
// is that we can't re-obtain the lock in the same JVM but from a different process if that happens. Nevertheless
// this is super trappy. See LUCENE-5738
boolean obtained = false;
if (LOCK_HELD.add(canonicalPath.toString())) {
try { try {
lock = channel.tryLock(); channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
obtained = lock != null; try {
} catch (IOException | OverlappingFileLockException e) { lock = channel.tryLock();
// At least on OS X, we will sometimes get an obtained = lock != null;
// intermittent "Permission Denied" IOException, } catch (IOException | OverlappingFileLockException e) {
// which seems to simply mean "you failed to get // At least on OS X, we will sometimes get an
// the lock". But other IOExceptions could be // intermittent "Permission Denied" IOException,
// "permanent" (eg, locking is not supported via // which seems to simply mean "you failed to get
// the filesystem). So, we record the failure // the lock". But other IOExceptions could be
// reason here; the timeout obtain (usually the // "permanent" (eg, locking is not supported via
// one calling us) will use this as "root cause" // the filesystem). So, we record the failure
// if it fails to get the lock. // reason here; the timeout obtain (usually the
failureReason = e; // one calling us) will use this as "root cause"
// if it fails to get the lock.
failureReason = e;
}
} finally {
if (obtained == false) { // not successful - clear up and move out
clearLockHeld(path);
final FileChannel toClose = channel;
channel = null;
IOUtils.closeWhileHandlingException(toClose);
}
}
}
return obtained;
}
@Override
public synchronized void close() throws IOException {
try {
if (lock != null) {
try {
lock.release();
lock = null;
} finally {
clearLockHeld(path);
}
} }
} finally { } finally {
if (obtained == false) { // not successful - clear up and move out IOUtils.close(channel);
clearLockHeld(path); channel = null;
final FileChannel toClose = channel;
channel = null;
IOUtils.closeWhileHandlingException(toClose);
}
} }
} }
return obtained;
}
@Override private static final void clearLockHeld(Path path) throws IOException {
public synchronized void close() throws IOException { path = path.toRealPath();
try { boolean remove = LOCK_HELD.remove(path.toString());
if (lock != null) { assert remove : "Lock was cleared but never marked as held";
try { }
lock.release();
lock = null; @Override
} finally { public synchronized boolean isLocked() {
clearLockHeld(path); // The test for is isLocked is not directly possible with native file locks:
}
} // First a shortcut, if a lock reference in this instance is available
} finally { if (lock != null) return true;
IOUtils.close(channel);
channel = null; // Look if lock file is definitely not present; if not, there can definitely be no lock!
if (Files.notExists(path)) return false;
// Try to obtain and release (if was locked) the lock
try {
boolean obtained = obtain();
if (obtained) close();
return !obtained;
} catch (IOException ioe) {
return false;
}
}
@Override
public String toString() {
return "NativeFSLock@" + path;
} }
} }
private static final void clearLockHeld(Path path) throws IOException {
path = path.toRealPath();
boolean remove = LOCK_HELD.remove(path.toString());
assert remove : "Lock was cleared but never marked as held";
}
@Override
public synchronized boolean isLocked() {
// The test for is isLocked is not directly possible with native file locks:
// First a shortcut, if a lock reference in this instance is available
if (lock != null) return true;
// Look if lock file is definitely not present; if not, there can definitely be no lock!
if (Files.notExists(path)) return false;
// Try to obtain and release (if was locked) the lock
try {
boolean obtained = obtain();
if (obtained) close();
return !obtained;
} catch (IOException ioe) {
return false;
}
}
@Override
public String toString() {
return "NativeFSLock@" + path;
}
} }

View File

@ -21,50 +21,45 @@ import java.io.IOException;
/** /**
* Use this {@link LockFactory} to disable locking entirely. * Use this {@link LockFactory} to disable locking entirely.
* Only one instance of this lock is created. You should call {@link * This is a singleton, you have to use {@link #INSTANCE}.
* #getNoLockFactory()} to get the instance.
* *
* @see LockFactory * @see LockFactory
*/ */
public class NoLockFactory extends LockFactory { public final class NoLockFactory extends LockFactory {
// Single instance returned whenever makeLock is called. /** The singleton */
private static NoLock singletonLock = new NoLock(); public static final NoLockFactory INSTANCE = new NoLockFactory();
private static NoLockFactory singleton = new NoLockFactory();
// visible for AssertingLock!
static final NoLock SINGLETON_LOCK = new NoLock();
private NoLockFactory() {} private NoLockFactory() {}
public static NoLockFactory getNoLockFactory() { @Override
return singleton; public Lock makeLock(Directory dir, String lockName) {
return SINGLETON_LOCK;
}
private static class NoLock extends Lock {
@Override
public boolean obtain() throws IOException {
return true;
}
@Override
public void close() {
}
@Override
public boolean isLocked() {
return false;
}
@Override
public String toString() {
return "NoLock";
}
} }
@Override
public Lock makeLock(String lockName) {
return singletonLock;
}
@Override
public void clearLock(String lockName) {}
}
class NoLock extends Lock {
@Override
public boolean obtain() throws IOException {
return true;
}
@Override
public void close() {
}
@Override
public boolean isLocked() {
return false;
}
@Override
public String toString() {
return "NoLock";
}
} }

View File

@ -33,8 +33,7 @@ import org.apache.lucene.util.Accountables;
/** /**
* A memory-resident {@link Directory} implementation. Locking * A memory-resident {@link Directory} implementation. Locking
* implementation is by default the {@link SingleInstanceLockFactory} * implementation is by default the {@link SingleInstanceLockFactory}.
* but can be changed with {@link #setLockFactory}.
* *
* <p><b>Warning:</b> This class is not intended to work with huge * <p><b>Warning:</b> This class is not intended to work with huge
* indexes. Everything beyond several hundred megabytes will waste * indexes. Everything beyond several hundred megabytes will waste
@ -52,17 +51,14 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
protected final Map<String,RAMFile> fileMap = new ConcurrentHashMap<>(); protected final Map<String,RAMFile> fileMap = new ConcurrentHashMap<>();
protected final AtomicLong sizeInBytes = new AtomicLong(); protected final AtomicLong sizeInBytes = new AtomicLong();
// *****
// Lock acquisition sequence: RAMDirectory, then RAMFile
// *****
/** Constructs an empty {@link Directory}. */ /** Constructs an empty {@link Directory}. */
public RAMDirectory() { public RAMDirectory() {
try { this(new SingleInstanceLockFactory());
setLockFactory(new SingleInstanceLockFactory()); }
} catch (IOException e) {
// Cannot happen /** Constructs an empty {@link Directory} with the given {@link LockFactory}. */
} public RAMDirectory(LockFactory lockFactory) {
super(lockFactory);
} }
/** /**
@ -105,11 +101,6 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
} }
} }
@Override
public String getLockID() {
return "lucene-" + Integer.toHexString(hashCode());
}
@Override @Override
public final String[] listAll() { public final String[] listAll() {
ensureOpen(); ensureOpen();
@ -156,11 +147,6 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
return Accountables.namedAccountables("file", fileMap); return Accountables.namedAccountables("file", fileMap);
} }
@Override
public String toString() {
return getClass().getSimpleName() + "(id=" + getLockID() + ")";
}
/** Removes an existing file in the directory. /** Removes an existing file in the directory.
* @throws IOException if the file does not exist * @throws IOException if the file does not exist
*/ */

View File

@ -37,21 +37,20 @@ public class SimpleFSDirectory extends FSDirectory {
/** Create a new SimpleFSDirectory for the named location. /** Create a new SimpleFSDirectory for the named location.
* *
* @param path the path of the directory * @param path the path of the directory
* @param lockFactory the lock factory to use, or null for the default * @param lockFactory the lock factory to use
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public SimpleFSDirectory(Path path, LockFactory lockFactory) throws IOException { public SimpleFSDirectory(Path path, LockFactory lockFactory) throws IOException {
super(path, lockFactory); super(path, lockFactory);
} }
/** Create a new SimpleFSDirectory for the named location and {@link NativeFSLockFactory}. /** Create a new SimpleFSDirectory for the named location and {@link FSLockFactory#getDefault()}.
* *
* @param path the path of the directory * @param path the path of the directory
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public SimpleFSDirectory(Path path) throws IOException { public SimpleFSDirectory(Path path) throws IOException {
super(path, null); this(path, FSLockFactory.getDefault());
} }
/** Creates an IndexInput for the file with the given name. */ /** Creates an IndexInput for the file with the given name. */

View File

@ -56,91 +56,71 @@ import java.nio.file.Path;
* not working properly in your environment, you can easily * not working properly in your environment, you can easily
* test it by using {@link VerifyingLockFactory}, {@link * test it by using {@link VerifyingLockFactory}, {@link
* LockVerifyServer} and {@link LockStressTest}.</p> * LockVerifyServer} and {@link LockStressTest}.</p>
*
* <p>This is a singleton, you have to use {@link #INSTANCE}.
* *
* @see LockFactory * @see LockFactory
*/ */
public class SimpleFSLockFactory extends FSLockFactory { public final class SimpleFSLockFactory extends FSLockFactory {
/** /**
* Create a SimpleFSLockFactory instance, with null (unset) * Singleton instance
* lock directory. When you pass this factory to a {@link FSDirectory}
* subclass, the lock directory is automatically set to the
* directory itself. Be sure to create one instance for each directory
* your create!
*/ */
public SimpleFSLockFactory() throws IOException { public static final SimpleFSLockFactory INSTANCE = new SimpleFSLockFactory();
this((Path) null);
} private SimpleFSLockFactory() {}
/**
* Instantiate using the provided directory (as a Path instance).
* @param lockDir where lock files should be created.
*/
public SimpleFSLockFactory(Path lockDir) throws IOException {
setLockDir(lockDir);
}
@Override @Override
public Lock makeLock(String lockName) { protected Lock makeFSLock(FSDirectory dir, String lockName) {
if (lockPrefix != null) { return new SimpleFSLock(dir.getDirectory(), lockName);
lockName = lockPrefix + "-" + lockName; }
static class SimpleFSLock extends Lock {
Path lockFile;
Path lockDir;
public SimpleFSLock(Path lockDir, String lockFileName) {
this.lockDir = lockDir;
lockFile = lockDir.resolve(lockFileName);
}
@Override
public boolean obtain() throws IOException {
try {
Files.createDirectories(lockDir);
Files.createFile(lockFile);
return true;
} catch (IOException ioe) {
// On Windows, on concurrent createNewFile, the 2nd process gets "access denied".
// In that case, the lock was not aquired successfully, so return false.
// We record the failure reason here; the obtain with timeout (usually the
// one calling us) will use this as "root cause" if it fails to get the lock.
failureReason = ioe;
return false;
}
}
@Override
public void close() throws LockReleaseFailedException {
// TODO: wierd that clearLock() throws the raw IOException...
try {
Files.deleteIfExists(lockFile);
} catch (Throwable cause) {
throw new LockReleaseFailedException("failed to delete " + lockFile, cause);
}
}
@Override
public boolean isLocked() {
return Files.exists(lockFile);
}
@Override
public String toString() {
return "SimpleFSLock@" + lockFile;
} }
return new SimpleFSLock(lockDir, lockName);
} }
@Override
public void clearLock(String lockName) throws IOException {
if (lockPrefix != null) {
lockName = lockPrefix + "-" + lockName;
}
Files.deleteIfExists(lockDir.resolve(lockName));
}
}
class SimpleFSLock extends Lock {
Path lockFile;
Path lockDir;
public SimpleFSLock(Path lockDir, String lockFileName) {
this.lockDir = lockDir;
lockFile = lockDir.resolve(lockFileName);
}
@Override
public boolean obtain() throws IOException {
try {
Files.createDirectories(lockDir);
Files.createFile(lockFile);
return true;
} catch (IOException ioe) {
// On Windows, on concurrent createNewFile, the 2nd process gets "access denied".
// In that case, the lock was not aquired successfully, so return false.
// We record the failure reason here; the obtain with timeout (usually the
// one calling us) will use this as "root cause" if it fails to get the lock.
failureReason = ioe;
return false;
}
}
@Override
public void close() throws LockReleaseFailedException {
// TODO: wierd that clearLock() throws the raw IOException...
try {
Files.deleteIfExists(lockFile);
} catch (Throwable cause) {
throw new LockReleaseFailedException("failed to delete " + lockFile, cause);
}
}
@Override
public boolean isLocked() {
return Files.exists(lockFile);
}
@Override
public String toString() {
return "SimpleFSLock@" + lockFile;
}
} }

View File

@ -31,61 +31,49 @@ import java.util.HashSet;
* @see LockFactory * @see LockFactory
*/ */
public class SingleInstanceLockFactory extends LockFactory { public final class SingleInstanceLockFactory extends LockFactory {
private HashSet<String> locks = new HashSet<>(); private final HashSet<String> locks = new HashSet<>();
@Override @Override
public Lock makeLock(String lockName) { public Lock makeLock(Directory dir, String lockName) {
// We do not use the LockPrefix at all, because the private
// HashSet instance effectively scopes the locking to this
// single Directory instance.
return new SingleInstanceLock(locks, lockName); return new SingleInstanceLock(locks, lockName);
} }
@Override private static class SingleInstanceLock extends Lock {
public void clearLock(String lockName) throws IOException {
synchronized(locks) { private final String lockName;
if (locks.contains(lockName)) { private final HashSet<String> locks;
public SingleInstanceLock(HashSet<String> locks, String lockName) {
this.locks = locks;
this.lockName = lockName;
}
@Override
public boolean obtain() throws IOException {
synchronized(locks) {
return locks.add(lockName);
}
}
@Override
public void close() {
synchronized(locks) {
locks.remove(lockName); locks.remove(lockName);
} }
} }
}
} @Override
public boolean isLocked() {
class SingleInstanceLock extends Lock { synchronized(locks) {
return locks.contains(lockName);
String lockName; }
private HashSet<String> locks; }
public SingleInstanceLock(HashSet<String> locks, String lockName) { @Override
this.locks = locks; public String toString() {
this.lockName = lockName; return super.toString() + ": " + lockName;
} }
@Override
public boolean obtain() throws IOException {
synchronized(locks) {
return locks.add(lockName);
}
}
@Override
public void close() {
synchronized(locks) {
locks.remove(lockName);
}
}
@Override
public boolean isLocked() {
synchronized(locks) {
return locks.contains(lockName);
}
}
@Override
public String toString() {
return super.toString() + ": " + lockName;
} }
} }

View File

@ -35,7 +35,7 @@ import java.io.OutputStream;
* @see LockStressTest * @see LockStressTest
*/ */
public class VerifyingLockFactory extends LockFactory { public final class VerifyingLockFactory extends LockFactory {
final LockFactory lf; final LockFactory lf;
final InputStream in; final InputStream in;
@ -94,13 +94,7 @@ public class VerifyingLockFactory extends LockFactory {
} }
@Override @Override
public synchronized Lock makeLock(String lockName) { public Lock makeLock(Directory dir, String lockName) {
return new CheckedLock(lf.makeLock(lockName)); return new CheckedLock(lf.makeLock(dir, lockName));
}
@Override
public synchronized void clearLock(String lockName)
throws IOException {
lf.clearLock(lockName);
} }
} }

View File

@ -23,6 +23,8 @@ import java.nio.file.Path;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.LockFactory;
/** /**
* Class containing some useful methods used by command line tools * Class containing some useful methods used by command line tools
@ -35,15 +37,26 @@ public final class CommandLineUtil {
} }
/** /**
* Creates a specific FSDirectory instance starting from its class name * Creates a specific FSDirectory instance starting from its class name, using the default lock factory
* @param clazzName The name of the FSDirectory class to load * @param clazzName The name of the FSDirectory class to load
* @param path The path to be used as parameter constructor * @param path The path to be used as parameter constructor
* @return the new FSDirectory instance * @return the new FSDirectory instance
*/ */
public static FSDirectory newFSDirectory(String clazzName, Path path) { public static FSDirectory newFSDirectory(String clazzName, Path path) {
return newFSDirectory(clazzName, path, FSLockFactory.getDefault());
}
/**
* Creates a specific FSDirectory instance starting from its class name
* @param clazzName The name of the FSDirectory class to load
* @param path The path to be used as parameter constructor
* @param lf The lock factory to be used
* @return the new FSDirectory instance
*/
public static FSDirectory newFSDirectory(String clazzName, Path path, LockFactory lf) {
try { try {
final Class<? extends FSDirectory> clazz = loadFSDirectoryClass(clazzName); final Class<? extends FSDirectory> clazz = loadFSDirectoryClass(clazzName);
return newFSDirectory(clazz, path); return newFSDirectory(clazz, path, lf);
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
throw new IllegalArgumentException(FSDirectory.class.getSimpleName() throw new IllegalArgumentException(FSDirectory.class.getSimpleName()
+ " implementation not found: " + clazzName, e); + " implementation not found: " + clazzName, e);
@ -103,10 +116,26 @@ public final class CommandLineUtil {
* @throws InvocationTargetException If the constructor throws an exception * @throws InvocationTargetException If the constructor throws an exception
*/ */
public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, Path path) public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, Path path)
throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException { throws ReflectiveOperationException {
return newFSDirectory(clazz, path, FSLockFactory.getDefault());
}
/**
* Creates a new specific FSDirectory instance
* @param clazz The class of the object to be created
* @param path The file to be used as parameter constructor
* @param lf The lock factory to be used
* @return The new FSDirectory instance
* @throws NoSuchMethodException If the Directory does not have a constructor that takes <code>Path</code>.
* @throws InstantiationException If the class is abstract or an interface.
* @throws IllegalAccessException If the constructor does not have public visibility.
* @throws InvocationTargetException If the constructor throws an exception
*/
public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, Path path, LockFactory lf)
throws ReflectiveOperationException {
// Assuming every FSDirectory has a ctor(Path): // Assuming every FSDirectory has a ctor(Path):
Constructor<? extends FSDirectory> ctor = clazz.getConstructor(Path.class); Constructor<? extends FSDirectory> ctor = clazz.getConstructor(Path.class, LockFactory.class);
return ctor.newInstance(path); return ctor.newInstance(path, lf);
} }
} }

View File

@ -31,12 +31,10 @@ import org.apache.lucene.util.LuceneTestCase;
public class TestCrash extends LuceneTestCase { public class TestCrash extends LuceneTestCase {
private IndexWriter initIndex(Random random, boolean initialCommit) throws IOException { private IndexWriter initIndex(Random random, boolean initialCommit) throws IOException {
return initIndex(random, newMockDirectory(random), initialCommit, true); return initIndex(random, newMockDirectory(random, NoLockFactory.INSTANCE), initialCommit, true);
} }
private IndexWriter initIndex(Random random, MockDirectoryWrapper dir, boolean initialCommit, boolean commitOnClose) throws IOException { private IndexWriter initIndex(Random random, MockDirectoryWrapper dir, boolean initialCommit, boolean commitOnClose) throws IOException {
dir.setLockFactory(NoLockFactory.getNoLockFactory());
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random)) IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random))
.setMaxBufferedDocs(10).setMergeScheduler(new ConcurrentMergeScheduler()).setCommitOnClose(commitOnClose)); .setMaxBufferedDocs(10).setMergeScheduler(new ConcurrentMergeScheduler()).setCommitOnClose(commitOnClose));
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions(); ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
@ -187,7 +185,7 @@ public class TestCrash extends LuceneTestCase {
public void testCrashAfterCloseNoWait() throws IOException { public void testCrashAfterCloseNoWait() throws IOException {
Random random = random(); Random random = random();
MockDirectoryWrapper dir = newMockDirectory(random); MockDirectoryWrapper dir = newMockDirectory(random, NoLockFactory.INSTANCE);
IndexWriter writer = initIndex(random, dir, false, false); IndexWriter writer = initIndex(random, dir, false, false);
try { try {

View File

@ -150,7 +150,6 @@ public class TestCrashCausesCorruptIndex extends LuceneTestCase {
public CrashAfterCreateOutput(Directory realDirectory) throws IOException { public CrashAfterCreateOutput(Directory realDirectory) throws IOException {
super(realDirectory); super(realDirectory);
setLockFactory(realDirectory.getLockFactory());
} }
public void setCrashAfterCreateOutput(String name) { public void setCrashAfterCreateOutput(String name) {

View File

@ -38,6 +38,7 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.TrackingDirectoryWrapper; import org.apache.lucene.store.TrackingDirectoryWrapper;
@ -111,7 +112,7 @@ public class TestDoc extends LuceneTestCase {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw, true); PrintWriter out = new PrintWriter(sw, true);
Directory directory = newFSDirectory(indexDir, null); Directory directory = newFSDirectory(indexDir);
if (directory instanceof MockDirectoryWrapper) { if (directory instanceof MockDirectoryWrapper) {
// We create unreferenced files (we don't even write // We create unreferenced files (we don't even write
@ -155,7 +156,7 @@ public class TestDoc extends LuceneTestCase {
sw = new StringWriter(); sw = new StringWriter();
out = new PrintWriter(sw, true); out = new PrintWriter(sw, true);
directory = newFSDirectory(indexDir, null); directory = newFSDirectory(indexDir);
if (directory instanceof MockDirectoryWrapper) { if (directory instanceof MockDirectoryWrapper) {
// We create unreferenced files (we don't even write // We create unreferenced files (we don't even write

View File

@ -27,15 +27,13 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.BaseDirectory;
import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -111,55 +109,18 @@ public class TestFieldsReader extends LuceneTestCase {
} }
public class FaultyFSDirectory extends BaseDirectory { public class FaultyFSDirectory extends FilterDirectory {
Directory fsDir;
AtomicBoolean doFail = new AtomicBoolean(); AtomicBoolean doFail = new AtomicBoolean();
public FaultyFSDirectory(Path dir) { public FaultyFSDirectory(Directory fsDir) {
fsDir = newFSDirectory(dir); super(fsDir);
lockFactory = fsDir.getLockFactory();
} }
@Override @Override
public IndexInput openInput(String name, IOContext context) throws IOException { public IndexInput openInput(String name, IOContext context) throws IOException {
return new FaultyIndexInput(doFail, fsDir.openInput(name, context)); return new FaultyIndexInput(doFail, in.openInput(name, context));
} }
@Override
public String[] listAll() throws IOException {
return fsDir.listAll();
}
@Override
public void deleteFile(String name) throws IOException {
fsDir.deleteFile(name);
}
@Override
public long fileLength(String name) throws IOException {
return fsDir.fileLength(name);
}
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
return fsDir.createOutput(name, context);
}
@Override
public void sync(Collection<String> names) throws IOException {
fsDir.sync(names);
}
@Override
public void renameFile(String source, String dest) throws IOException {
fsDir.renameFile(source, dest);
}
@Override
public void close() throws IOException {
fsDir.close();
}
public void startFailing() { public void startFailing() {
doFail.set(true); doFail.set(true);
} }
@ -228,7 +189,8 @@ public class TestFieldsReader extends LuceneTestCase {
Path indexDir = createTempDir("testfieldswriterexceptions"); Path indexDir = createTempDir("testfieldswriterexceptions");
try { try {
FaultyFSDirectory dir = new FaultyFSDirectory(indexDir); Directory fsDir = newFSDirectory(indexDir);
FaultyFSDirectory dir = new FaultyFSDirectory(fsDir);
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())) IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE); .setOpenMode(OpenMode.CREATE);
IndexWriter writer = new IndexWriter(dir, iwc); IndexWriter writer = new IndexWriter(dir, iwc);

View File

@ -66,14 +66,11 @@ import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants; import org.apache.lucene.util.Constants;
@ -517,45 +514,6 @@ public class TestIndexWriter extends LuceneTestCase {
dir.close(); dir.close();
} }
// Make sure that a Directory implementation that does
// not use LockFactory at all (ie overrides makeLock and
// implements its own private locking) works OK. This
// was raised on java-dev as loss of backwards
// compatibility.
public void testNullLockFactory() throws IOException {
final class MyRAMDirectory extends MockDirectoryWrapper {
private LockFactory myLockFactory;
MyRAMDirectory(Directory delegate) {
super(random(), delegate);
lockFactory = null;
myLockFactory = new SingleInstanceLockFactory();
}
@Override
public Lock makeLock(String name) {
return myLockFactory.makeLock(name);
}
}
Directory dir = new MyRAMDirectory(new RAMDirectory());
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
for (int i = 0; i < 100; i++) {
addDoc(writer);
}
writer.close();
Term searchTerm = new Term("content", "aaa");
IndexReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("did not get right number of hits", 100, hits.length);
reader.close();
writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE));
writer.close();
dir.close();
}
public void testFlushWithNoMerging() throws IOException { public void testFlushWithNoMerging() throws IOException {
Directory dir = newDirectory(); Directory dir = newDirectory();
IndexWriter writer = new IndexWriter( IndexWriter writer = new IndexWriter(
@ -1465,7 +1423,7 @@ public class TestIndexWriter extends LuceneTestCase {
// Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF), // Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF),
// then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed // then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed
// when listAll() was called in IndexFileDeleter. // when listAll() was called in IndexFileDeleter.
Directory dir = newFSDirectory(createTempDir("emptyFSDirNoLock"), NoLockFactory.getNoLockFactory()); Directory dir = newFSDirectory(createTempDir("emptyFSDirNoLock"), NoLockFactory.INSTANCE);
new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))).close(); new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))).close();
dir.close(); dir.close();
} }
@ -1536,8 +1494,7 @@ public class TestIndexWriter extends LuceneTestCase {
} }
public void testNoSegmentFile() throws IOException { public void testNoSegmentFile() throws IOException {
BaseDirectoryWrapper dir = newDirectory(); BaseDirectoryWrapper dir = newDirectory(random(), NoLockFactory.INSTANCE);
dir.setLockFactory(NoLockFactory.getNoLockFactory());
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2)); .setMaxBufferedDocs(2));
@ -1779,11 +1736,10 @@ public class TestIndexWriter extends LuceneTestCase {
} }
public void testWhetherDeleteAllDeletesWriteLock() throws Exception { public void testWhetherDeleteAllDeletesWriteLock() throws Exception {
Directory d = newFSDirectory(createTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock"));
// Must use SimpleFSLockFactory... NativeFSLockFactory // Must use SimpleFSLockFactory... NativeFSLockFactory
// somehow "knows" a lock is held against write.lock // somehow "knows" a lock is held against write.lock
// even if you remove that file: // even if you remove that file:
d.setLockFactory(new SimpleFSLockFactory()); Directory d = newFSDirectory(createTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock"), SimpleFSLockFactory.INSTANCE);
RandomIndexWriter w1 = new RandomIndexWriter(random(), d); RandomIndexWriter w1 = new RandomIndexWriter(random(), d);
w1.deleteAll(); w1.deleteAll();
try { try {

View File

@ -20,7 +20,6 @@ package org.apache.lucene.store;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Random; import java.util.Random;
@ -31,15 +30,14 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.ArrayUtil;
public class TestBufferedIndexInput extends LuceneTestCase { public class TestBufferedIndexInput extends LuceneTestCase {
@ -267,18 +265,14 @@ public class TestBufferedIndexInput extends LuceneTestCase {
} }
} }
private static class MockFSDirectory extends BaseDirectory { private static class MockFSDirectory extends FilterDirectory {
List<IndexInput> allIndexInputs = new ArrayList<>(); final List<IndexInput> allIndexInputs = new ArrayList<>();
final Random rand;
Random rand;
private Directory dir;
public MockFSDirectory(Path path, Random rand) throws IOException { public MockFSDirectory(Path path, Random rand) throws IOException {
super(new SimpleFSDirectory(path));
this.rand = rand; this.rand = rand;
lockFactory = NoLockFactory.getNoLockFactory();
dir = new SimpleFSDirectory(path, null);
} }
public void tweakBufferSizes() { public void tweakBufferSizes() {
@ -296,46 +290,9 @@ public class TestBufferedIndexInput extends LuceneTestCase {
public IndexInput openInput(String name, IOContext context) throws IOException { public IndexInput openInput(String name, IOContext context) throws IOException {
// Make random changes to buffer size // Make random changes to buffer size
//bufferSize = 1+Math.abs(rand.nextInt() % 10); //bufferSize = 1+Math.abs(rand.nextInt() % 10);
IndexInput f = dir.openInput(name, context); IndexInput f = super.openInput(name, context);
allIndexInputs.add(f); allIndexInputs.add(f);
return f; return f;
} }
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
return dir.createOutput(name, context);
}
@Override
public void close() throws IOException {
dir.close();
}
@Override
public void deleteFile(String name)
throws IOException
{
dir.deleteFile(name);
}
@Override
public String[] listAll()
throws IOException
{
return dir.listAll();
}
@Override
public void sync(Collection<String> names) throws IOException {
dir.sync(names);
}
@Override
public void renameFile(String source, String dest) throws IOException {
dir.renameFile(source, dest);
}
@Override
public long fileLength(String name) throws IOException {
return dir.fileLength(name);
}
} }
} }

View File

@ -17,17 +17,12 @@ package org.apache.lucene.store;
* limitations under the License. * limitations under the License.
*/ */
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
public class TestDirectory extends BaseDirectoryTestCase { public class TestDirectory extends BaseDirectoryTestCase {
@ -63,9 +58,9 @@ public class TestDirectory extends BaseDirectoryTestCase {
} }
final FSDirectory[] dirs = new FSDirectory[] { final FSDirectory[] dirs = new FSDirectory[] {
new SimpleFSDirectory(path, null), new SimpleFSDirectory(path),
new NIOFSDirectory(path, null), new NIOFSDirectory(path),
new MMapDirectory(path, null) new MMapDirectory(path)
}; };
for (int i=0; i<dirs.length; i++) { for (int i=0; i<dirs.length; i++) {
@ -142,11 +137,12 @@ public class TestDirectory extends BaseDirectoryTestCase {
} }
// LUCENE-1468 // LUCENE-1468
@SuppressWarnings("resource")
public void testCopySubdir() throws Throwable { public void testCopySubdir() throws Throwable {
Path path = createTempDir("testsubdir"); Path path = createTempDir("testsubdir");
try { try {
Files.createDirectory(path.resolve("subdir")); Files.createDirectory(path.resolve("subdir"));
Directory fsDir = new SimpleFSDirectory(path, null); Directory fsDir = new SimpleFSDirectory(path);
assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length); assertEquals(0, new RAMDirectory(fsDir, newIOContext(random())).listAll().length);
} finally { } finally {
IOUtils.rm(path); IOUtils.rm(path);
@ -156,13 +152,13 @@ public class TestDirectory extends BaseDirectoryTestCase {
// LUCENE-1468 // LUCENE-1468
public void testNotDirectory() throws Throwable { public void testNotDirectory() throws Throwable {
Path path = createTempDir("testnotdir"); Path path = createTempDir("testnotdir");
Directory fsDir = new SimpleFSDirectory(path, null); Directory fsDir = new SimpleFSDirectory(path);
try { try {
IndexOutput out = fsDir.createOutput("afile", newIOContext(random())); IndexOutput out = fsDir.createOutput("afile", newIOContext(random()));
out.close(); out.close();
assertTrue(slowFileExists(fsDir, "afile")); assertTrue(slowFileExists(fsDir, "afile"));
try { try {
new SimpleFSDirectory(path.resolve("afile"), null); new SimpleFSDirectory(path.resolve("afile"));
fail("did not hit expected exception"); fail("did not hit expected exception");
} catch (IOException nsde) { } catch (IOException nsde) {
// Expected // Expected

View File

@ -45,12 +45,8 @@ public class TestLockFactory extends LuceneTestCase {
// methods are called at the right time, locks are created, etc. // methods are called at the right time, locks are created, etc.
public void testCustomLockFactory() throws IOException { public void testCustomLockFactory() throws IOException {
Directory dir = new MockDirectoryWrapper(random(), new RAMDirectory());
MockLockFactory lf = new MockLockFactory(); MockLockFactory lf = new MockLockFactory();
dir.setLockFactory(lf); Directory dir = new MockDirectoryWrapper(random(), new RAMDirectory(lf));
// Lock prefix should have been set:
assertTrue("lock prefix was not set by the RAMDirectory", lf.lockPrefixSet);
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
@ -78,11 +74,8 @@ public class TestLockFactory extends LuceneTestCase {
// exceptions raised: // exceptions raised:
// Verify: NoLockFactory allows two IndexWriters // Verify: NoLockFactory allows two IndexWriters
public void testRAMDirectoryNoLocking() throws IOException { public void testRAMDirectoryNoLocking() throws IOException {
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory()); MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory(NoLockFactory.INSTANCE));
dir.setLockFactory(NoLockFactory.getNoLockFactory()); dir.setAssertLocks(false); // we are gonna explicitly test we get this back
dir.setWrapLockFactory(false); // we are gonna explicitly test we get this back
assertTrue("RAMDirectory.setLockFactory did not take",
NoLockFactory.class.isInstance(dir.getLockFactory()));
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
writer.commit(); // required so the second open succeed writer.commit(); // required so the second open succeed
@ -105,10 +98,10 @@ public class TestLockFactory extends LuceneTestCase {
// Verify: SingleInstanceLockFactory is the default lock for RAMDirectory // Verify: SingleInstanceLockFactory is the default lock for RAMDirectory
// Verify: RAMDirectory does basic locking correctly (can't create two IndexWriters) // Verify: RAMDirectory does basic locking correctly (can't create two IndexWriters)
public void testDefaultRAMDirectory() throws IOException { public void testDefaultRAMDirectory() throws IOException {
Directory dir = new RAMDirectory(); RAMDirectory dir = new RAMDirectory();
assertTrue("RAMDirectory did not use correct LockFactory: got " + dir.getLockFactory(), assertTrue("RAMDirectory did not use correct LockFactory: got " + dir.lockFactory,
SingleInstanceLockFactory.class.isInstance(dir.getLockFactory())); dir.lockFactory instanceof SingleInstanceLockFactory);
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
@ -141,7 +134,7 @@ public class TestLockFactory extends LuceneTestCase {
@Nightly @Nightly
public void testStressLocksNativeFSLockFactory() throws Exception { public void testStressLocksNativeFSLockFactory() throws Exception {
Path dir = createTempDir("index.TestLockFactory7"); Path dir = createTempDir("index.TestLockFactory7");
_testStressLocks(new NativeFSLockFactory(dir), dir); _testStressLocks(NativeFSLockFactory.INSTANCE, dir);
} }
public void _testStressLocks(LockFactory lockFactory, Path indexDir) throws Exception { public void _testStressLocks(LockFactory lockFactory, Path indexDir) throws Exception {
@ -171,11 +164,10 @@ public class TestLockFactory extends LuceneTestCase {
// Verify: NativeFSLockFactory works correctly // Verify: NativeFSLockFactory works correctly
public void testNativeFSLockFactory() throws IOException { public void testNativeFSLockFactory() throws IOException {
NativeFSLockFactory f = new NativeFSLockFactory(createTempDir(LuceneTestCase.getTestClass().getSimpleName())); Directory dir = FSDirectory.open(createTempDir(LuceneTestCase.getTestClass().getSimpleName()), NativeFSLockFactory.INSTANCE);
f.setLockPrefix("test"); Lock l = dir.makeLock("commit");
Lock l = f.makeLock("commit"); Lock l2 = dir.makeLock("commit");
Lock l2 = f.makeLock("commit");
assertTrue("failed to obtain lock", l.obtain()); assertTrue("failed to obtain lock", l.obtain());
assertTrue("succeeded in obtaining lock twice", !l2.obtain()); assertTrue("succeeded in obtaining lock twice", !l2.obtain());
@ -200,55 +192,14 @@ public class TestLockFactory extends LuceneTestCase {
Path lockFile = tempDir.resolve("test.lock"); Path lockFile = tempDir.resolve("test.lock");
Files.createFile(lockFile); Files.createFile(lockFile);
Lock l = new NativeFSLockFactory(tempDir).makeLock("test.lock"); Directory dir = FSDirectory.open(tempDir, NativeFSLockFactory.INSTANCE);
Lock l = dir.makeLock("test.lock");
assertTrue("failed to obtain lock", l.obtain()); assertTrue("failed to obtain lock", l.obtain());
l.close(); l.close();
assertFalse("failed to release lock", l.isLocked()); assertFalse("failed to release lock", l.isLocked());
Files.deleteIfExists(lockFile); Files.deleteIfExists(lockFile);
} }
// Verify: NativeFSLockFactory assigns null as lockPrefix if the lockDir is inside directory
public void testNativeFSLockFactoryPrefix() throws IOException {
Path fdir1 = createTempDir("TestLockFactory.8");
Path fdir2 = createTempDir("TestLockFactory.8.Lockdir");
Directory dir1 = newFSDirectory(fdir1, new NativeFSLockFactory(fdir1));
// same directory, but locks are stored somewhere else. The prefix of the lock factory should != null
Directory dir2 = newFSDirectory(fdir1, new NativeFSLockFactory(fdir2));
String prefix1 = dir1.getLockFactory().getLockPrefix();
assertNull("Lock prefix for lockDir same as directory should be null", prefix1);
String prefix2 = dir2.getLockFactory().getLockPrefix();
assertNotNull("Lock prefix for lockDir outside of directory should be not null", prefix2);
dir1.close();
dir2.close();
IOUtils.rm(fdir1, fdir2);
}
// Verify: default LockFactory has no prefix (ie
// write.lock is stored in index):
public void testDefaultFSLockFactoryPrefix() throws IOException {
// Make sure we get null prefix, which wont happen if setLockFactory is ever called.
Path dirName = createTempDir("TestLockFactory.10");
Directory dir = new SimpleFSDirectory(dirName);
assertNull("Default lock prefix should be null", dir.getLockFactory().getLockPrefix());
dir.close();
dir = new MMapDirectory(dirName);
assertNull("Default lock prefix should be null", dir.getLockFactory().getLockPrefix());
dir.close();
dir = new NIOFSDirectory(dirName);
assertNull("Default lock prefix should be null", dir.getLockFactory().getLockPrefix());
dir.close();
IOUtils.rm(dirName);
}
private class WriterThread extends Thread { private class WriterThread extends Thread {
private Directory dir; private Directory dir;
private int numIteration; private int numIteration;
@ -349,29 +300,19 @@ public class TestLockFactory extends LuceneTestCase {
} }
} }
public class MockLockFactory extends LockFactory { class MockLockFactory extends LockFactory {
public boolean lockPrefixSet;
public Map<String,Lock> locksCreated = Collections.synchronizedMap(new HashMap<String,Lock>()); public Map<String,Lock> locksCreated = Collections.synchronizedMap(new HashMap<String,Lock>());
public int makeLockCount = 0; public int makeLockCount = 0;
@Override @Override
public void setLockPrefix(String lockPrefix) { public synchronized Lock makeLock(Directory dir, String lockName) {
super.setLockPrefix(lockPrefix);
lockPrefixSet = true;
}
@Override
synchronized public Lock makeLock(String lockName) {
Lock lock = new MockLock(); Lock lock = new MockLock();
locksCreated.put(lockName, lock); locksCreated.put(lockName, lock);
makeLockCount++; makeLockCount++;
return lock; return lock;
} }
@Override
public void clearLock(String specificLockName) {}
public class MockLock extends Lock { public class MockLock extends Lock {
public int lockAttempts; public int lockAttempts;

View File

@ -39,8 +39,7 @@ public class TestMockDirectoryWrapper extends LuceneTestCase {
} }
public void testFailIfIndexWriterNotClosedChangeLockFactory() throws IOException { public void testFailIfIndexWriterNotClosedChangeLockFactory() throws IOException {
MockDirectoryWrapper dir = newMockDirectory(); MockDirectoryWrapper dir = newMockDirectory(random(), new SingleInstanceLockFactory());
dir.setLockFactory(new SingleInstanceLockFactory());
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null)); IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));
try { try {
dir.close(); dir.close();

View File

@ -40,7 +40,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
@Override @Override
protected Directory getDirectory(Path path) throws IOException { protected Directory getDirectory(Path path) throws IOException {
return new MMapDirectory(path, null, 1<<TestUtil.nextInt(random(), 10, 28)); return new MMapDirectory(path, 1<<TestUtil.nextInt(random(), 10, 28));
} }
@Override @Override
@ -178,7 +178,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testSeekZero() throws Exception { public void testSeekZero() throws Exception {
for (int i = 0; i < 31; i++) { for (int i = 0; i < 31; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekZero"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekZero"), 1<<i);
IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random()));
io.close(); io.close();
IndexInput ii = mmapDir.openInput("zeroBytes", newIOContext(random())); IndexInput ii = mmapDir.openInput("zeroBytes", newIOContext(random()));
@ -190,7 +190,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testSeekSliceZero() throws Exception { public void testSeekSliceZero() throws Exception {
for (int i = 0; i < 31; i++) { for (int i = 0; i < 31; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceZero"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceZero"), 1<<i);
IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("zeroBytes", newIOContext(random()));
io.close(); io.close();
IndexInput slicer = mmapDir.openInput("zeroBytes", newIOContext(random())); IndexInput slicer = mmapDir.openInput("zeroBytes", newIOContext(random()));
@ -204,7 +204,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testSeekEnd() throws Exception { public void testSeekEnd() throws Exception {
for (int i = 0; i < 17; i++) { for (int i = 0; i < 17; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekEnd"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekEnd"), 1<<i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte bytes[] = new byte[1<<i]; byte bytes[] = new byte[1<<i];
random().nextBytes(bytes); random().nextBytes(bytes);
@ -222,7 +222,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testSeekSliceEnd() throws Exception { public void testSeekSliceEnd() throws Exception {
for (int i = 0; i < 17; i++) { for (int i = 0; i < 17; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceEnd"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeekSliceEnd"), 1<<i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte bytes[] = new byte[1<<i]; byte bytes[] = new byte[1<<i];
random().nextBytes(bytes); random().nextBytes(bytes);
@ -242,7 +242,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testSeeking() throws Exception { public void testSeeking() throws Exception {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeeking"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSeeking"), 1<<i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte bytes[] = new byte[1<<(i+1)]; // make sure we switch buffers byte bytes[] = new byte[1<<(i+1)]; // make sure we switch buffers
random().nextBytes(bytes); random().nextBytes(bytes);
@ -269,7 +269,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
// the various offset+length and just does readBytes. // the various offset+length and just does readBytes.
public void testSlicedSeeking() throws Exception { public void testSlicedSeeking() throws Exception {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSlicedSeeking"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSlicedSeeking"), 1<<i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte bytes[] = new byte[1<<(i+1)]; // make sure we switch buffers byte bytes[] = new byte[1<<(i+1)]; // make sure we switch buffers
random().nextBytes(bytes); random().nextBytes(bytes);
@ -293,7 +293,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testSliceOfSlice() throws Exception { public void testSliceOfSlice() throws Exception {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSliceOfSlice"), null, 1<<i); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testSliceOfSlice"), 1<<i);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
byte bytes[] = new byte[1<<(i+1)]; // make sure we switch buffers byte bytes[] = new byte[1<<(i+1)]; // make sure we switch buffers
random().nextBytes(bytes); random().nextBytes(bytes);
@ -329,13 +329,14 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testRandomChunkSizes() throws Exception { public void testRandomChunkSizes() throws Exception {
int num = atLeast(10); int num = atLeast(10);
for (int i = 0; i < num; i++) for (int i = 0; i < num; i++) {
assertChunking(random(), TestUtil.nextInt(random(), 20, 100)); assertChunking(random(), TestUtil.nextInt(random(), 20, 100));
}
} }
private void assertChunking(Random random, int chunkSize) throws Exception { private void assertChunking(Random random, int chunkSize) throws Exception {
Path path = createTempDir("mmap" + chunkSize); Path path = createTempDir("mmap" + chunkSize);
MMapDirectory mmapDir = new MMapDirectory(path, null, chunkSize); MMapDirectory mmapDir = new MMapDirectory(path, chunkSize);
// we will map a lot, try to turn on the unmap hack // we will map a lot, try to turn on the unmap hack
if (MMapDirectory.UNMAP_SUPPORTED) if (MMapDirectory.UNMAP_SUPPORTED)
mmapDir.setUseUnmap(true); mmapDir.setUseUnmap(true);
@ -368,7 +369,7 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
public void testImplementations() throws Exception { public void testImplementations() throws Exception {
for (int i = 2; i < 12; i++) { for (int i = 2; i < 12; i++) {
final int chunkSize = 1<<i; final int chunkSize = 1<<i;
MMapDirectory mmapDir = new MMapDirectory(createTempDir("testImplementations"), null, chunkSize); MMapDirectory mmapDir = new MMapDirectory(createTempDir("testImplementations"), chunkSize);
IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random())); IndexOutput io = mmapDir.createOutput("bytes", newIOContext(random()));
int size = random().nextInt(chunkSize * 2) + 3; // add some buffer of 3 for slice tests int size = random().nextInt(chunkSize * 2) + 3; // add some buffer of 3 for slice tests
byte bytes[] = new byte[size]; byte bytes[] = new byte[size];

View File

@ -65,7 +65,7 @@ public class TestWindowsMMap extends LuceneTestCase {
// may take some time until the files are finally dereferenced. So clean the // may take some time until the files are finally dereferenced. So clean the
// directory up front, or otherwise new IndexWriter will fail. // directory up front, or otherwise new IndexWriter will fail.
Path dirPath = createTempDir("testLuceneMmap"); Path dirPath = createTempDir("testLuceneMmap");
MMapDirectory dir = new MMapDirectory(dirPath, null); MMapDirectory dir = new MMapDirectory(dirPath);
// plan to add a set of useful stopwords, consider changing some of the // plan to add a set of useful stopwords, consider changing some of the
// interior filters. // interior filters.

View File

@ -90,6 +90,7 @@ public class NativeUnixDirectory extends FSDirectory {
/** Create a new NIOFSDirectory for the named location. /** Create a new NIOFSDirectory for the named location.
* *
* @param path the path of the directory * @param path the path of the directory
* @param lockFactory to use
* @param mergeBufferSize Size of buffer to use for * @param mergeBufferSize Size of buffer to use for
* merging. See {@link #DEFAULT_MERGE_BUFFER_SIZE}. * merging. See {@link #DEFAULT_MERGE_BUFFER_SIZE}.
* @param minBytesDirect Merges, or files to be opened for * @param minBytesDirect Merges, or files to be opened for
@ -99,8 +100,8 @@ public class NativeUnixDirectory extends FSDirectory {
* @param delegate fallback Directory for non-merges * @param delegate fallback Directory for non-merges
* @throws IOException If there is a low-level I/O error * @throws IOException If there is a low-level I/O error
*/ */
public NativeUnixDirectory(Path path, int mergeBufferSize, long minBytesDirect, Directory delegate) throws IOException { public NativeUnixDirectory(Path path, int mergeBufferSize, long minBytesDirect, LockFactory lockFactory, Directory delegate) throws IOException {
super(path, delegate.getLockFactory()); super(path, lockFactory);
if ((mergeBufferSize & ALIGN) != 0) { if ((mergeBufferSize & ALIGN) != 0) {
throw new IllegalArgumentException("mergeBufferSize must be 0 mod " + ALIGN + " (got: " + mergeBufferSize + ")"); throw new IllegalArgumentException("mergeBufferSize must be 0 mod " + ALIGN + " (got: " + mergeBufferSize + ")");
} }
@ -110,13 +111,24 @@ public class NativeUnixDirectory extends FSDirectory {
} }
/** Create a new NIOFSDirectory for the named location. /** Create a new NIOFSDirectory for the named location.
*
* @param path the path of the directory
* @param lockFactory the lock factory to use
* @param delegate fallback Directory for non-merges
* @throws IOException If there is a low-level I/O error
*/
public NativeUnixDirectory(Path path, LockFactory lockFactory, Directory delegate) throws IOException {
this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, lockFactory, delegate);
}
/** Create a new NIOFSDirectory for the named location with {@link FSLockFactory#getDefault()}.
* *
* @param path the path of the directory * @param path the path of the directory
* @param delegate fallback Directory for non-merges * @param delegate fallback Directory for non-merges
* @throws IOException If there is a low-level I/O error * @throws IOException If there is a low-level I/O error
*/ */
public NativeUnixDirectory(Path path, Directory delegate) throws IOException { public NativeUnixDirectory(Path path, Directory delegate) throws IOException {
this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, delegate); this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, FSLockFactory.getDefault(), delegate);
} }
@Override @Override

View File

@ -40,8 +40,7 @@ public class RAFDirectory extends FSDirectory {
/** Create a new RAFDirectory for the named location. /** Create a new RAFDirectory for the named location.
* *
* @param path the path of the directory * @param path the path of the directory
* @param lockFactory the lock factory to use, or null for the default * @param lockFactory the lock factory to use
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public RAFDirectory(Path path, LockFactory lockFactory) throws IOException { public RAFDirectory(Path path, LockFactory lockFactory) throws IOException {
@ -49,14 +48,13 @@ public class RAFDirectory extends FSDirectory {
path.toFile(); // throw exception if we can't get a File path.toFile(); // throw exception if we can't get a File
} }
/** Create a new SimpleFSDirectory for the named location and {@link NativeFSLockFactory}. /** Create a new SimpleFSDirectory for the named location and {@link FSLockFactory#getDefault()}.
* *
* @param path the path of the directory * @param path the path of the directory
* @throws IOException if there is a low-level I/O error * @throws IOException if there is a low-level I/O error
*/ */
public RAFDirectory(Path path) throws IOException { public RAFDirectory(Path path) throws IOException {
super(path, null); this(path, FSLockFactory.getDefault());
path.toFile(); // throw exception if we can't get a File
} }
/** Creates an IndexInput for the file with the given name. */ /** Creates an IndexInput for the file with the given name. */

View File

@ -22,7 +22,6 @@ import java.io.EOFException;
import java.nio.file.Path; import java.nio.file.Path;
import org.apache.lucene.store.Directory; // javadoc import org.apache.lucene.store.Directory; // javadoc
import org.apache.lucene.store.NativeFSLockFactory; // javadoc
/** /**
* Native {@link Directory} implementation for Microsoft Windows. * Native {@link Directory} implementation for Microsoft Windows.
@ -52,21 +51,20 @@ public class WindowsDirectory extends FSDirectory {
/** Create a new WindowsDirectory for the named location. /** Create a new WindowsDirectory for the named location.
* *
* @param path the path of the directory * @param path the path of the directory
* @param lockFactory the lock factory to use, or null for the default * @param lockFactory the lock factory to use
* ({@link NativeFSLockFactory});
* @throws IOException If there is a low-level I/O error * @throws IOException If there is a low-level I/O error
*/ */
public WindowsDirectory(Path path, LockFactory lockFactory) throws IOException { public WindowsDirectory(Path path, LockFactory lockFactory) throws IOException {
super(path, lockFactory); super(path, lockFactory);
} }
/** Create a new WindowsDirectory for the named location and {@link NativeFSLockFactory}. /** Create a new WindowsDirectory for the named location and {@link FSLockFactory#getDefault()}.
* *
* @param path the path of the directory * @param path the path of the directory
* @throws IOException If there is a low-level I/O error * @throws IOException If there is a low-level I/O error
*/ */
public WindowsDirectory(Path path) throws IOException { public WindowsDirectory(Path path) throws IOException {
super(path, null); this(path, FSLockFactory.getDefault());
} }
@Override @Override

View File

@ -317,28 +317,6 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
dir.close(); dir.close();
} }
// test that cfs reader is read-only
public void testClearLockDisabled() throws IOException {
final String testfile = "_123.test";
Directory dir = newDirectory();
IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT);
out.writeInt(3);
out.close();
SegmentInfo si = newSegmentInfo(dir, "_123");
si.getCodec().compoundFormat().write(dir, si, Collections.<String>emptyList(), MergeState.CheckAbort.NONE, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.clearLock("foobar");
fail("didn't get expected exception");
} catch (UnsupportedOperationException expected) {
// expected UOE
}
cfs.close();
dir.close();
}
/** /**
* This test creates a compound file based on a large number of files of * This test creates a compound file based on a large number of files of
* various length. The file content is generated randomly. The sizes range * various length. The file content is generated randomly. The sizes range

View File

@ -74,7 +74,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
boolean assertNoDeleteOpenFile = false; boolean assertNoDeleteOpenFile = false;
boolean preventDoubleWrite = true; boolean preventDoubleWrite = true;
boolean trackDiskUsage = false; boolean trackDiskUsage = false;
boolean wrapLockFactory = true; boolean wrapLocking = true;
boolean useSlowOpenClosers = true; boolean useSlowOpenClosers = true;
boolean enableVirusScanner = true; boolean enableVirusScanner = true;
boolean allowRandomFileNotFoundException = true; boolean allowRandomFileNotFoundException = true;
@ -86,7 +86,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
volatile boolean crashed; volatile boolean crashed;
private ThrottledIndexOutput throttledOutput; private ThrottledIndexOutput throttledOutput;
private Throttling throttling = Throttling.SOMETIMES; private Throttling throttling = Throttling.SOMETIMES;
protected LockFactory lockFactory;
final AtomicInteger inputCloneCount = new AtomicInteger(); final AtomicInteger inputCloneCount = new AtomicInteger();
@ -129,8 +128,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
this.randomState = new Random(random.nextInt()); this.randomState = new Random(random.nextInt());
this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput
.mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null); .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null);
// force wrapping of lockfactory
this.lockFactory = new MockLockFactoryWrapper(this, delegate.getLockFactory());
init(); init();
} }
@ -709,16 +706,16 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
} }
/** /**
* Set to false if you want to return the pure lockfactory * Set to false if you want to return the pure {@link LockFactory} and not
* and not wrap it with MockLockFactoryWrapper. * wrap all lock with {@code AssertingLock}.
* <p> * <p>
* Be careful if you turn this off: MockDirectoryWrapper might * Be careful if you turn this off: {@code MockDirectoryWrapper} might
* no longer be able to detect if you forget to close an IndexWriter, * no longer be able to detect if you forget to close an {@link IndexWriter},
* and spit out horribly scary confusing exceptions instead of * and spit out horribly scary confusing exceptions instead of
* simply telling you that. * simply telling you that.
*/ */
public void setWrapLockFactory(boolean v) { public void setAssertLocks(boolean v) {
this.wrapLockFactory = v; this.wrapLocking = v;
} }
@Override @Override
@ -987,39 +984,43 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
@Override @Override
public synchronized Lock makeLock(String name) { public synchronized Lock makeLock(String name) {
maybeYield(); maybeYield();
return getLockFactory().makeLock(name); if (wrapLocking) {
} return new AssertingLock(super.makeLock(name), name);
@Override
public synchronized void clearLock(String name) throws IOException {
maybeYield();
getLockFactory().clearLock(name);
}
@Override
public synchronized void setLockFactory(LockFactory lockFactory) throws IOException {
maybeYield();
// sneaky: we must pass the original this way to the dir, because
// some impls (e.g. FSDir) do instanceof here.
in.setLockFactory(lockFactory);
// now set our wrapped factory here
this.lockFactory = new MockLockFactoryWrapper(this, lockFactory);
}
@Override
public synchronized LockFactory getLockFactory() {
maybeYield();
if (wrapLockFactory) {
return lockFactory;
} else { } else {
return in.getLockFactory(); return super.makeLock(name);
} }
} }
private final class AssertingLock extends Lock {
private final Lock delegateLock;
private final String name;
AssertingLock(Lock delegate, String name) {
this.delegateLock = delegate;
this.name = name;
}
@Override @Override
public synchronized String getLockID() { public boolean obtain() throws IOException {
maybeYield(); if (delegateLock.obtain()) {
return in.getLockID(); assert delegateLock == NoLockFactory.SINGLETON_LOCK || !openLocks.containsKey(name);
openLocks.put(name, new RuntimeException("lock \"" + name + "\" was not released"));
return true;
} else {
return false;
}
}
@Override
public void close() throws IOException {
delegateLock.close();
openLocks.remove(name);
}
@Override
public boolean isLocked() throws IOException {
return delegateLock.isLocked();
}
} }
@Override @Override

View File

@ -1,92 +0,0 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
/**
* Used by MockDirectoryWrapper to wrap another factory
* and track open locks.
*/
public class MockLockFactoryWrapper extends LockFactory {
MockDirectoryWrapper dir;
LockFactory delegate;
public MockLockFactoryWrapper(MockDirectoryWrapper dir, LockFactory delegate) {
this.dir = dir;
this.delegate = delegate;
}
@Override
public void setLockPrefix(String lockPrefix) {
delegate.setLockPrefix(lockPrefix);
}
@Override
public String getLockPrefix() {
return delegate.getLockPrefix();
}
@Override
public Lock makeLock(String lockName) {
return new MockLock(delegate.makeLock(lockName), lockName);
}
@Override
public void clearLock(String lockName) throws IOException {
delegate.clearLock(lockName);
dir.openLocks.remove(lockName);
}
@Override
public String toString() {
return "MockLockFactoryWrapper(" + delegate.toString() + ")";
}
private class MockLock extends Lock {
private Lock delegateLock;
private String name;
MockLock(Lock delegate, String name) {
this.delegateLock = delegate;
this.name = name;
}
@Override
public boolean obtain() throws IOException {
if (delegateLock.obtain()) {
assert (delegate instanceof NoLockFactory) || dir.openLocks.containsKey(name) == false;
dir.openLocks.put(name, new RuntimeException("lock \"" + name + "\" was not released"));
return true;
} else {
return false;
}
}
@Override
public void close() throws IOException {
delegateLock.close();
dir.openLocks.remove(name);
}
@Override
public boolean isLocked() throws IOException {
return delegateLock.isLocked();
}
}
}

View File

@ -114,6 +114,7 @@ import org.apache.lucene.search.QueryUtils.FCInvisibleMultiReader;
import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IOContext.Context; import org.apache.lucene.store.IOContext.Context;
@ -1208,6 +1209,14 @@ public abstract class LuceneTestCase extends Assert {
return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), rarely(r)); return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), rarely(r));
} }
/**
* Returns a new Directory instance, using the specified random.
* See {@link #newDirectory()} for more information.
*/
public static BaseDirectoryWrapper newDirectory(Random r, LockFactory lf) {
return wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY, lf), rarely(r));
}
public static MockDirectoryWrapper newMockDirectory() { public static MockDirectoryWrapper newMockDirectory() {
return newMockDirectory(random()); return newMockDirectory(random());
} }
@ -1216,8 +1225,16 @@ public abstract class LuceneTestCase extends Assert {
return (MockDirectoryWrapper) wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), false); return (MockDirectoryWrapper) wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY), false);
} }
public static MockDirectoryWrapper newMockDirectory(Random r, LockFactory lf) {
return (MockDirectoryWrapper) wrapDirectory(r, newDirectoryImpl(r, TEST_DIRECTORY, lf), false);
}
public static MockDirectoryWrapper newMockFSDirectory(Path f) { public static MockDirectoryWrapper newMockFSDirectory(Path f) {
return (MockDirectoryWrapper) newFSDirectory(f, null, false); return (MockDirectoryWrapper) newFSDirectory(f, FSLockFactory.getDefault(), false);
}
public static MockDirectoryWrapper newMockFSDirectory(Path f, LockFactory lf) {
return (MockDirectoryWrapper) newFSDirectory(f, lf, false);
} }
/** /**
@ -1231,7 +1248,7 @@ public abstract class LuceneTestCase extends Assert {
/** Returns a new FSDirectory instance over the given file, which must be a folder. */ /** Returns a new FSDirectory instance over the given file, which must be a folder. */
public static BaseDirectoryWrapper newFSDirectory(Path f) { public static BaseDirectoryWrapper newFSDirectory(Path f) {
return newFSDirectory(f, null); return newFSDirectory(f, FSLockFactory.getDefault());
} }
/** Returns a new FSDirectory instance over the given file, which must be a folder. */ /** Returns a new FSDirectory instance over the given file, which must be a folder. */
@ -1255,11 +1272,8 @@ public abstract class LuceneTestCase extends Assert {
clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass); clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass);
} }
Directory fsdir = newFSDirectoryImpl(clazz, f); Directory fsdir = newFSDirectoryImpl(clazz, f, lf);
BaseDirectoryWrapper wrapped = wrapDirectory(random(), fsdir, bare); BaseDirectoryWrapper wrapped = wrapDirectory(random(), fsdir, bare);
if (lf != null) {
wrapped.setLockFactory(lf);
}
return wrapped; return wrapped;
} catch (Exception e) { } catch (Exception e) {
Rethrow.rethrow(e); Rethrow.rethrow(e);
@ -1454,17 +1468,21 @@ public abstract class LuceneTestCase extends Assert {
} }
} }
private static Directory newFSDirectoryImpl(Class<? extends FSDirectory> clazz, Path path) throws IOException { private static Directory newFSDirectoryImpl(Class<? extends FSDirectory> clazz, Path path, LockFactory lf) throws IOException {
FSDirectory d = null; FSDirectory d = null;
try { try {
d = CommandLineUtil.newFSDirectory(clazz, path); d = CommandLineUtil.newFSDirectory(clazz, path, lf);
} catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { } catch (ReflectiveOperationException e) {
Rethrow.rethrow(e); Rethrow.rethrow(e);
} }
return d; return d;
} }
static Directory newDirectoryImpl(Random random, String clazzName) { static Directory newDirectoryImpl(Random random, String clazzName) {
return newDirectoryImpl(random, clazzName, FSLockFactory.getDefault());
}
static Directory newDirectoryImpl(Random random, String clazzName, LockFactory lf) {
if (clazzName.equals("random")) { if (clazzName.equals("random")) {
if (rarely(random)) { if (rarely(random)) {
clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES); clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES);
@ -1478,21 +1496,27 @@ public abstract class LuceneTestCase extends Assert {
// If it is a FSDirectory type, try its ctor(Path) // If it is a FSDirectory type, try its ctor(Path)
if (FSDirectory.class.isAssignableFrom(clazz)) { if (FSDirectory.class.isAssignableFrom(clazz)) {
final Path dir = createTempDir("index-" + clazzName); final Path dir = createTempDir("index-" + clazzName);
return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), dir); return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), dir, lf);
} }
// See if it has a Path ctor even though it's not an // See if it has a Path/LockFactory ctor even though it's not an
// FSDir subclass: // FSDir subclass:
Constructor<? extends Directory> pathCtor = null;
try { try {
pathCtor = clazz.getConstructor(Path.class); Constructor<? extends Directory> pathCtor = clazz.getConstructor(Path.class, LockFactory.class);
final Path dir = createTempDir("index");
return pathCtor.newInstance(dir, lf);
} catch (NoSuchMethodException nsme) { } catch (NoSuchMethodException nsme) {
// Ignore // Ignore
} }
if (pathCtor != null) { // the remaining dirs are no longer filesystem based, so we must check that the passedLockFactory is not file based:
final Path dir = createTempDir("index"); if (!(lf instanceof FSLockFactory)) {
return pathCtor.newInstance(dir); // try ctor with only LockFactory (e.g. RAMDirectory)
try {
return clazz.getConstructor(LockFactory.class).newInstance(lf);
} catch (NoSuchMethodException nsme) {
// Ignore
}
} }
// try empty ctor // try empty ctor

View File

@ -215,9 +215,9 @@ public final class TestUtil {
* look for any other corruption. */ * look for any other corruption. */
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException { public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
// TODO: actually use the dir's lock factory, unless test uses a special method? // TODO: actually use the dir's locking, unless test uses a special method?
// some tests e.g. exception tests become much more complicated if they have to close the writer // some tests e.g. exception tests become much more complicated if they have to close the writer
try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.getNoLockFactory().makeLock("bogus"))) { try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.makeLock(dir, "bogus"))) {
checker.setCrossCheckTermVectors(crossCheckTermVectors); checker.setCrossCheckTermVectors(crossCheckTermVectors);
checker.setFailFast(failFast); checker.setFailFast(failFast);
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false); checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);

View File

@ -94,7 +94,7 @@ public class TreeMergeOutputFormat extends FileOutputFormat<Text, NullWritable>
writeShardNumberFile(context); writeShardNumberFile(context);
heartBeater.needHeartBeat(); heartBeater.needHeartBeat();
try { try {
Directory mergedIndex = new HdfsDirectory(workDir, NoLockFactory.getNoLockFactory(), context.getConfiguration()); Directory mergedIndex = new HdfsDirectory(workDir, NoLockFactory.INSTANCE, context.getConfiguration());
// TODO: shouldn't we pull the Version from the solrconfig.xml? // TODO: shouldn't we pull the Version from the solrconfig.xml?
IndexWriterConfig writerConfig = new IndexWriterConfig(null) IndexWriterConfig writerConfig = new IndexWriterConfig(null)
@ -128,7 +128,7 @@ public class TreeMergeOutputFormat extends FileOutputFormat<Text, NullWritable>
Directory[] indexes = new Directory[shards.size()]; Directory[] indexes = new Directory[shards.size()];
for (int i = 0; i < shards.size(); i++) { for (int i = 0; i < shards.size(); i++) {
indexes[i] = new HdfsDirectory(shards.get(i), NoLockFactory.getNoLockFactory(), context.getConfiguration()); indexes[i] = new HdfsDirectory(shards.get(i), NoLockFactory.INSTANCE, context.getConfiguration());
} }
context.setStatus("Logically merging " + shards.size() + " shards into one shard"); context.setStatus("Logically merging " + shards.size() + " shards into one shard");

View File

@ -347,7 +347,7 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
} }
if (directory == null) { if (directory == null) {
directory = create(fullPath, createLockFactory(fullPath, rawLockType), dirContext); directory = create(fullPath, createLockFactory(rawLockType), dirContext);
boolean success = false; boolean success = false;
try { try {
directory = rateLimit(directory); directory = rateLimit(directory);

View File

@ -81,11 +81,10 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
/** /**
* Creates a new LockFactory for a given path. * Creates a new LockFactory for a given path.
* @param lockPath the path of the index directory
* @param rawLockType A string value as passed in config. Every factory should at least support 'none' to disable locking. * @param rawLockType A string value as passed in config. Every factory should at least support 'none' to disable locking.
* @throws IOException If there is a low-level I/O error. * @throws IOException If there is a low-level I/O error.
*/ */
protected abstract LockFactory createLockFactory(String lockPath, String rawLockType) throws IOException; protected abstract LockFactory createLockFactory(String rawLockType) throws IOException;
/** /**
* Returns true if a Directory exists for a given path. * Returns true if a Directory exists for a given path.

View File

@ -17,25 +17,21 @@ package org.apache.solr.core;
* limitations under the License. * limitations under the License.
*/ */
import java.io.File; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.Locale; import java.util.Locale;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.store.NativeFSLockFactory;
import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
@ -108,7 +104,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory {
} }
@Override @Override
protected LockFactory createLockFactory(String lockPath, String rawLockType) throws IOException { protected LockFactory createLockFactory(String rawLockType) throws IOException {
if (null == rawLockType) { if (null == rawLockType) {
LOG.warn("No lockType configured, assuming 'hdfs'."); LOG.warn("No lockType configured, assuming 'hdfs'.");
rawLockType = "hdfs"; rawLockType = "hdfs";
@ -116,11 +112,11 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory {
final String lockType = rawLockType.toLowerCase(Locale.ROOT).trim(); final String lockType = rawLockType.toLowerCase(Locale.ROOT).trim();
switch (lockType) { switch (lockType) {
case "hdfs": case "hdfs":
return new HdfsLockFactory(new Path(lockPath), getConf()); return HdfsLockFactory.INSTANCE;
case "single": case "single":
return new SingleInstanceLockFactory(); return new SingleInstanceLockFactory();
case "none": case "none":
return NoLockFactory.getNoLockFactory(); return NoLockFactory.INSTANCE;
default: default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unrecognized lockType: " + rawLockType); "Unrecognized lockType: " + rawLockType);

View File

@ -49,7 +49,7 @@ public class MMapDirectoryFactory extends StandardDirectoryFactory {
public void init(NamedList args) { public void init(NamedList args) {
super.init(args); super.init(args);
SolrParams params = SolrParams.toSolrParams( args ); SolrParams params = SolrParams.toSolrParams( args );
maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_BUFF); maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_CHUNK_SIZE);
if (maxChunk <= 0){ if (maxChunk <= 0){
throw new IllegalArgumentException("maxChunk must be greater than 0"); throw new IllegalArgumentException("maxChunk must be greater than 0");
} }

View File

@ -36,7 +36,7 @@ public class RAMDirectoryFactory extends EphemeralDirectoryFactory {
public static Logger LOG = LoggerFactory.getLogger(RAMDirectoryFactory.class); public static Logger LOG = LoggerFactory.getLogger(RAMDirectoryFactory.class);
@Override @Override
protected LockFactory createLockFactory(String lockPath, String rawLockType) throws IOException { protected LockFactory createLockFactory(String rawLockType) throws IOException {
if (!(rawLockType == null || "single".equalsIgnoreCase(rawLockType.trim()))) { if (!(rawLockType == null || "single".equalsIgnoreCase(rawLockType.trim()))) {
throw new SolrException(ErrorCode.FORBIDDEN, throw new SolrException(ErrorCode.FORBIDDEN,
"RAMDirectory can only be used with the 'single' lock factory type."); "RAMDirectory can only be used with the 'single' lock factory type.");
@ -46,9 +46,7 @@ public class RAMDirectoryFactory extends EphemeralDirectoryFactory {
@Override @Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException { protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
final Directory dir = new RAMDirectory(); return new RAMDirectory(lockFactory);
dir.setLockFactory(lockFactory); // more or less a no-op, just for completeness
return dir;
} }
} }

View File

@ -32,7 +32,6 @@ import org.apache.lucene.store.RateLimitedDirectoryWrapper;
import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -55,7 +54,7 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory {
} }
@Override @Override
protected LockFactory createLockFactory(String lockPath, String rawLockType) throws IOException { protected LockFactory createLockFactory(String rawLockType) throws IOException {
if (null == rawLockType) { if (null == rawLockType) {
// we default to "native" // we default to "native"
log.warn("No lockType configured, assuming 'native'."); log.warn("No lockType configured, assuming 'native'.");
@ -64,13 +63,13 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory {
final String lockType = rawLockType.toLowerCase(Locale.ROOT).trim(); final String lockType = rawLockType.toLowerCase(Locale.ROOT).trim();
switch (lockType) { switch (lockType) {
case "simple": case "simple":
return new SimpleFSLockFactory(new File(lockPath).toPath()); return SimpleFSLockFactory.INSTANCE;
case "native": case "native":
return new NativeFSLockFactory(new File(lockPath).toPath()); return NativeFSLockFactory.INSTANCE;
case "single": case "single":
return new SingleInstanceLockFactory(); return new SingleInstanceLockFactory();
case "none": case "none":
return NoLockFactory.getNoLockFactory(); return NoLockFactory.INSTANCE;
default: default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unrecognized lockType: " + rawLockType); "Unrecognized lockType: " + rawLockType);

View File

@ -32,6 +32,7 @@ import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Lock; import org.apache.lucene.store.Lock;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
@ -52,10 +53,9 @@ public class SnapShooter {
private static final Logger LOG = LoggerFactory.getLogger(SnapShooter.class.getName()); private static final Logger LOG = LoggerFactory.getLogger(SnapShooter.class.getName());
private String snapDir = null; private String snapDir = null;
private SolrCore solrCore; private SolrCore solrCore;
private SimpleFSLockFactory lockFactory;
private String snapshotName = null; private String snapshotName = null;
private String directoryName = null; private String directoryName = null;
private File snapShotDir = null; private FSDirectory snapShotDir = null;
private Lock lock = null; private Lock lock = null;
public SnapShooter(SolrCore core, String location, String snapshotName) { public SnapShooter(SolrCore core, String location, String snapshotName) {
@ -67,11 +67,6 @@ public class SnapShooter {
File dir = new File(snapDir); File dir = new File(snapDir);
if (!dir.exists()) dir.mkdirs(); if (!dir.exists()) dir.mkdirs();
} }
try {
lockFactory = new SimpleFSLockFactory(new File(snapDir).toPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
this.snapshotName = snapshotName; this.snapshotName = snapshotName;
if(snapshotName != null) { if(snapshotName != null) {
@ -122,19 +117,20 @@ public class SnapShooter {
} }
void validateCreateSnapshot() throws IOException { void validateCreateSnapshot() throws IOException {
Lock lock = lockFactory.makeLock(directoryName + ".lock"); final File snapShotFile = new File(snapDir, directoryName);
snapShotDir = new File(snapDir, directoryName); if (snapShotFile.exists()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot directory already exists: " + snapShotFile.getAbsolutePath());
}
if (!snapShotFile.mkdirs()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Unable to create snapshot directory: " + snapShotFile.getAbsolutePath());
}
snapShotDir = new SimpleFSDirectory(snapShotFile.toPath(), SimpleFSLockFactory.INSTANCE);
Lock lock = snapShotDir.makeLock("write.lock");
if (lock.isLocked()) { if (lock.isLocked()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Unable to acquire lock for snapshot directory: " + snapShotDir.getAbsolutePath()); "Unable to acquire lock for snapshot directory: " + snapShotFile.getAbsolutePath());
}
if (snapShotDir.exists()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot directory already exists: " + snapShotDir.getAbsolutePath());
}
if (!snapShotDir.mkdirs()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Unable to create snapshot directory: " + snapShotDir.getAbsolutePath());
} }
} }
@ -146,11 +142,10 @@ public class SnapShooter {
try { try {
Collection<String> files = indexCommit.getFileNames(); Collection<String> files = indexCommit.getFileNames();
FileCopier fileCopier = new FileCopier();
Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType); Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
try { try {
fileCopier.copyFiles(dir, files, snapShotDir); copyFiles(dir, files, snapShotDir);
} finally { } finally {
solrCore.getDirectoryFactory().release(dir); solrCore.getDirectoryFactory().release(dir);
} }
@ -161,7 +156,7 @@ public class SnapShooter {
details.add("snapshotName", snapshotName); details.add("snapshotName", snapshotName);
LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName)); LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName));
} catch (Exception e) { } catch (Exception e) {
SnapPuller.delTree(snapShotDir); SnapPuller.delTree(snapShotDir.getDirectory().toFile());
LOG.error("Exception while creating snapshot", e); LOG.error("Exception while creating snapshot", e);
details.add("snapShootException", e.getMessage()); details.add("snapShootException", e.getMessage());
} finally { } finally {
@ -245,35 +240,13 @@ public class SnapShooter {
public static final String DATE_FMT = "yyyyMMddHHmmssSSS"; public static final String DATE_FMT = "yyyyMMddHHmmssSSS";
private class FileCopier { private void copyFiles(Directory sourceDir, Collection<String> files, Directory destDir) throws IOException {
for (String indexFile : files) {
public void copyFiles(Directory sourceDir, Collection<String> files, copyFile(sourceDir, indexFile, destDir);
File destDir) throws IOException {
// does destinations directory exist ?
if (destDir != null && !destDir.exists()) {
destDir.mkdirs();
}
FSDirectory dir = FSDirectory.open(destDir.toPath());
try {
for (String indexFile : files) {
copyFile(sourceDir, indexFile, new File(destDir, indexFile), dir);
}
} finally {
dir.close();
}
}
public void copyFile(Directory sourceDir, String indexFile, File destination, Directory destDir)
throws IOException {
// make sure we can write to destination
if (destination.exists() && !destination.canWrite()) {
String message = "Unable to open file " + destination + " for writing.";
throw new IOException(message);
}
sourceDir.copy(destDir, indexFile, indexFile, DirectoryFactory.IOCONTEXT_NO_CACHE);
} }
} }
private void copyFile(Directory sourceDir, String indexFile, Directory destDir) throws IOException {
sourceDir.copy(destDir, indexFile, indexFile, DirectoryFactory.IOCONTEXT_NO_CACHE);
}
} }

View File

@ -20,17 +20,15 @@ package org.apache.solr.store.blockcache;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.Set; import java.util.Set;
import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.solr.store.hdfs.HdfsDirectory; import org.apache.solr.store.hdfs.HdfsDirectory;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -38,7 +36,7 @@ import org.slf4j.LoggerFactory;
/** /**
* @lucene.experimental * @lucene.experimental
*/ */
public class BlockDirectory extends Directory { public class BlockDirectory extends FilterDirectory {
public static Logger LOG = LoggerFactory.getLogger(BlockDirectory.class); public static Logger LOG = LoggerFactory.getLogger(BlockDirectory.class);
public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
@ -86,7 +84,6 @@ public class BlockDirectory extends Directory {
public void releaseResources() {} public void releaseResources() {}
}; };
private final Directory directory;
private final int blockSize; private final int blockSize;
private final String dirName; private final String dirName;
private final Cache cache; private final Cache cache;
@ -97,8 +94,8 @@ public class BlockDirectory extends Directory {
public BlockDirectory(String dirName, Directory directory, Cache cache, public BlockDirectory(String dirName, Directory directory, Cache cache,
Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled, Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
boolean blockCacheWriteEnabled) throws IOException { boolean blockCacheWriteEnabled) throws IOException {
super(directory);
this.dirName = dirName; this.dirName = dirName;
this.directory = directory;
blockSize = BLOCK_SIZE; blockSize = BLOCK_SIZE;
this.cache = cache; this.cache = cache;
if (blockCacheFileTypes == null || blockCacheFileTypes.isEmpty()) { if (blockCacheFileTypes == null || blockCacheFileTypes.isEmpty()) {
@ -114,14 +111,11 @@ public class BlockDirectory extends Directory {
if (!blockCacheWriteEnabled) { if (!blockCacheWriteEnabled) {
LOG.info("Block cache on write is disabled"); LOG.info("Block cache on write is disabled");
} }
if (directory.getLockFactory() != null) {
setLockFactory(directory.getLockFactory());
}
} }
private IndexInput openInput(String name, int bufferSize, IOContext context) private IndexInput openInput(String name, int bufferSize, IOContext context)
throws IOException { throws IOException {
final IndexInput source = directory.openInput(name, context); final IndexInput source = super.openInput(name, context);
if (useReadCache(name, context)) { if (useReadCache(name, context)) {
return new CachedIndexInput(source, blockSize, name, return new CachedIndexInput(source, blockSize, name,
getFileCacheName(name), cache, bufferSize); getFileCacheName(name), cache, bufferSize);
@ -241,7 +235,7 @@ public class BlockDirectory extends Directory {
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
// the local file system folder may be gone // the local file system folder may be gone
} finally { } finally {
directory.close(); super.close();
cache.releaseResources(); cache.releaseResources();
} }
} }
@ -251,24 +245,20 @@ public class BlockDirectory extends Directory {
} }
private long getFileModified(String name) throws IOException { private long getFileModified(String name) throws IOException {
if (directory instanceof FSDirectory) { if (in instanceof FSDirectory) {
File directory = ((FSDirectory) this.directory).getDirectory().toFile(); File directory = ((FSDirectory) in).getDirectory().toFile();
File file = new File(directory, name); File file = new File(directory, name);
if (!file.exists()) { if (!file.exists()) {
throw new FileNotFoundException("File [" + name + "] not found"); throw new FileNotFoundException("File [" + name + "] not found");
} }
return file.lastModified(); return file.lastModified();
} else if (directory instanceof HdfsDirectory) { } else if (in instanceof HdfsDirectory) {
return ((HdfsDirectory) directory).fileModified(name); return ((HdfsDirectory) in).fileModified(name);
} else { } else {
throw new RuntimeException("Not supported"); throw new UnsupportedOperationException();
} }
} }
public void clearLock(String name) throws IOException {
directory.clearLock(name);
}
String getFileCacheLocation(String name) { String getFileCacheLocation(String name) {
return dirName + "/" + name; return dirName + "/" + name;
} }
@ -282,42 +272,6 @@ public class BlockDirectory extends Directory {
return cache; return cache;
} }
@Override
public void copy(Directory to, String src, String dest, IOContext context)
throws IOException {
directory.copy(to, src, dest, context);
}
public LockFactory getLockFactory() {
return directory.getLockFactory();
}
public String getLockID() {
return directory.getLockID();
}
public Lock makeLock(String name) {
return directory.makeLock(name);
}
public void setLockFactory(LockFactory lockFactory) throws IOException {
directory.setLockFactory(lockFactory);
}
@Override
public void sync(Collection<String> names) throws IOException {
directory.sync(names);
}
// @SuppressWarnings("deprecation")
// public void sync(String name) throws IOException {
// _directory.sync(name);
// }
public String toString() {
return directory.toString();
}
/** /**
* Determine whether read caching should be used for a particular * Determine whether read caching should be used for a particular
* file/context. * file/context.
@ -363,47 +317,18 @@ public class BlockDirectory extends Directory {
@Override @Override
public IndexOutput createOutput(String name, IOContext context) public IndexOutput createOutput(String name, IOContext context)
throws IOException { throws IOException {
IndexOutput dest = directory.createOutput(name, context); final IndexOutput dest = super.createOutput(name, context);
if (useWriteCache(name, context)) { if (useWriteCache(name, context)) {
return new CachedIndexOutput(this, dest, blockSize, name, cache, return new CachedIndexOutput(this, dest, blockSize, name, cache, blockSize);
blockSize);
} }
return dest; return dest;
} }
public void deleteFile(String name) throws IOException { public void deleteFile(String name) throws IOException {
cache.delete(getFileCacheName(name)); cache.delete(getFileCacheName(name));
directory.deleteFile(name); super.deleteFile(name);
} }
@Override
public void renameFile(String source, String dest) throws IOException {
directory.renameFile(source, dest);
}
public long fileLength(String name) throws IOException {
return directory.fileLength(name);
}
// @SuppressWarnings("deprecation")
// public long fileModified(String name) throws IOException {
// return _directory.fileModified(name);
// }
public String[] listAll() throws IOException {
return directory.listAll();
}
// @SuppressWarnings("deprecation")
// public void touchFile(String name) throws IOException {
// _directory.touchFile(name);
// }
public Directory getDirectory() {
return directory;
}
public boolean isBlockCacheReadEnabled() { public boolean isBlockCacheReadEnabled() {
return blockCacheReadEnabled; return blockCacheReadEnabled;
} }

View File

@ -45,15 +45,19 @@ public class HdfsDirectory extends BaseDirectory {
public static final int BUFFER_SIZE = 8192; public static final int BUFFER_SIZE = 8192;
private static final String LF_EXT = ".lf"; private static final String LF_EXT = ".lf";
protected Path hdfsDirPath; protected final Path hdfsDirPath;
protected Configuration configuration; protected final Configuration configuration;
private final FileSystem fileSystem; private final FileSystem fileSystem;
private final FileContext fileContext; private final FileContext fileContext;
public HdfsDirectory(Path hdfsDirPath, Configuration configuration) throws IOException {
this(hdfsDirPath, HdfsLockFactory.INSTANCE, configuration);
}
public HdfsDirectory(Path hdfsDirPath, LockFactory lockFactory, Configuration configuration) public HdfsDirectory(Path hdfsDirPath, LockFactory lockFactory, Configuration configuration)
throws IOException { throws IOException {
setLockFactory(lockFactory); super(lockFactory);
this.hdfsDirPath = hdfsDirPath; this.hdfsDirPath = hdfsDirPath;
this.configuration = configuration; this.configuration = configuration;
fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), configuration); fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), configuration);

View File

@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.RemoteException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock; import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.LockReleaseFailedException; import org.apache.lucene.store.LockReleaseFailedException;
@ -35,76 +36,24 @@ import org.slf4j.LoggerFactory;
public class HdfsLockFactory extends LockFactory { public class HdfsLockFactory extends LockFactory {
public static Logger log = LoggerFactory.getLogger(HdfsLockFactory.class); public static Logger log = LoggerFactory.getLogger(HdfsLockFactory.class);
private Path lockPath; public static final HdfsLockFactory INSTANCE = new HdfsLockFactory();
private Configuration configuration;
public HdfsLockFactory(Path lockPath, Configuration configuration) { private HdfsLockFactory() {}
this.lockPath = lockPath;
this.configuration = configuration;
}
@Override @Override
public Lock makeLock(String lockName) { public Lock makeLock(Directory dir, String lockName) {
if (!(dir instanceof HdfsDirectory)) {
if (lockPrefix != null) { throw new UnsupportedOperationException("HdfsLockFactory can only be used with HdfsDirectory subclasses, got: " + dir);
lockName = lockPrefix + "-" + lockName;
} }
final HdfsDirectory hdfsDir = (HdfsDirectory) dir;
HdfsLock lock = new HdfsLock(lockPath, lockName, configuration); return new HdfsLock(hdfsDir.getHdfsDirPath(), lockName, hdfsDir.getConfiguration());
return lock;
}
@Override
public void clearLock(String lockName) throws IOException {
FileSystem fs = null;
try {
fs = FileSystem.newInstance(lockPath.toUri(), configuration);
while (true) {
if (fs.exists(lockPath)) {
if (lockPrefix != null) {
lockName = lockPrefix + "-" + lockName;
}
Path lockFile = new Path(lockPath, lockName);
try {
if (fs.exists(lockFile) && !fs.delete(lockFile, false)) {
throw new IOException("Cannot delete " + lockFile);
}
} catch (RemoteException e) {
if (e.getClassName().equals(
"org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
try {
Thread.sleep(5000);
} catch (InterruptedException e1) {
Thread.interrupted();
}
continue;
}
throw e;
}
break;
}
}
} finally {
IOUtils.closeQuietly(fs);
}
}
public Path getLockPath() {
return lockPath;
}
public void setLockPath(Path lockPath) {
this.lockPath = lockPath;
} }
static class HdfsLock extends Lock { static class HdfsLock extends Lock {
private Path lockPath; private final Path lockPath;
private String lockName; private final String lockName;
private Configuration conf; private final Configuration conf;
public HdfsLock(Path lockPath, String lockName, Configuration conf) { public HdfsLock(Path lockPath, String lockName, Configuration conf) {
this.lockPath = lockPath; this.lockPath = lockPath;

View File

@ -48,7 +48,7 @@ public class SolrCoreCheckLockOnStartupTest extends SolrTestCaseJ4 {
@Test @Test
public void testSimpleLockErrorOnStartup() throws Exception { public void testSimpleLockErrorOnStartup() throws Exception {
Directory directory = newFSDirectory(new File(initCoreDataDir, "index").toPath(), new SimpleFSLockFactory()); Directory directory = newFSDirectory(new File(initCoreDataDir, "index").toPath(), SimpleFSLockFactory.INSTANCE);
//creates a new IndexWriter without releasing the lock yet //creates a new IndexWriter without releasing the lock yet
IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(null)); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(null));
@ -74,7 +74,7 @@ public class SolrCoreCheckLockOnStartupTest extends SolrTestCaseJ4 {
File indexDir = new File(initCoreDataDir, "index"); File indexDir = new File(initCoreDataDir, "index");
log.info("Acquiring lock on {}", indexDir.getAbsolutePath()); log.info("Acquiring lock on {}", indexDir.getAbsolutePath());
Directory directory = newFSDirectory(indexDir.toPath(), new NativeFSLockFactory()); Directory directory = newFSDirectory(indexDir.toPath(), NativeFSLockFactory.INSTANCE);
//creates a new IndexWriter without releasing the lock yet //creates a new IndexWriter without releasing the lock yet
IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(null)); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(null));

View File

@ -74,7 +74,7 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set("dfs.permissions.enabled", "false"); conf.set("dfs.permissions.enabled", "false");
directory = new HdfsDirectory(new Path(dfsCluster.getURI().toString() + createTempDir().toFile().getAbsolutePath() + "/hdfs"), NoLockFactory.getNoLockFactory(), conf); directory = new HdfsDirectory(new Path(dfsCluster.getURI().toString() + createTempDir().toFile().getAbsolutePath() + "/hdfs"), NoLockFactory.INSTANCE, conf);
random = random(); random = random();
} }

View File

@ -65,8 +65,9 @@ public class HdfsLockFactoryTest extends SolrTestCaseJ4 {
public void testBasic() throws IOException { public void testBasic() throws IOException {
URI uri = dfsCluster.getURI(); URI uri = dfsCluster.getURI();
Path lockPath = new Path(uri.toString(), "/basedir/lock"); Path lockPath = new Path(uri.toString(), "/basedir/lock");
HdfsLockFactory lockFactory = new HdfsLockFactory(lockPath, new Configuration()); Configuration conf = new Configuration();
Lock lock = lockFactory.makeLock("testlock"); HdfsDirectory dir = new HdfsDirectory(lockPath, conf);
Lock lock = dir.makeLock("testlock");
boolean success = lock.obtain(); boolean success = lock.obtain();
assertTrue("We could not get the lock when it should be available", success); assertTrue("We could not get the lock when it should be available", success);
success = lock.obtain(); success = lock.obtain();
@ -76,6 +77,7 @@ public class HdfsLockFactoryTest extends SolrTestCaseJ4 {
assertTrue("We could not get the lock when it should be available", success); assertTrue("We could not get the lock when it should be available", success);
success = lock.obtain(); success = lock.obtain();
assertFalse("We got the lock but it should be unavailble", success); assertFalse("We got the lock but it should be unavailble", success);
dir.close();
} }

View File

@ -38,8 +38,8 @@ public class MockDirectoryFactory extends EphemeralDirectoryFactory {
private boolean allowReadingFilesStillOpenForWrite = Boolean.getBoolean(SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE); private boolean allowReadingFilesStillOpenForWrite = Boolean.getBoolean(SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE);
@Override @Override
protected LockFactory createLockFactory(String lockPath, String rawLockType) throws IOException { protected LockFactory createLockFactory(String rawLockType) throws IOException {
return NoLockFactory.getNoLockFactory(); // dummy, actually unused return NoLockFactory.INSTANCE; // dummy, actually unused
} }
@Override @Override