From d06ba3caaf73512e4ebd3773e454df604de9018d Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 12 Sep 2013 16:59:34 +0000 Subject: [PATCH] LUCENE-5204: Make Directory easier to wrap. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1522661 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 9 +- .../apache/lucene/store/BaseDirectory.java | 69 +++++++++++ .../lucene/store/CompoundFileDirectory.java | 2 +- .../org/apache/lucene/store/Directory.java | 32 +---- .../org/apache/lucene/store/FSDirectory.java | 2 +- .../lucene/store/FileSwitchDirectory.java | 2 +- .../apache/lucene/store/FilterDirectory.java | 113 ++++++++++++++++++ .../org/apache/lucene/store/RAMDirectory.java | 2 +- .../store/RateLimitedDirectoryWrapper.java | 98 ++------------- .../store/TrackingDirectoryWrapper.java | 80 ++----------- .../index/TestCrashCausesCorruptIndex.java | 3 +- .../apache/lucene/index/TestFieldsReader.java | 3 +- .../lucene/store/TestBufferedIndexInput.java | 2 +- .../lucene/store/BaseDirectoryWrapper.java | 81 +------------ .../lucene/store/MockDirectoryWrapper.java | 104 +++++++--------- .../apache/solr/store/hdfs/HdfsDirectory.java | 3 +- 16 files changed, 270 insertions(+), 335 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java create mode 100644 lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 4adac4b3cbe..13d10665359 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -53,6 +53,13 @@ New Features for advanced use cases where String is too restrictive (Luca Cavanna, Robert Muir, Mike McCandless) +Changes in backwards compatibility policy + +* LUCENE-5204: Directory doesn't have default implementations for + LockFactory-related methods, which have been moved to BaseDirectory. If you + had a custom Directory implementation that extended Directory, you need to + extend BaseDirectory instead. (Adrien Grand) + ======================= Lucene 4.5.0 ======================= New features @@ -310,7 +317,7 @@ Changes in backwards compatibility policy * LUCENE-5101: CachingWrapperFilter doesn't always return FixedBitSet instances anymore. Users of the join module can use oal.search.join.FixedBitSetCachingWrapperFilter instead. (Adrien Grand) - + Build * SOLR-5159: Manifest includes non-parsed maven variables. diff --git a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java new file mode 100644 index 00000000000..e0561993673 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java @@ -0,0 +1,69 @@ +package org.apache.lucene.store; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** + * Base implementation for a concrete {@link Directory}. + * @lucene.experimental + */ +public abstract class BaseDirectory extends Directory { + + volatile protected boolean isOpen = true; + + /** Holds the LockFactory instance (implements locking for + * this Directory instance). */ + protected LockFactory lockFactory; + + /** Sole constructor. */ + protected BaseDirectory() { + super(); + } + + @Override + public Lock makeLock(String name) { + return lockFactory.makeLock(name); + } + + @Override + public void clearLock(String name) throws IOException { + if (lockFactory != null) { + lockFactory.clearLock(name); + } + } + + @Override + public void setLockFactory(LockFactory lockFactory) throws IOException { + assert lockFactory != null; + this.lockFactory = lockFactory; + lockFactory.setLockPrefix(this.getLockID()); + } + + @Override + public LockFactory getLockFactory() { + return this.lockFactory; + } + + @Override + protected final void ensureOpen() throws AlreadyClosedException { + if (!isOpen) + throw new AlreadyClosedException("this Directory is closed"); + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java b/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java index 80a23d933d0..d95dca59414 100644 --- a/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java @@ -71,7 +71,7 @@ import java.io.IOException; * * @lucene.experimental */ -public final class CompoundFileDirectory extends Directory { +public final class CompoundFileDirectory extends BaseDirectory { /** Offset/Length for a slice inside of a compound file */ public static final class FileEntry { diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java index 540465594a2..0efc3f93977 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java @@ -44,12 +44,6 @@ import org.apache.lucene.util.IOUtils; */ public abstract class Directory implements Closeable { - volatile protected boolean isOpen = true; - - /** Holds the LockFactory instance (implements locking for - * this Directory instance). */ - protected LockFactory lockFactory; - /** * Returns an array of strings, one for each file in the directory. * @@ -115,20 +109,15 @@ public abstract class Directory implements Closeable { /** Construct a {@link Lock}. * @param name the name of the lock file */ - public Lock makeLock(String name) { - return lockFactory.makeLock(name); - } + public abstract Lock makeLock(String name); + /** * Attempt to clear (forcefully unlock and remove) the * specified lock. Only call this at a time when you are * certain this lock is no longer in use. * @param name name of the lock to be cleared. */ - public void clearLock(String name) throws IOException { - if (lockFactory != null) { - lockFactory.clearLock(name); - } - } + public abstract void clearLock(String name) throws IOException; /** Closes the store. */ @Override @@ -144,11 +133,7 @@ public abstract class Directory implements Closeable { * * @param lockFactory instance of {@link LockFactory}. */ - public void setLockFactory(LockFactory lockFactory) throws IOException { - assert lockFactory != null; - this.lockFactory = lockFactory; - lockFactory.setLockPrefix(this.getLockID()); - } + public abstract void setLockFactory(LockFactory lockFactory) throws IOException; /** * Get the LockFactory that this Directory instance is @@ -156,9 +141,7 @@ public abstract class Directory implements Closeable { * may be null for Directory implementations that provide * their own locking implementation. */ - public LockFactory getLockFactory() { - return this.lockFactory; - } + public abstract LockFactory getLockFactory(); /** * Return a string identifier that uniquely differentiates @@ -252,10 +235,7 @@ public abstract class Directory implements Closeable { /** * @throws AlreadyClosedException if this Directory is closed */ - protected final void ensureOpen() throws AlreadyClosedException { - if (!isOpen) - throw new AlreadyClosedException("this Directory is closed"); - } + protected void ensureOpen() throws AlreadyClosedException {} /** * Allows to create one or more sliced {@link IndexInput} instances from a single diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java index 1dea78ed74e..01a32d2231f 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java @@ -110,7 +110,7 @@ import org.apache.lucene.util.IOUtils; * * @see Directory */ -public abstract class FSDirectory extends Directory { +public abstract class FSDirectory extends BaseDirectory { protected final File directory; // The underlying filesystem directory protected final Set staleFiles = synchronizedSet(new HashSet()); // Files written, but not yet sync'ed diff --git a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java index 99bb98b5bab..981670378a6 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java @@ -39,7 +39,7 @@ import java.util.HashSet; * @lucene.experimental */ -public class FileSwitchDirectory extends Directory { +public class FileSwitchDirectory extends BaseDirectory { private final Directory secondaryDir; private final Directory primaryDir; private final Set primaryExtensions; diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java new file mode 100644 index 00000000000..e7b01b38984 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java @@ -0,0 +1,113 @@ +package org.apache.lucene.store; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +/** Directory implementation that delegates calls to another directory. + * This class can be used to add limitations on top of an existing + * {@link Directory} implementation such as + * {@link RateLimitedDirectoryWrapper rate limiting} or to add additional + * sanity checks for tests. However, if you plan to write your own + * {@link Directory} implementation, you should consider extending directly + * {@link Directory} or {@link BaseDirectory} rather than try to reuse + * functionality of existing {@link Directory}s by extending this class. + * @lucene.internal */ +public class FilterDirectory extends Directory { + + protected final Directory in; + + /** Sole constructor, typically called from sub-classes. */ + protected FilterDirectory(Directory in) { + this.in = in; + } + + /** Return the wrapped {@link Directory}. */ + public final Directory getDelegate() { + return in; + } + + @Override + public String[] listAll() throws IOException { + return in.listAll(); + } + + @Override + public boolean fileExists(String name) throws IOException { + return in.fileExists(name); + } + + @Override + public void deleteFile(String name) throws IOException { + in.deleteFile(name); + } + + @Override + public long fileLength(String name) throws IOException { + return in.fileLength(name); + } + + @Override + public IndexOutput createOutput(String name, IOContext context) + throws IOException { + return in.createOutput(name, context); + } + + @Override + public void sync(Collection names) throws IOException { + in.sync(names); + } + + @Override + public IndexInput openInput(String name, IOContext context) + throws IOException { + return in.openInput(name, context); + } + + @Override + public Lock makeLock(String name) { + return in.makeLock(name); + } + + @Override + public void clearLock(String name) throws IOException { + in.clearLock(name); + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public void setLockFactory(LockFactory lockFactory) throws IOException { + in.setLockFactory(lockFactory); + } + + @Override + public LockFactory getLockFactory() { + return in.getLockFactory(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(" + in.toString() + ")"; + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java index 367d6d02f35..1730378430b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java @@ -45,7 +45,7 @@ import java.util.concurrent.atomic.AtomicLong; * implementation working directly on the file system cache of the * operating system, so copying data to Java heap space is not useful. */ -public class RAMDirectory extends Directory { +public class RAMDirectory extends BaseDirectory { protected final Map fileMap = new ConcurrentHashMap(); protected final AtomicLong sizeInBytes = new AtomicLong(); diff --git a/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java index dcf194e72e4..bc2be4f22cf 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/store/RateLimitedDirectoryWrapper.java @@ -16,7 +16,6 @@ package org.apache.lucene.store; * limitations under the License. */ import java.io.IOException; -import java.util.Collection; import org.apache.lucene.store.IOContext.Context; @@ -28,123 +27,40 @@ import org.apache.lucene.store.IOContext.Context; * @see #setRateLimiter(RateLimiter, IOContext.Context) * @lucene.experimental */ -public final class RateLimitedDirectoryWrapper extends Directory { - - private final Directory delegate; +public final class RateLimitedDirectoryWrapper extends FilterDirectory { + // we need to be volatile here to make sure we see all the values that are set // / modified concurrently private volatile RateLimiter[] contextRateLimiters = new RateLimiter[IOContext.Context .values().length]; public RateLimitedDirectoryWrapper(Directory wrapped) { - this.delegate = wrapped; - } - - public Directory getDelegate() { - return delegate; - } - - @Override - public String[] listAll() throws IOException { - ensureOpen(); - return delegate.listAll(); - } - - @Override - public boolean fileExists(String name) throws IOException { - ensureOpen(); - return delegate.fileExists(name); - } - - @Override - public void deleteFile(String name) throws IOException { - ensureOpen(); - delegate.deleteFile(name); - } - - @Override - public long fileLength(String name) throws IOException { - ensureOpen(); - return delegate.fileLength(name); + super(wrapped); } @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); - final IndexOutput output = delegate.createOutput(name, context); + final IndexOutput output = super.createOutput(name, context); final RateLimiter limiter = getRateLimiter(context.context); if (limiter != null) { return new RateLimitedIndexOutput(limiter, output); } return output; } - - @Override - public void sync(Collection names) throws IOException { - ensureOpen(); - delegate.sync(names); - } - - @Override - public IndexInput openInput(String name, IOContext context) - throws IOException { - ensureOpen(); - return delegate.openInput(name, context); - } - - @Override - public void close() throws IOException { - isOpen = false; - delegate.close(); - } - + @Override public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException { ensureOpen(); - return delegate.createSlicer(name, context); - } - - @Override - public Lock makeLock(String name) { - ensureOpen(); - return delegate.makeLock(name); - } - - @Override - public void clearLock(String name) throws IOException { - ensureOpen(); - delegate.clearLock(name); - } - - @Override - public void setLockFactory(LockFactory lockFactory) throws IOException { - ensureOpen(); - delegate.setLockFactory(lockFactory); - } - - @Override - public LockFactory getLockFactory() { - ensureOpen(); - return delegate.getLockFactory(); - } - - @Override - public String getLockID() { - ensureOpen(); - return delegate.getLockID(); - } - - @Override - public String toString() { - return "RateLimitedDirectoryWrapper(" + delegate.toString() + ")"; + return in.createSlicer(name, context); } @Override public void copy(Directory to, String src, String dest, IOContext context) throws IOException { ensureOpen(); - delegate.copy(to, src, dest, context); + in.copy(to, src, dest, context); } private RateLimiter getRateLimiter(IOContext.Context context) { diff --git a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java index 45868541b4f..ce04b1985ba 100644 --- a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java @@ -17,105 +17,42 @@ package org.apache.lucene.store; * limitations under the License. */ -import java.io.Closeable; import java.io.IOException; import java.util.Collections; -import java.util.Collection; import java.util.HashSet; import java.util.Set; /** A delegating Directory that records which files were * written to and deleted. */ -public final class TrackingDirectoryWrapper extends Directory implements Closeable { +public final class TrackingDirectoryWrapper extends FilterDirectory { - private final Directory other; private final Set createdFileNames = Collections.synchronizedSet(new HashSet()); - public TrackingDirectoryWrapper(Directory other) { - this.other = other; - } - - @Override - public String[] listAll() throws IOException { - return other.listAll(); - } - - @Override - public boolean fileExists(String name) throws IOException { - return other.fileExists(name); + public TrackingDirectoryWrapper(Directory in) { + super(in); } @Override public void deleteFile(String name) throws IOException { createdFileNames.remove(name); - other.deleteFile(name); - } - - @Override - public long fileLength(String name) throws IOException { - return other.fileLength(name); + in.deleteFile(name); } @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { createdFileNames.add(name); - return other.createOutput(name, context); - } - - @Override - public void sync(Collection names) throws IOException { - other.sync(names); - } - - @Override - public IndexInput openInput(String name, IOContext context) throws IOException { - return other.openInput(name, context); - } - - @Override - public Lock makeLock(String name) { - return other.makeLock(name); - } - - @Override - public void clearLock(String name) throws IOException { - other.clearLock(name); - } - - @Override - public void close() throws IOException { - other.close(); - } - - @Override - public void setLockFactory(LockFactory lockFactory) throws IOException { - other.setLockFactory(lockFactory); - } - - @Override - public LockFactory getLockFactory() { - return other.getLockFactory(); - } - - @Override - public String getLockID() { - return other.getLockID(); - } - - @Override - public String toString() { - return "TrackingDirectoryWrapper(" + other.toString() + ")"; + return in.createOutput(name, context); } @Override public void copy(Directory to, String src, String dest, IOContext context) throws IOException { createdFileNames.add(dest); - other.copy(to, src, dest, context); + in.copy(to, src, dest, context); } @Override public Directory.IndexInputSlicer createSlicer(final String name, final IOContext context) throws IOException { - return other.createSlicer(name, context); + return in.createSlicer(name, context); } // maybe clone before returning.... all callers are @@ -124,7 +61,4 @@ public final class TrackingDirectoryWrapper extends Directory implements Closeab return createdFileNames; } - public Directory getDelegate() { - return other; - } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java index 4cac7bb2769..90f6206e530 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java @@ -27,6 +27,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.BaseDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; @@ -146,7 +147,7 @@ public class TestCrashCausesCorruptIndex extends LuceneTestCase { * This test class provides direct access to "simulating" a crash right after * realDirectory.createOutput(..) has been called on a certain specified name. */ - private static class CrashAfterCreateOutput extends Directory { + private static class CrashAfterCreateOutput extends BaseDirectory { private Directory realDirectory; private String crashAfterCreateOutput; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java index 052ad6fd2c7..78462cb951f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -36,6 +36,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.FieldCache; +import org.apache.lucene.store.BaseDirectory; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -112,7 +113,7 @@ public class TestFieldsReader extends LuceneTestCase { } - public static class FaultyFSDirectory extends Directory { + public static class FaultyFSDirectory extends BaseDirectory { Directory fsDir; diff --git a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java index 13fb48d1098..2010c2aeb07 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java @@ -283,7 +283,7 @@ public class TestBufferedIndexInput extends LuceneTestCase { } } - private static class MockFSDirectory extends Directory { + private static class MockFSDirectory extends BaseDirectory { List allIndexInputs = new ArrayList(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java index 9e202041ad3..2791f246549 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryWrapper.java @@ -18,7 +18,6 @@ package org.apache.lucene.store; */ import java.io.IOException; -import java.util.Collection; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.util._TestUtil; @@ -29,15 +28,14 @@ import org.apache.lucene.util._TestUtil; // do NOT make any methods in this class synchronized, volatile // do NOT import anything from the concurrency package. // no randoms, no nothing. -public class BaseDirectoryWrapper extends Directory { - /** our in directory */ - protected final Directory delegate; +public class BaseDirectoryWrapper extends FilterDirectory { private boolean checkIndexOnClose = true; private boolean crossCheckTermVectorsOnClose = true; + protected volatile boolean isOpen = true; public BaseDirectoryWrapper(Directory delegate) { - this.delegate = delegate; + super(delegate); } @Override @@ -46,7 +44,7 @@ public class BaseDirectoryWrapper extends Directory { if (checkIndexOnClose && DirectoryReader.indexExists(this)) { _TestUtil.checkIndex(this, crossCheckTermVectorsOnClose); } - delegate.close(); + super.close(); } public boolean isOpen() { @@ -73,80 +71,13 @@ public class BaseDirectoryWrapper extends Directory { return crossCheckTermVectorsOnClose; } - // directory methods: delegate - - @Override - public String[] listAll() throws IOException { - return delegate.listAll(); - } - - @Override - public boolean fileExists(String name) throws IOException { - return delegate.fileExists(name); - } - - @Override - public void deleteFile(String name) throws IOException { - delegate.deleteFile(name); - } - - @Override - public long fileLength(String name) throws IOException { - return delegate.fileLength(name); - } - - @Override - public IndexOutput createOutput(String name, IOContext context) throws IOException { - return delegate.createOutput(name, context); - } - - @Override - public void sync(Collection names) throws IOException { - delegate.sync(names); - } - - @Override - public IndexInput openInput(String name, IOContext context) throws IOException { - return delegate.openInput(name, context); - } - - @Override - public Lock makeLock(String name) { - return delegate.makeLock(name); - } - - @Override - public void clearLock(String name) throws IOException { - delegate.clearLock(name); - } - - @Override - public void setLockFactory(LockFactory lockFactory) throws IOException { - delegate.setLockFactory(lockFactory); - } - - @Override - public LockFactory getLockFactory() { - return delegate.getLockFactory(); - } - - @Override - public String getLockID() { - return delegate.getLockID(); - } - - @Override - public String toString() { - return "BaseDirectoryWrapper(" + delegate.toString() + ")"; - } - @Override public void copy(Directory to, String src, String dest, IOContext context) throws IOException { - delegate.copy(to, src, dest, context); + in.copy(to, src, dest, context); } @Override public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException { - return delegate.createSlicer(name, context); + return in.createSlicer(name, context); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index c193923f7a1..576ea7710e6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -80,6 +80,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { volatile boolean crashed; private ThrottledIndexOutput throttledOutput; private Throttling throttling = Throttling.SOMETIMES; + protected LockFactory lockFactory; final AtomicInteger inputCloneCount = new AtomicInteger(); @@ -121,10 +122,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { this.lockFactory = new MockLockFactoryWrapper(this, delegate.getLockFactory()); init(); } - - public Directory getDelegate() { - return this.delegate; - } public int getInputCloneCount() { return inputCloneCount.get(); @@ -160,21 +157,15 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } /** - * Returns true if {@link #getDelegate() delegate} must sync its files. + * Returns true if {@link #in} must sync its files. * Currently, only {@link NRTCachingDirectory} requires sync'ing its files * because otherwise they are cached in an internal {@link RAMDirectory}. If * other directories require that too, they should be added to this method. */ private boolean mustSync() { - Directory delegate = this.delegate; - while (true) { - if (delegate instanceof RateLimitedDirectoryWrapper) { - delegate = ((RateLimitedDirectoryWrapper) delegate).getDelegate(); - } else if (delegate instanceof TrackingDirectoryWrapper) { - delegate = ((TrackingDirectoryWrapper) delegate).getDelegate(); - } else { - break; - } + Directory delegate = in; + while (delegate instanceof FilterDirectory) { + delegate = ((FilterDirectory) delegate).getDelegate(); } return delegate instanceof NRTCachingDirectory; } @@ -191,31 +182,22 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { for (String name : names) { // randomly fail with IOE on any file maybeThrowIOException(name); - delegate.sync(Collections.singleton(name)); + in.sync(Collections.singleton(name)); unSyncedFiles.remove(name); } } else { unSyncedFiles.removeAll(names); } } - - @Override - public String toString() { - // NOTE: do not maybeYield here, since it consumes - // randomness and can thus (unexpectedly during - // debugging) change the behavior of a seed - // maybeYield(); - return "MockDirWrapper(" + delegate + ")"; - } public synchronized final long sizeInBytes() throws IOException { - if (delegate instanceof RAMDirectory) - return ((RAMDirectory) delegate).sizeInBytes(); + if (in instanceof RAMDirectory) + return ((RAMDirectory) in).sizeInBytes(); else { // hack long size = 0; - for (String file : delegate.listAll()) - size += delegate.fileLength(file); + for (String file : in.listAll()) + size += in.fileLength(file); return size; } } @@ -252,7 +234,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { long length = fileLength(name); byte[] zeroes = new byte[256]; long upto = 0; - IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); + IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState)); while(upto < length) { final int limit = (int) Math.min(length-upto, zeroes.length); out.writeBytes(zeroes, 0, limit); @@ -268,12 +250,12 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { String tempFileName; while (true) { tempFileName = ""+randomState.nextInt(); - if (!delegate.fileExists(tempFileName)) { + if (!in.fileExists(tempFileName)) { break; } } - final IndexOutput tempOut = delegate.createOutput(tempFileName, LuceneTestCase.newIOContext(randomState)); - IndexInput ii = delegate.openInput(name, LuceneTestCase.newIOContext(randomState)); + final IndexOutput tempOut = in.createOutput(tempFileName, LuceneTestCase.newIOContext(randomState)); + IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState)); tempOut.copyBytes(ii, ii.length()/2); tempOut.close(); ii.close(); @@ -281,8 +263,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { // Delete original and copy bytes back: deleteFile(name, true); - final IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); - ii = delegate.openInput(tempFileName, LuceneTestCase.newIOContext(randomState)); + final IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState)); + ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState)); out.copyBytes(ii, ii.length()); out.close(); ii.close(); @@ -294,7 +276,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { action = "fully truncated"; // Totally truncate the file to zero bytes deleteFile(name, true); - IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); + IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState)); out.setLength(0); out.close(); } @@ -436,7 +418,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { openFilesDeleted.remove(name); } } - delegate.deleteFile(name); + in.deleteFile(name); } public synchronized Set getOpenDeletedFiles() { @@ -476,8 +458,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { unSyncedFiles.add(name); createdFiles.add(name); - if (delegate instanceof RAMDirectory) { - RAMDirectory ramdir = (RAMDirectory) delegate; + if (in instanceof RAMDirectory) { + RAMDirectory ramdir = (RAMDirectory) in; RAMFile file = new RAMFile(ramdir); RAMFile existing = ramdir.fileMap.get(name); @@ -493,7 +475,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } } //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); - IndexOutput delegateOutput = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState, context)); + IndexOutput delegateOutput = in.createOutput(name, LuceneTestCase.newIOContext(randomState, context)); if (randomState.nextInt(10) == 0){ // once in a while wrap the IO in a Buffered IO with random buffer sizes delegateOutput = new BufferedIndexOutputWrapper(1+randomState.nextInt(BufferedIndexOutput.DEFAULT_BUFFER_SIZE), delegateOutput); @@ -504,7 +486,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. if (throttling == Throttling.ALWAYS || - (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0) && !(delegate instanceof RateLimitedDirectoryWrapper)) { + (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0) && !(in instanceof RateLimitedDirectoryWrapper)) { if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")"); } @@ -544,8 +526,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { if (failOnOpenInput) { maybeThrowDeterministicException(); } - if (!delegate.fileExists(name)) { - throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + delegate) : new NoSuchFileException(name + " in dir=" + delegate); + if (!in.fileExists(name)) { + throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + in) : new NoSuchFileException(name + " in dir=" + in); } // cannot open a file for input if it's still open for @@ -554,7 +536,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } - IndexInput delegateInput = delegate.openInput(name, LuceneTestCase.newIOContext(randomState, context)); + IndexInput delegateInput = in.openInput(name, LuceneTestCase.newIOContext(randomState, context)); final IndexInput ii; int randomInt = randomState.nextInt(500); @@ -577,10 +559,10 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { /** Provided for testing purposes. Use sizeInBytes() instead. */ public synchronized final long getRecomputedSizeInBytes() throws IOException { - if (!(delegate instanceof RAMDirectory)) + if (!(in instanceof RAMDirectory)) return sizeInBytes(); long size = 0; - for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) { + for(final RAMFile file: ((RAMDirectory)in).fileMap.values()) { size += file.getSizeInBytes(); } return size; @@ -593,10 +575,10 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { */ public final synchronized long getRecomputedActualSizeInBytes() throws IOException { - if (!(delegate instanceof RAMDirectory)) + if (!(in instanceof RAMDirectory)) return sizeInBytes(); long size = 0; - for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values()) + for (final RAMFile file : ((RAMDirectory)in).fileMap.values()) size += file.length; return size; } @@ -666,8 +648,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { String[] startFiles = allFiles.toArray(new String[0]); IndexWriterConfig iwc = new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null); iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE); - new IndexWriter(delegate, iwc).rollback(); - String[] endFiles = delegate.listAll(); + new IndexWriter(in, iwc).rollback(); + String[] endFiles = in.listAll(); Set startSet = new TreeSet(Arrays.asList(startFiles)); Set endSet = new TreeSet(Arrays.asList(endFiles)); @@ -692,13 +674,13 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } SegmentInfos sis = new SegmentInfos(); try { - sis.read(delegate, file); + sis.read(in, file); } catch (IOException ioe) { // OK: likely some of the .si files were deleted } try { - Set ghosts = new HashSet(sis.files(delegate, false)); + Set ghosts = new HashSet(sis.files(in, false)); for (String s : ghosts) { if (endSet.contains(s) && !startSet.contains(s)) { assert pendingDeletions.contains(s); @@ -763,7 +745,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } } } - delegate.close(); + in.close(); } synchronized void removeOpenFile(Closeable c, String name) { @@ -856,19 +838,19 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { @Override public synchronized String[] listAll() throws IOException { maybeYield(); - return delegate.listAll(); + return in.listAll(); } @Override public synchronized boolean fileExists(String name) throws IOException { maybeYield(); - return delegate.fileExists(name); + return in.fileExists(name); } @Override public synchronized long fileLength(String name) throws IOException { maybeYield(); - return delegate.fileLength(name); + return in.fileLength(name); } @Override @@ -888,7 +870,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { maybeYield(); // sneaky: we must pass the original this way to the dir, because // some impls (e.g. FSDir) do instanceof here. - delegate.setLockFactory(lockFactory); + in.setLockFactory(lockFactory); // now set our wrapped factory here this.lockFactory = new MockLockFactoryWrapper(this, lockFactory); } @@ -899,28 +881,28 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { if (wrapLockFactory) { return lockFactory; } else { - return delegate.getLockFactory(); + return in.getLockFactory(); } } @Override public synchronized String getLockID() { maybeYield(); - return delegate.getLockID(); + return in.getLockID(); } @Override public synchronized void copy(Directory to, String src, String dest, IOContext context) throws IOException { maybeYield(); // randomize the IOContext here? - delegate.copy(to, src, dest, context); + in.copy(to, src, dest, context); } @Override public IndexInputSlicer createSlicer(final String name, IOContext context) throws IOException { maybeYield(); - if (!delegate.fileExists(name)) { + if (!in.fileExists(name)) { throw randomState.nextBoolean() ? new FileNotFoundException(name) : new NoSuchFileException(name); } // cannot open a file for input if it's still open for @@ -929,7 +911,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } - final IndexInputSlicer delegateHandle = delegate.createSlicer(name, context); + final IndexInputSlicer delegateHandle = in.createSlicer(name, context); final IndexInputSlicer handle = new IndexInputSlicer() { private boolean isClosed; diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java index 640e125b0d6..df1701b34bd 100644 --- a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java +++ b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.lucene.store.BaseDirectory; import org.apache.lucene.store.BufferedIndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -39,7 +40,7 @@ import org.apache.solr.store.blockcache.CustomBufferedIndexInput; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class HdfsDirectory extends Directory { +public class HdfsDirectory extends BaseDirectory { public static Logger LOG = LoggerFactory.getLogger(HdfsDirectory.class); public static final int BUFFER_SIZE = 8192;