Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve tests; fix RAFDirectory.openInput to respect pending delete

This commit is contained in:
Mike McCandless 2016-02-05 12:19:24 -05:00
parent 85c546b742
commit 24f55abfd5
10 changed files with 29 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -754,9 +754,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* IO error
*/
public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
Directory unwrapped = FilterDirectory.unwrap(d);
if (unwrapped instanceof FSDirectory && ((FSDirectory) unwrapped).checkPendingDeletions()) {
throw new IllegalArgumentException("Directory still has pending deleted files; cannot initialize IndexWriter");
if (d instanceof FSDirectory && ((FSDirectory) d).checkPendingDeletions()) {
throw new IllegalArgumentException("Directory " + d + " is still has pending deleted files; cannot initialize IndexWriter");
}
conf.setIndexWriter(this); // prevent reuse by other instances

6
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -280,11 +280,11 @@ public abstract class FSDirectory extends BaseDirectory {
@Override
public void sync(Collection<String> names) throws IOException {
ensureOpen();
maybeDeletePendingFiles();
for (String name : names) {
fsync(name);
}
maybeDeletePendingFiles();
}
@Override
@ -293,11 +293,12 @@ public abstract class FSDirectory extends BaseDirectory {
if (pendingDeletes.contains(source)) {
throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
}
maybeDeletePendingFiles();
pendingDeletes.remove(dest);
Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
// TODO: should we move directory fsync to a separate 'syncMetadata' method?
// for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns
IOUtils.fsync(directory, true);
maybeDeletePendingFiles();
}
@Override
@ -327,6 +328,7 @@ public abstract class FSDirectory extends BaseDirectory {
throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
}
privateDeleteFile(name);
maybeDeletePendingFiles();
}
/** Tries to delete any pending deleted files, and returns true if

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -2737,7 +2737,7 @@ public class TestIndexWriter extends LuceneTestCase {
try {
w = new IndexWriter(dir, iwc);
} catch (IllegalArgumentException iae) {
assertEquals("Directory still has pending deleted files; cannot initialize IndexWriter", iae.getMessage());
assertTrue(iae.getMessage().contains("still has pending deleted files; cannot initialize IndexWriter"));
}
in.close();
}

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java
View File

@ -26,10 +26,12 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.TestUtil;
@SuppressFileSystems("WindowsFS")
public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase {
public void test() throws Exception {
MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors"));

6
lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
View File

@ -48,7 +48,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
static int seed=0;
public void testRandomIWReader() throws Throwable {
Directory dir = newDirectory();
Directory dir = newMaybeVirusCheckingDirectory();
// TODO: verify equals using IW.getReader
DocsAndWriter dw = indexRandomIWReader(5, 3, 100, dir);
@ -61,8 +61,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
}
public void testRandom() throws Throwable {
Directory dir1 = newDirectory();
Directory dir2 = newDirectory();
Directory dir1 = newMaybeVirusCheckingDirectory();
Directory dir2 = newMaybeVirusCheckingDirectory();
// mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
boolean doReaderPooling = random().nextBoolean();
Map<String,Document> docs = indexRandom(5, 3, 100, dir1, doReaderPooling);

2
lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
View File

@ -104,7 +104,7 @@ public class TestStressNRT extends LuceneTestCase {
List<Thread> threads = new ArrayList<>();
Directory dir = newDirectory();
Directory dir = newMaybeVirusCheckingDirectory();
final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())));
writer.setDoRandomForceMergeAssert(false);

1
lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
View File

@ -66,6 +66,7 @@ public class RAFDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
ensureCanRead(name);
final File path = directory.resolve(name).toFile();
RandomAccessFile raf = new RandomAccessFile(path, "r");
return new RAFIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);

7
lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
View File

@ -28,8 +28,9 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.LuceneTestCase;
/**
* Acts like Windows, where random programs may open the files you just wrote in an unfriendly
* way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc.
* Acts like a virus checker on Windows, where random programs may open the files you just wrote in an unfriendly
* way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc. This is more evil
* than WindowsFS which just prevents deletion of files you still old open.
*/
public class VirusCheckingFS extends FilterFileSystemProvider {
@ -75,5 +76,5 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
super.delete(path);
}
// TODO: rename? createOutput? deleteIfExists?
// TODO: we could be more evil here, e.g. rename, createOutput, deleteIfExists
}

11
lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
View File

@ -1282,8 +1282,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
// expected
}
// write the file again
try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
if (random().nextBoolean()) {
try (IndexOutput out = fsDir.createOutput(fileName + "z", IOContext.DEFAULT)) {
}
// Make sure we can rename onto the deleted file:
fsDir.renameFile(fileName + "z", fileName);
} else {
// write the file again
try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
}
}
assertEquals(0, fsDir.fileLength(fileName));
assertTrue(Arrays.asList(fsDir.listAll()).contains(fileName));

2
lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
View File

@ -160,6 +160,8 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
// no unexpected exceptions are raised:
public void testStressLocks() throws Exception {
Path tempPath = createTempDir();
assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));
Directory dir = getDirectory(tempPath);
// First create a 1 doc index: