Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cut back to Directory.deleteFile(String); disable 'could not removed segments_N so I don't remove any other files it may reference' heroics

This commit is contained in:
Mike McCandless 2016-02-03 10:35:57 -05:00
parent b4a2bf2b41
commit a741ea53c9
42 changed files with 181 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
View File

@ -877,7 +877,7 @@ public class Dictionary {
success = true;
} finally {
if (success) {
tempDir.deleteFiles(Collections.singleton(unsorted.getName()));
tempDir.deleteFile(unsorted.getName());
} else {
IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName());
}
@ -966,7 +966,7 @@ public class Dictionary {
success2 = true;
} finally {
if (success2) {
tempDir.deleteFiles(Collections.singleton(sorted));
tempDir.deleteFile(sorted);
} else {
IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted);
}

28
lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java
View File

@ -57,25 +57,21 @@ public class TestFilesystemResourceLoader extends LuceneTestCase {
public void testBaseDir() throws Exception {
final Path base = createTempDir("fsResourceLoaderBase");
Writer os = Files.newBufferedWriter(base.resolve("template.txt"), StandardCharsets.UTF_8);
try {
Writer os = Files.newBufferedWriter(base.resolve("template.txt"), StandardCharsets.UTF_8);
try {
os.write("foobar\n");
} finally {
IOUtils.closeWhileHandlingException(os);
}
ResourceLoader rl = new FilesystemResourceLoader(base);
assertEquals("foobar", WordlistLoader.getLines(rl.openResource("template.txt"), StandardCharsets.UTF_8).get(0));
// Same with full path name:
String fullPath = base.resolve("template.txt").toAbsolutePath().toString();
assertEquals("foobar",
WordlistLoader.getLines(rl.openResource(fullPath), StandardCharsets.UTF_8).get(0));
assertClasspathDelegation(rl);
assertNotFound(rl);
os.write("foobar\n");
} finally {
IOUtils.rm(base);
IOUtils.closeWhileHandlingException(os);
}
ResourceLoader rl = new FilesystemResourceLoader(base);
assertEquals("foobar", WordlistLoader.getLines(rl.openResource("template.txt"), StandardCharsets.UTF_8).get(0));
// Same with full path name:
String fullPath = base.resolve("template.txt").toAbsolutePath().toString();
assertEquals("foobar",
WordlistLoader.getLines(rl.openResource(fullPath), StandardCharsets.UTF_8).get(0));
assertClasspathDelegation(rl);
assertNotFound(rl);
}
public void testDelegation() throws Exception {

2
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
View File

@ -148,7 +148,7 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
public void sync(Collection<String> names) { throw new UnsupportedOperationException(); }
@Override
public void deleteFiles(Collection<String> name) { throw new UnsupportedOperationException(); }
public void deleteFile(String name) { throw new UnsupportedOperationException(); }
@Override
public void renameFile(String source, String dest) { throw new UnsupportedOperationException(); }

2
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
View File

@ -161,7 +161,7 @@ final class Lucene50CompoundReader extends Directory {
/** Not implemented
* @throws UnsupportedOperationException always: not supported by CFS */
@Override
public void deleteFiles(Collection<String> name) {
public void deleteFile(String name) {
throw new UnsupportedOperationException();
}

30
lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
View File

@ -694,20 +694,24 @@ final class IndexFileDeleter implements Closeable {
private void deleteFiles(Collection<String> names) throws IOException {
assert locked();
ensureOpen();
if (names.isEmpty()) {
return;
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "delete \"" + names + "\"");
}
try {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "delete \"" + names + "\"");
}
directory.deleteFiles(names);
} catch (NoSuchFileException | FileNotFoundException e) { // if delete fails
// IndexWriter should only ask us to delete files it knows it wrote, so if we hit this, something is wrong!
if (Constants.WINDOWS) {
// LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state:
} else {
throw e;
// nocommit put annoying windows-specific segments_N heroics back?
for(String name : names) {
try {
directory.deleteFile(name);
} catch (NoSuchFileException | FileNotFoundException e) {
// IndexWriter should only ask us to delete files it knows it wrote, so if we hit this, something is wrong!
if (Constants.WINDOWS) {
// LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state, and falsely
// return NSFE/FNFE
} else {
throw e;
}
}
}
}

1
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -4617,6 +4617,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* commits are no longer needed. Otherwise, those commits will
* be deleted the next time commit() is called.
*/
// nocommit remove this
public synchronized void deleteUnusedFiles() throws IOException {
ensureOpen(false);
deleter.revisitPolicy();

4
lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
View File

@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@ -212,7 +212,7 @@ public class PersistentSnapshotDeletionPolicy extends SnapshotDeletionPolicy {
private synchronized void clearPriorSnapshots() throws IOException {
for(String file : dir.listAll()) {
if (file.startsWith(SNAPSHOTS_PREFIX)) {
dir.deleteFiles(Collections.singleton(file));
dir.deleteFile(file);
}
}
}

5
lucene/core/src/java/org/apache/lucene/store/Directory.java
View File

@ -50,9 +50,8 @@ public abstract class Directory implements Closeable {
// nocommit should this sort?
public abstract String[] listAll() throws IOException;
/** Removes the specified files from the directory. If an exception is thrown, behavior is undefined
* (none, some or all of the files may have in fact been deleted). */
public abstract void deleteFiles(Collection<String> name) throws IOException;
/** Removes an existing file in the directory. */
public abstract void deleteFile(String name) throws IOException;
/**
* Returns the length of a file in the directory. This method follows the

97
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -126,10 +126,8 @@ public abstract class FSDirectory extends BaseDirectory {
protected final Path directory; // The underlying filesystem directory
/** Files we previously tried to delete, but hit exception (on Windows) last time we tried.
* These files are in "pending delete" state, where we refuse to openInput or createOutput
* them, nor include them in .listAll. */
/** Maps files that we are trying to delete (or we tried already but failed)
* before attempting to delete that key. */
protected final Set<String> pendingDeletes = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
/** Used to generate temp file names in {@link #createTempOutput}. */
@ -200,12 +198,11 @@ public abstract class FSDirectory extends BaseDirectory {
}
}
/** Lists all files (including subdirectories) in the
* directory.
/** Lists all files (including subdirectories) in the directory.
*
* @throws IOException if there was an I/O error during listing */
public static String[] listAll(Path dir) throws IOException {
return listAll(dir, Collections.emptySet());
return listAll(dir, null);
}
private static String[] listAll(Path dir, Set<String> skipNames) throws IOException {
@ -214,7 +211,7 @@ public abstract class FSDirectory extends BaseDirectory {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path path : stream) {
String name = path.getFileName().toString();
if (skipNames.contains(name) == false) {
if (skipNames != null && skipNames.contains(name) == false) {
entries.add(name);
}
}
@ -235,18 +232,11 @@ public abstract class FSDirectory extends BaseDirectory {
return Files.size(directory.resolve(name));
}
@Override
public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
// nocommit isn't it an error if they were already pending delete?
pendingDeletes.addAll(names);
deletePendingFiles();
}
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
ensureCanWrite(name);
// nocommit do we need to check pending deletes?
deletePendingFiles();
return new FSIndexOutput(name);
}
@ -264,14 +254,6 @@ public abstract class FSDirectory extends BaseDirectory {
}
}
protected void ensureCanWrite(String name) throws IOException {
deletePendingFiles();
if (pendingDeletes.contains(name)) {
throw new IOException("file \"" + name + "\" is pending delete and cannot be overwritten");
}
Files.deleteIfExists(directory.resolve(name)); // delete existing, if any
}
protected void ensureCanRead(String name) throws IOException {
deletePendingFiles();
if (pendingDeletes.contains(name)) {
@ -319,12 +301,11 @@ public abstract class FSDirectory extends BaseDirectory {
IOUtils.fsync(directory.resolve(name), false);
}
/** Returns true if the file was successfully removed. */
private synchronized boolean deleteFile(String name) throws IOException {
@Override
public void deleteFile(String name) throws IOException {
pendingDeletes.remove(name);
try {
Files.delete(directory.resolve(name));
return true;
} catch (NoSuchFileException | FileNotFoundException e) {
// We were asked to delete a non-existent file:
throw e;
@ -339,9 +320,7 @@ public abstract class FSDirectory extends BaseDirectory {
// TODO: can/should we do if (Constants.WINDOWS) here, else throw the exc?
// but what about a Linux box with a CIFS mount?
//System.out.println("FS.deleteFile failed (" + ioe + "): will retry later");
pendingDeletes.add(name);
return false;
}
}
@ -354,61 +333,17 @@ public abstract class FSDirectory extends BaseDirectory {
/** Try to delete any pending files that we had previously tried to delete but failed
* because we are on Windows and the files were still held open. */
public synchronized void deletePendingFiles() throws IOException {
public void deletePendingFiles() throws IOException {
// nocommit do we need exponential backoff here for windows?
// TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?
// Clone the set because it will change as we iterate:
List<String> toDelete = new ArrayList<>(pendingDeletes);
System.out.println("del pending: " + pendingDeletes);
Set<String> toDelete = new HashSet<>(pendingDeletes);
// First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
// before we remove any files they reference. If any delete of segments_N fails, we leave all other files
// undeleted so index is never in a corrupt state:
Throwable firstException = null;
for (String fileName : toDelete) {
if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
try {
if (deleteFile(fileName) == false) {
// nocommit
System.out.println(" false on " + fileName + "; skipping the rest");
return;
}
} catch (Throwable t) {
if (firstException == null) {
firstException = t;
} else {
firstException.addSuppressed(t);
}
// nocommit
System.out.println(" fail on " + fileName + ":");
t.printStackTrace(System.out);
throw t;
}
}
// nocommit heroic exceptions here or not?
for(String name : toDelete) {
deleteFile(name);
}
// Only delete other files if we were able to remove the segments_N files; this way we never
// leave a corrupt commit in the index even in the presense of virus checkers:
for(String fileName : toDelete) {
if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
try {
deleteFile(fileName);
} catch (Throwable t) {
if (firstException == null) {
firstException = t;
} else {
firstException.addSuppressed(t);
}
// nocommit
System.out.println(" fail on " + fileName + ":");
t.printStackTrace(System.out);
throw t;
}
}
}
// Does nothing if firstException is null:
IOUtils.reThrow(firstException);
}
final class FSIndexOutput extends OutputStreamIndexOutput {

19
lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
View File

@ -140,20 +140,11 @@ public class FileSwitchDirectory extends Directory {
}
@Override
public void deleteFiles(Collection<String> names) throws IOException {
Set<String> primaryToDelete = new HashSet<>();
Set<String> secondaryToDelete = new HashSet<>();
for(String name : names) {
if (getDirectory(name) == primaryDir) {
primaryToDelete.add(name);
} else {
secondaryToDelete.add(name);
}
}
try {
primaryDir.deleteFiles(primaryToDelete);
} finally {
secondaryDir.deleteFiles(secondaryToDelete);
public void deleteFile(String name) throws IOException {
if (getDirectory(name) == primaryDir) {
primaryDir.deleteFile(name);
} else {
secondaryDir.deleteFile(name);
}
}

4
lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
View File

@ -58,8 +58,8 @@ public class FilterDirectory extends Directory {
}
@Override
public void deleteFiles(Collection<String> names) throws IOException {
in.deleteFiles(names);
public void deleteFile(String name) throws IOException {
in.deleteFile(name);
}
@Override

4
lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
View File

@ -33,9 +33,9 @@ public final class LockValidatingDirectoryWrapper extends FilterDirectory {
}
@Override
public void deleteFiles(Collection<String> names) throws IOException {
public void deleteFile(String name) throws IOException {
writeLock.ensureValid();
in.deleteFiles(names);
in.deleteFile(name);
}
@Override

29
lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
View File

@ -20,12 +20,10 @@ package org.apache.lucene.store;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.lucene.store.RAMDirectory; // javadocs
@ -112,23 +110,14 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
}
@Override
public synchronized void deleteFiles(Collection<String> names) throws IOException {
public synchronized void deleteFile(String name) throws IOException {
if (VERBOSE) {
System.out.println("nrtdir.deleteFiles names=" + names);
System.out.println("nrtdir.deleteFile name=" + name);
}
Set<String> cacheToDelete = new HashSet<>();
Set<String> toDelete = new HashSet<>();
for(String name : names) {
if (cache.fileNameExists(name)) {
cacheToDelete.add(name);
} else {
toDelete.add(name);
}
}
try {
cache.deleteFiles(cacheToDelete);
} finally {
in.deleteFiles(toDelete);
if (cache.fileNameExists(name)) {
cache.deleteFile(name);
} else {
in.deleteFile(name);
}
}
@ -155,14 +144,14 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
System.out.println(" to cache");
}
try {
in.deleteFiles(Collections.singleton(name));
in.deleteFile(name);
} catch (IOException ioe) {
// This is fine: file may not exist
}
return cache.createOutput(name, context);
} else {
try {
cache.deleteFiles(Collections.singleton(name));
cache.deleteFile(name);
} catch (IOException ioe) {
// This is fine: file may not exist
}
@ -332,7 +321,7 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
synchronized(this) {
// Must sync here because other sync methods have
// if (cache.fileNameExists(name)) { ... } else { ... }:
cache.deleteFiles(Collections.singleton(fileName));
cache.deleteFile(fileName);
}
}
}

16
lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
View File

@ -157,16 +157,14 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
}
@Override
public void deleteFiles(Collection<String> names) throws IOException {
public void deleteFile(String name) throws IOException {
ensureOpen();
for(String name : names) {
RAMFile file = fileMap.remove(name);
if (file != null) {
file.directory = null;
sizeInBytes.addAndGet(-file.sizeInBytes);
} else {
throw new FileNotFoundException(name);
}
RAMFile file = fileMap.remove(name);
if (file != null) {
file.directory = null;
sizeInBytes.addAndGet(-file.sizeInBytes);
} else {
throw new FileNotFoundException(name);
}
}

9
lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@ -34,11 +33,9 @@ public final class TrackingDirectoryWrapper extends FilterDirectory {
}
@Override
public void deleteFiles(Collection<String> names) throws IOException {
in.deleteFiles(names);
for(String name : names) {
createdFileNames.remove(name);
}
public void deleteFile(String name) throws IOException {
in.deleteFile(name);
createdFileNames.remove(name);
}
@Override

30
lucene/core/src/java/org/apache/lucene/util/IOUtils.java
View File

@ -38,10 +38,8 @@ import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@ -193,10 +191,12 @@ public final class IOUtils {
* Note that the files should not be null.
*/
public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) {
try {
dir.deleteFiles(files);
} catch (Throwable ignored) {
// ignore
for(String name : files) {
try {
dir.deleteFile(name);
} catch (Throwable ignored) {
// ignore
}
}
}
@ -215,15 +215,21 @@ public final class IOUtils {
* @param names file names to delete
*/
public static void deleteFiles(Directory dir, Collection<String> names) throws IOException {
Set<String> nonNullNames = new HashSet<>();
for(String name : names) {
Throwable th = null;
for (String name : names) {
if (name != null) {
nonNullNames.add(name);
try {
dir.deleteFile(name);
} catch (Throwable t) {
addSuppressed(th, t);
if (th == null) {
th = t;
}
}
}
}
if (names.isEmpty() == false) {
dir.deleteFiles(names);
}
reThrow(th);
}
public static void deleteFiles(Directory dir, String... files) throws IOException {

7
lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
View File

@ -22,7 +22,6 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@ -40,8 +39,8 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntroSorter;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
@ -821,7 +820,7 @@ public class BKDWriter implements Closeable {
//System.out.println("sort time: " + ((t1-t0)/1000000.0) + " msec");
if (tempInput != null) {
tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempDir.deleteFile(tempInput.getName());
tempInput = null;
} else {
assert heapPointWriter != null;
@ -914,7 +913,7 @@ public class BKDWriter implements Closeable {
try {
tempInput.close();
} finally {
tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempDir.deleteFile(tempInput.getName());
tempInput = null;
}
}

3
lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.util.bkd;
*/
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -75,7 +74,7 @@ final class OfflinePointWriter implements PointWriter {
@Override
public void destroy() throws IOException {
tempDir.deleteFiles(Collections.singleton(out.getName()));
tempDir.deleteFile(out.getName());
}
@Override

7
lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
View File

@ -17,9 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.NumericDocValuesField;
@ -48,7 +45,9 @@ public class TestCodecHoldsOpenFiles extends LuceneTestCase {
w.commit();
w.close();
d.deleteFiles(Arrays.asList(d.listAll()));
for (String name : d.listAll()) {
d.deleteFile(name);
}
for(LeafReaderContext cxt : r.leaves()) {
TestUtil.checkReader(cxt.reader());

12
lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -35,9 +34,8 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
@ -297,7 +295,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
break;
}
dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
}
@ -375,7 +373,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
while(gen > 0) {
IndexReader reader = DirectoryReader.open(dir);
reader.close();
dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0) {
@ -602,7 +600,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
}
}
if (i < N) {
dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
@ -716,7 +714,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
}
}
if (i < N) {
dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}

9
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@ -40,8 +39,8 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -708,7 +707,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
// Blow away the index:
for(String fileName : dir.listAll()) {
dir.deleteFiles(Collections.singleton(fileName));
dir.deleteFile(fileName);
}
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
@ -757,7 +756,9 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
DirectoryReader r = DirectoryReader.open(dir);
// Blow away the index:
dir.deleteFiles(Arrays.asList(dir.listAll()));
for(String name : dir.listAll()) {
dir.deleteFile(name);
}
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
doc = new Document();

5
lucene/core/src/test/org/apache/lucene/index/TestDoc.java
View File

@ -48,7 +48,6 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
/** JUnit adaptation of an older test case DocTest. */
@ -234,7 +233,9 @@ public class TestDoc extends LuceneTestCase {
Collection<String> filesToDelete = si.files();
codec.compoundFormat().write(dir, si, context);
si.setUseCompoundFile(true);
si1.info.dir.deleteFiles(filesToDelete);
for(String name : filesToDelete) {
si1.info.dir.deleteFile(name);
}
}
return new SegmentCommitInfo(si, 0, -1L, -1L, -1L);

32
lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
View File

@ -18,6 +18,9 @@ package org.apache.lucene.index;
*/
import java.io.*;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
@ -26,6 +29,8 @@ import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.mockfile.FilterPath;
import org.apache.lucene.mockfile.VirusCheckingFS;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -219,8 +224,14 @@ public class TestIndexFileDeleter extends LuceneTestCase {
}
public void testVirusScannerDoesntCorruptIndex() throws IOException {
MockDirectoryWrapper dir = newMockDirectory();
dir.setPreventDoubleWrite(false); // we arent trying to test this
Path path = createTempDir();
VirusCheckingFS fs = new VirusCheckingFS(path.getFileSystem(), random());
FileSystem filesystem = fs.getFileSystem(URI.create("file:///"));
fs.disable();
Path path2 = new FilterPath(path, filesystem);
Directory dir = newFSDirectory(path2);
// add empty commit
new IndexWriter(dir, new IndexWriterConfig(null)).close();
@ -228,25 +239,12 @@ public class TestIndexFileDeleter extends LuceneTestCase {
dir.createOutput("_0.si", IOContext.DEFAULT).close();
// start virus scanner
final AtomicBoolean stopScanning = new AtomicBoolean();
dir.failOn(new MockDirectoryWrapper.Failure() {
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
if (stopScanning.get()) {
return;
}
for (StackTraceElement f : new Exception().getStackTrace()) {
if ("deleteFile".equals(f.getMethodName())) {
throw new IOException("temporarily cannot delete file");
}
}
}
});
fs.enable();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));
iw.addDocument(new Document());
// stop virus scanner
stopScanning.set(true);
fs.disable();
iw.commit();
iw.close();
dir.close();

10
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -57,14 +57,14 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
@SuppressCodecs("SimpleText") // too slow here
@ -916,7 +916,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
isCommit = true;
}
if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFiles".equals(trace[i].getMethodName())) {
if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName())) {
isDelete = true;
}
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) {
@ -1205,7 +1205,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
}
in.close();
out.close();
dir.deleteFiles(Collections.singleton(fileNameIn));
dir.deleteFile(fileNameIn);
IndexReader reader = null;
try {
@ -1255,7 +1255,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
assertTrue(si.info.getUseCompoundFile());
List<String> victims = new ArrayList<String>(si.info.files());
Collections.shuffle(victims, random());
dir.deleteFiles(Collections.singleton(victims.get(0)));
dir.deleteFile(victims.get(0));
corrupted = true;
break;
}

8
lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@ -26,9 +25,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
/** LUCENE-5574 */
@SuppressFileSystems("WindowsFS") // the bug doesn't happen on windows.
@ -62,7 +60,9 @@ public class TestNRTReaderCleanup extends LuceneTestCase {
w.close();
// Blow away index and make a new writer:
dir.deleteFiles(Arrays.asList(dir.listAll()));
for(String name : dir.listAll()) {
dir.deleteFile(name);
}
w = new RandomIndexWriter(random(), dir);
w.addDocument(doc);

4
lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
View File

@ -21,10 +21,8 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -82,7 +80,7 @@ public class TestDirectory extends LuceneTestCase {
}
// delete with a different dir
dirs[(i+1)%dirs.length].deleteFiles(Collections.singleton(fname));
dirs[(i+1)%dirs.length].deleteFile(fname);
for (int j=0; j<dirs.length; j++) {
FSDirectory d2 = dirs[j];

3
lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@ -88,7 +87,7 @@ public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
Lock lock = dir.obtainLock("test.lock");
lock.ensureValid();
dir.deleteFiles(Collections.singleton("test.lock"));
dir.deleteFile("test.lock");
try {
lock.ensureValid();

3
lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@ -41,7 +40,7 @@ public class TestSimpleFSLockFactory extends BaseLockFactoryTestCase {
lock.ensureValid();
try {
dir.deleteFiles(Collections.singleton("test.lock"));
dir.deleteFile("test.lock");
} catch (Exception e) {
// we can't delete a file for some reason, just clean up and assume the test.
IOUtils.closeWhileHandlingException(lock);

2
lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
View File

@ -43,7 +43,7 @@ public class TestTrackingDirectoryWrapper extends BaseDirectoryTestCase {
TrackingDirectoryWrapper dir = new TrackingDirectoryWrapper(new RAMDirectory());
dir.createOutput("foo", newIOContext(random())).close();
assertEquals(asSet("foo"), dir.getCreatedFiles());
dir.deleteFiles(Collections.singleton("foo"));
dir.deleteFile("foo");
assertEquals(Collections.emptySet(), dir.getCreatedFiles());
}

5
lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
View File

@ -22,7 +22,6 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import org.apache.lucene.index.PointValues.IntersectVisitor;
@ -818,9 +817,9 @@ public class TestBKD extends LuceneTestCase {
}
}
in.close();
dir.deleteFiles(Collections.singleton("bkd"));
dir.deleteFile("bkd");
if (toMerge != null) {
dir.deleteFiles(Collections.singleton("bkd2"));
dir.deleteFile("bkd2");
}
success = true;
} finally {

7
lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.util.fst;
*/
import java.util.Arrays;
import java.util.Collections;
import java.util.Random;
import org.apache.lucene.store.Directory;
@ -128,7 +127,7 @@ public class Test2BFST extends LuceneTestCase {
fst = new FST<>(in, outputs);
in.close();
} else {
dir.deleteFiles(Collections.singleton("fst"));
dir.deleteFile("fst");
}
}
}
@ -205,7 +204,7 @@ public class Test2BFST extends LuceneTestCase {
fst = new FST<>(in, outputs);
in.close();
} else {
dir.deleteFiles(Collections.singleton("fst"));
dir.deleteFile("fst");
}
}
}
@ -289,7 +288,7 @@ public class Test2BFST extends LuceneTestCase {
fst = new FST<>(in, outputs);
in.close();
} else {
dir.deleteFiles(Collections.singleton("fst"));
dir.deleteFile("fst");
}
}
}

3
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -22,7 +22,6 @@ import java.nio.ByteBuffer;
import java.nio.LongBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Random;
@ -841,7 +840,7 @@ public class TestPackedInts extends LuceneTestCase {
assertEquals(mutable.get(i), reader.get(i));
}
in.close();
directory.deleteFiles(Collections.singleton("packed-ints.bin"));
directory.deleteFile("packed-ints.bin");
}
directory.close();
}

1
lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
View File

@ -148,7 +148,6 @@ public class NativeUnixDirectory extends FSDirectory {
if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect) {
return delegate.createOutput(name, context);
} else {
ensureCanWrite(name);
return new NativeUnixIndexOutput(getDirectory().resolve(name), name, mergeBufferSize);
}
}

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
View File

@ -487,7 +487,7 @@ public class AnalyzingSuggester extends Lookup implements Accountable {
tempSortedFileName = sorter.sort(tempInput.getName());
// Free disk space:
tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempDir.deleteFile(tempInput.getName());
reader = new OfflineSorter.ByteSequencesReader(tempDir.openInput(tempSortedFileName, IOContext.READONCE));

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
View File

@ -69,7 +69,7 @@ public class ExternalRefSorter implements BytesRefSorter, Closeable {
success = true;
} finally {
if (success) {
sorter.getDirectory().deleteFiles(Collections.singleton(input.getName()));
sorter.getDirectory().deleteFile(input.getName());
} else {
IOUtils.deleteFilesIgnoringExceptions(sorter.getDirectory(), input.getName());
}

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
View File

@ -204,7 +204,7 @@ public class FSTCompletionLookup extends Lookup implements Accountable {
// We don't know the distribution of scores and we need to bucket them, so we'll sort
// and divide into equal buckets.
tempSortedFileName = sorter.sort(tempInput.getName());
tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempDir.deleteFile(tempInput.getName());
FSTCompletionBuilder builder = new FSTCompletionBuilder(
buckets, externalSorter, sharedTailLength);

2
lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java
View File

@ -76,7 +76,7 @@ public class VocabularyAssert {
/** Run a vocabulary test against a tab-separated data file inside a zip file */
public static void assertVocabulary(Analyzer a, Path zipFile, String vocOut) throws IOException {
Path tmp = LuceneTestCase.createTempDir();
Path tmp = LuceneTestCase.createTempDir().resolve("unzipped");
try (InputStream in = Files.newInputStream(zipFile)) {
TestUtil.unzip(in, tmp);
}

2
lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
View File

@ -247,7 +247,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
cfs.deleteFiles(Collections.singleton(testfile));
cfs.deleteFile(testfile);
fail("didn't get expected exception");
} catch (UnsupportedOperationException expected) {
// expected UOE

6
lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
View File

@ -46,6 +46,10 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
this.random = new Random(random.nextLong());
}
public void enable() {
enabled = true;
}
public void disable() {
enabled = false;
}
@ -57,7 +61,7 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
&& Files.exists(path) // important that we NOT delay a NoSuchFileException until later
&& path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
&& random.nextInt(5) == 1) {
if (true || LuceneTestCase.VERBOSE) {
if (LuceneTestCase.VERBOSE) {
System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
}
throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");

5
lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
View File

@ -164,7 +164,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
int count = dir.listAll().length;
dir.createOutput("foo.txt", IOContext.DEFAULT).close();
assertEquals(count+1, dir.listAll().length);
dir.deleteFiles(Collections.singleton("foo.txt"));
dir.deleteFile("foo.txt");
assertEquals(count, dir.listAll().length);
dir.close();
}
@ -751,7 +751,8 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
in2.close();
dir.deleteFiles(Arrays.asList(new String[] {"test", "test2"}));
dir.deleteFile("test");
dir.deleteFile("test2");
dir.close();
}

29
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -43,7 +43,6 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoDeletionPolicy;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -279,7 +278,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
if (damage == 0) {
action = "deleted";
deleteFiles(Collections.singleton(name));
deleteFile(name);
} else if (damage == 1) {
action = "zeroed";
// Zero out file entirely
@ -313,7 +312,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
ii.close();
// Delete original and copy bytes back:
deleteFiles(Collections.singleton(name));
deleteFile(name);
try(IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState));
@ -328,14 +327,14 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
throw ioe;
}
}
deleteFiles(Collections.singleton(tempFileName));
deleteFile(tempFileName);
} else if (damage == 3) {
// The file survived intact:
action = "didn't change";
} else {
action = "fully truncated";
// Totally truncate the file to zero bytes
deleteFiles(Collections.singleton(name));
deleteFile(name);
try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
} catch (IOException ioe) {
// VirusCheckingFS may have blocked the delete, at which point FSDir cannot overwrite here
@ -449,7 +448,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
}
@Override
public synchronized void deleteFiles(Collection<String> names) throws IOException {
public synchronized void deleteFile(String name) throws IOException {
maybeYield();
maybeThrowDeterministicException();
@ -458,19 +457,17 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
throw new IOException("cannot delete after crash");
}
for(String name : names) {
if (openFiles.containsKey(name)) {
openFilesDeleted.add(name);
if (assertNoDeleteOpenFile) {
throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
}
} else {
openFilesDeleted.remove(name);
if (openFiles.containsKey(name)) {
openFilesDeleted.add(name);
if (assertNoDeleteOpenFile) {
throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
}
} else {
openFilesDeleted.remove(name);
}
unSyncedFiles.removeAll(names);
in.deleteFiles(names);
unSyncedFiles.remove(name);
in.deleteFile(name);
}
// sets the cause of the incoming ioe to be the stack

2
lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
View File

@ -317,7 +317,7 @@ public class FSTTester<T> {
fst = new FST<>(in, outputs);
} finally {
in.close();
dir.deleteFiles(Collections.singleton("fst.bin"));
dir.deleteFile("fst.bin");
}
}