From a741ea53c9570d290fea95751a48b857fadd87dd Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 3 Feb 2016 10:35:57 -0500 Subject: [PATCH] cut back to Directory.deleteFile(String); disable 'could not removed segments_N so I don't remove any other files it may reference' heroics --- .../lucene/analysis/hunspell/Dictionary.java | 4 +- .../util/TestFilesystemResourceLoader.java | 28 +++--- .../simpletext/SimpleTextCompoundFormat.java | 2 +- .../lucene50/Lucene50CompoundReader.java | 2 +- .../apache/lucene/index/IndexFileDeleter.java | 30 +++--- .../org/apache/lucene/index/IndexWriter.java | 1 + .../PersistentSnapshotDeletionPolicy.java | 4 +- .../org/apache/lucene/store/Directory.java | 5 +- .../org/apache/lucene/store/FSDirectory.java | 97 +++---------------- .../lucene/store/FileSwitchDirectory.java | 19 +--- .../apache/lucene/store/FilterDirectory.java | 4 +- .../store/LockValidatingDirectoryWrapper.java | 4 +- .../lucene/store/NRTCachingDirectory.java | 29 ++---- .../org/apache/lucene/store/RAMDirectory.java | 16 ++- .../store/TrackingDirectoryWrapper.java | 9 +- .../java/org/apache/lucene/util/IOUtils.java | 30 +++--- .../org/apache/lucene/util/bkd/BKDWriter.java | 7 +- .../lucene/util/bkd/OfflinePointWriter.java | 3 +- .../lucene/index/TestCodecHoldsOpenFiles.java | 7 +- .../lucene/index/TestDeletionPolicy.java | 12 +-- .../index/TestDirectoryReaderReopen.java | 9 +- .../test/org/apache/lucene/index/TestDoc.java | 5 +- .../lucene/index/TestIndexFileDeleter.java | 32 +++--- .../index/TestIndexWriterExceptions.java | 10 +- .../lucene/index/TestNRTReaderCleanup.java | 8 +- .../apache/lucene/store/TestDirectory.java | 4 +- .../lucene/store/TestNativeFSLockFactory.java | 3 +- .../lucene/store/TestSimpleFSLockFactory.java | 3 +- .../store/TestTrackingDirectoryWrapper.java | 2 +- .../org/apache/lucene/util/bkd/TestBKD.java | 5 +- .../org/apache/lucene/util/fst/Test2BFST.java | 7 +- .../lucene/util/packed/TestPackedInts.java | 3 +- .../lucene/store/NativeUnixDirectory.java | 1 - .../suggest/analyzing/AnalyzingSuggester.java | 2 +- .../search/suggest/fst/ExternalRefSorter.java | 2 +- .../suggest/fst/FSTCompletionLookup.java | 2 +- .../lucene/analysis/VocabularyAssert.java | 2 +- .../index/BaseCompoundFormatTestCase.java | 2 +- .../lucene/mockfile/VirusCheckingFS.java | 6 +- .../lucene/store/BaseDirectoryTestCase.java | 5 +- .../lucene/store/MockDirectoryWrapper.java | 29 +++--- .../org/apache/lucene/util/fst/FSTTester.java | 2 +- 42 files changed, 181 insertions(+), 276 deletions(-) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java index 49c7045ee7d..d5db8391bae 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java @@ -877,7 +877,7 @@ public class Dictionary { success = true; } finally { if (success) { - tempDir.deleteFiles(Collections.singleton(unsorted.getName())); + tempDir.deleteFile(unsorted.getName()); } else { IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName()); } @@ -966,7 +966,7 @@ public class Dictionary { success2 = true; } finally { if (success2) { - tempDir.deleteFiles(Collections.singleton(sorted)); + tempDir.deleteFile(sorted); } else { IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted); } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java index 80de40ffbd8..6ca814ef2fd 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java @@ -57,25 +57,21 @@ public class TestFilesystemResourceLoader extends LuceneTestCase { public void testBaseDir() throws Exception { final Path base = createTempDir("fsResourceLoaderBase"); + Writer os = Files.newBufferedWriter(base.resolve("template.txt"), StandardCharsets.UTF_8); try { - Writer os = Files.newBufferedWriter(base.resolve("template.txt"), StandardCharsets.UTF_8); - try { - os.write("foobar\n"); - } finally { - IOUtils.closeWhileHandlingException(os); - } - - ResourceLoader rl = new FilesystemResourceLoader(base); - assertEquals("foobar", WordlistLoader.getLines(rl.openResource("template.txt"), StandardCharsets.UTF_8).get(0)); - // Same with full path name: - String fullPath = base.resolve("template.txt").toAbsolutePath().toString(); - assertEquals("foobar", - WordlistLoader.getLines(rl.openResource(fullPath), StandardCharsets.UTF_8).get(0)); - assertClasspathDelegation(rl); - assertNotFound(rl); + os.write("foobar\n"); } finally { - IOUtils.rm(base); + IOUtils.closeWhileHandlingException(os); } + + ResourceLoader rl = new FilesystemResourceLoader(base); + assertEquals("foobar", WordlistLoader.getLines(rl.openResource("template.txt"), StandardCharsets.UTF_8).get(0)); + // Same with full path name: + String fullPath = base.resolve("template.txt").toAbsolutePath().toString(); + assertEquals("foobar", + WordlistLoader.getLines(rl.openResource(fullPath), StandardCharsets.UTF_8).get(0)); + assertClasspathDelegation(rl); + assertNotFound(rl); } public void testDelegation() throws Exception { diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java index 8398f66fcc5..c994df77247 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java @@ -148,7 +148,7 @@ public class SimpleTextCompoundFormat extends CompoundFormat { public void sync(Collection names) { throw new UnsupportedOperationException(); } @Override - public void deleteFiles(Collection name) { throw new UnsupportedOperationException(); } + public void deleteFile(String name) { throw new UnsupportedOperationException(); } @Override public void renameFile(String source, String dest) { throw new UnsupportedOperationException(); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java index a6ccfd4986e..831fb4cf4ef 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java @@ -161,7 +161,7 @@ final class Lucene50CompoundReader extends Directory { /** Not implemented * @throws UnsupportedOperationException always: not supported by CFS */ @Override - public void deleteFiles(Collection name) { + public void deleteFile(String name) { throw new UnsupportedOperationException(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java index 569471eab4b..2e5d5cff19c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java @@ -694,20 +694,24 @@ final class IndexFileDeleter implements Closeable { private void deleteFiles(Collection names) throws IOException { assert locked(); ensureOpen(); - if (names.isEmpty()) { - return; + + if (infoStream.isEnabled("IFD")) { + infoStream.message("IFD", "delete \"" + names + "\""); } - try { - if (infoStream.isEnabled("IFD")) { - infoStream.message("IFD", "delete \"" + names + "\""); - } - directory.deleteFiles(names); - } catch (NoSuchFileException | FileNotFoundException e) { // if delete fails - // IndexWriter should only ask us to delete files it knows it wrote, so if we hit this, something is wrong! - if (Constants.WINDOWS) { - // LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state: - } else { - throw e; + + // nocommit put annoying windows-specific segments_N heroics back? + + for(String name : names) { + try { + directory.deleteFile(name); + } catch (NoSuchFileException | FileNotFoundException e) { + // IndexWriter should only ask us to delete files it knows it wrote, so if we hit this, something is wrong! + if (Constants.WINDOWS) { + // LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state, and falsely + // return NSFE/FNFE + } else { + throw e; + } } } } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index e62cafbd480..ce793093575 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -4617,6 +4617,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { * commits are no longer needed. Otherwise, those commits will * be deleted the next time commit() is called. */ + // nocommit remove this public synchronized void deleteUnusedFiles() throws IOException { ensureOpen(false); deleter.revisitPolicy(); diff --git a/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java b/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java index 2a286ff60aa..e95c5676f73 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java @@ -22,8 +22,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Map.Entry; import java.util.Map; +import java.util.Map.Entry; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -212,7 +212,7 @@ public class PersistentSnapshotDeletionPolicy extends SnapshotDeletionPolicy { private synchronized void clearPriorSnapshots() throws IOException { for(String file : dir.listAll()) { if (file.startsWith(SNAPSHOTS_PREFIX)) { - dir.deleteFiles(Collections.singleton(file)); + dir.deleteFile(file); } } } diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java index bb12c928533..b6da4cc0e6d 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java @@ -50,9 +50,8 @@ public abstract class Directory implements Closeable { // nocommit should this sort? public abstract String[] listAll() throws IOException; - /** Removes the specified files from the directory. If an exception is thrown, behavior is undefined - * (none, some or all of the files may have in fact been deleted). */ - public abstract void deleteFiles(Collection name) throws IOException; + /** Removes an existing file in the directory. */ + public abstract void deleteFile(String name) throws IOException; /** * Returns the length of a file in the directory. This method follows the diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java index 9eb78b43d4b..a2edb844826 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java @@ -126,10 +126,8 @@ public abstract class FSDirectory extends BaseDirectory { protected final Path directory; // The underlying filesystem directory - - /** Files we previously tried to delete, but hit exception (on Windows) last time we tried. - * These files are in "pending delete" state, where we refuse to openInput or createOutput - * them, nor include them in .listAll. */ + /** Maps files that we are trying to delete (or we tried already but failed) + * before attempting to delete that key. */ protected final Set pendingDeletes = Collections.newSetFromMap(new ConcurrentHashMap()); /** Used to generate temp file names in {@link #createTempOutput}. */ @@ -200,12 +198,11 @@ public abstract class FSDirectory extends BaseDirectory { } } - /** Lists all files (including subdirectories) in the - * directory. + /** Lists all files (including subdirectories) in the directory. * * @throws IOException if there was an I/O error during listing */ public static String[] listAll(Path dir) throws IOException { - return listAll(dir, Collections.emptySet()); + return listAll(dir, null); } private static String[] listAll(Path dir, Set skipNames) throws IOException { @@ -214,7 +211,7 @@ public abstract class FSDirectory extends BaseDirectory { try (DirectoryStream stream = Files.newDirectoryStream(dir)) { for (Path path : stream) { String name = path.getFileName().toString(); - if (skipNames.contains(name) == false) { + if (skipNames != null && skipNames.contains(name) == false) { entries.add(name); } } @@ -235,18 +232,11 @@ public abstract class FSDirectory extends BaseDirectory { return Files.size(directory.resolve(name)); } - @Override - public void deleteFiles(Collection names) throws IOException { - ensureOpen(); - // nocommit isn't it an error if they were already pending delete? - pendingDeletes.addAll(names); - deletePendingFiles(); - } - @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); - ensureCanWrite(name); + // nocommit do we need to check pending deletes? + deletePendingFiles(); return new FSIndexOutput(name); } @@ -264,14 +254,6 @@ public abstract class FSDirectory extends BaseDirectory { } } - protected void ensureCanWrite(String name) throws IOException { - deletePendingFiles(); - if (pendingDeletes.contains(name)) { - throw new IOException("file \"" + name + "\" is pending delete and cannot be overwritten"); - } - Files.deleteIfExists(directory.resolve(name)); // delete existing, if any - } - protected void ensureCanRead(String name) throws IOException { deletePendingFiles(); if (pendingDeletes.contains(name)) { @@ -319,12 +301,11 @@ public abstract class FSDirectory extends BaseDirectory { IOUtils.fsync(directory.resolve(name), false); } - /** Returns true if the file was successfully removed. */ - private synchronized boolean deleteFile(String name) throws IOException { + @Override + public void deleteFile(String name) throws IOException { pendingDeletes.remove(name); try { Files.delete(directory.resolve(name)); - return true; } catch (NoSuchFileException | FileNotFoundException e) { // We were asked to delete a non-existent file: throw e; @@ -339,9 +320,7 @@ public abstract class FSDirectory extends BaseDirectory { // TODO: can/should we do if (Constants.WINDOWS) here, else throw the exc? // but what about a Linux box with a CIFS mount? - //System.out.println("FS.deleteFile failed (" + ioe + "): will retry later"); pendingDeletes.add(name); - return false; } } @@ -354,61 +333,17 @@ public abstract class FSDirectory extends BaseDirectory { /** Try to delete any pending files that we had previously tried to delete but failed * because we are on Windows and the files were still held open. */ - public synchronized void deletePendingFiles() throws IOException { + public void deletePendingFiles() throws IOException { + // nocommit do we need exponential backoff here for windows? + // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed? - // Clone the set because it will change as we iterate: - List toDelete = new ArrayList<>(pendingDeletes); - System.out.println("del pending: " + pendingDeletes); + Set toDelete = new HashSet<>(pendingDeletes); - // First pass: delete any segments_N files. We do these first to be certain stale commit points are removed - // before we remove any files they reference. If any delete of segments_N fails, we leave all other files - // undeleted so index is never in a corrupt state: - Throwable firstException = null; - for (String fileName : toDelete) { - if (fileName.startsWith(IndexFileNames.SEGMENTS)) { - try { - if (deleteFile(fileName) == false) { - // nocommit - System.out.println(" false on " + fileName + "; skipping the rest"); - return; - } - } catch (Throwable t) { - if (firstException == null) { - firstException = t; - } else { - firstException.addSuppressed(t); - } - // nocommit - System.out.println(" fail on " + fileName + ":"); - t.printStackTrace(System.out); - throw t; - } - } + // nocommit heroic exceptions here or not? + for(String name : toDelete) { + deleteFile(name); } - - // Only delete other files if we were able to remove the segments_N files; this way we never - // leave a corrupt commit in the index even in the presense of virus checkers: - for(String fileName : toDelete) { - if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) { - try { - deleteFile(fileName); - } catch (Throwable t) { - if (firstException == null) { - firstException = t; - } else { - firstException.addSuppressed(t); - } - // nocommit - System.out.println(" fail on " + fileName + ":"); - t.printStackTrace(System.out); - throw t; - } - } - } - - // Does nothing if firstException is null: - IOUtils.reThrow(firstException); } final class FSIndexOutput extends OutputStreamIndexOutput { diff --git a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java index 13bc2175c80..59b8d57ad85 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java @@ -140,20 +140,11 @@ public class FileSwitchDirectory extends Directory { } @Override - public void deleteFiles(Collection names) throws IOException { - Set primaryToDelete = new HashSet<>(); - Set secondaryToDelete = new HashSet<>(); - for(String name : names) { - if (getDirectory(name) == primaryDir) { - primaryToDelete.add(name); - } else { - secondaryToDelete.add(name); - } - } - try { - primaryDir.deleteFiles(primaryToDelete); - } finally { - secondaryDir.deleteFiles(secondaryToDelete); + public void deleteFile(String name) throws IOException { + if (getDirectory(name) == primaryDir) { + primaryDir.deleteFile(name); + } else { + secondaryDir.deleteFile(name); } } diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java index 9ee2928d66b..7c550c1d610 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java @@ -58,8 +58,8 @@ public class FilterDirectory extends Directory { } @Override - public void deleteFiles(Collection names) throws IOException { - in.deleteFiles(names); + public void deleteFile(String name) throws IOException { + in.deleteFile(name); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java index 3ed659a43a3..389c56ddfe4 100644 --- a/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java @@ -33,9 +33,9 @@ public final class LockValidatingDirectoryWrapper extends FilterDirectory { } @Override - public void deleteFiles(Collection names) throws IOException { + public void deleteFile(String name) throws IOException { writeLock.ensureValid(); - in.deleteFiles(names); + in.deleteFile(name); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java index 1b8404bd197..5a38b441206 100644 --- a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java @@ -20,12 +20,10 @@ package org.apache.lucene.store; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.NoSuchFileException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.apache.lucene.store.RAMDirectory; // javadocs @@ -112,23 +110,14 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable } @Override - public synchronized void deleteFiles(Collection names) throws IOException { + public synchronized void deleteFile(String name) throws IOException { if (VERBOSE) { - System.out.println("nrtdir.deleteFiles names=" + names); + System.out.println("nrtdir.deleteFile name=" + name); } - Set cacheToDelete = new HashSet<>(); - Set toDelete = new HashSet<>(); - for(String name : names) { - if (cache.fileNameExists(name)) { - cacheToDelete.add(name); - } else { - toDelete.add(name); - } - } - try { - cache.deleteFiles(cacheToDelete); - } finally { - in.deleteFiles(toDelete); + if (cache.fileNameExists(name)) { + cache.deleteFile(name); + } else { + in.deleteFile(name); } } @@ -155,14 +144,14 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable System.out.println(" to cache"); } try { - in.deleteFiles(Collections.singleton(name)); + in.deleteFile(name); } catch (IOException ioe) { // This is fine: file may not exist } return cache.createOutput(name, context); } else { try { - cache.deleteFiles(Collections.singleton(name)); + cache.deleteFile(name); } catch (IOException ioe) { // This is fine: file may not exist } @@ -332,7 +321,7 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable synchronized(this) { // Must sync here because other sync methods have // if (cache.fileNameExists(name)) { ... } else { ... }: - cache.deleteFiles(Collections.singleton(fileName)); + cache.deleteFile(fileName); } } } diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java index 7be4679c33b..d1dc0d071b6 100644 --- a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java @@ -157,16 +157,14 @@ public class RAMDirectory extends BaseDirectory implements Accountable { } @Override - public void deleteFiles(Collection names) throws IOException { + public void deleteFile(String name) throws IOException { ensureOpen(); - for(String name : names) { - RAMFile file = fileMap.remove(name); - if (file != null) { - file.directory = null; - sizeInBytes.addAndGet(-file.sizeInBytes); - } else { - throw new FileNotFoundException(name); - } + RAMFile file = fileMap.remove(name); + if (file != null) { + file.directory = null; + sizeInBytes.addAndGet(-file.sizeInBytes); + } else { + throw new FileNotFoundException(name); } } diff --git a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java index 49d70f86874..aa7214c2bff 100644 --- a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java @@ -18,7 +18,6 @@ package org.apache.lucene.store; */ import java.io.IOException; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -34,11 +33,9 @@ public final class TrackingDirectoryWrapper extends FilterDirectory { } @Override - public void deleteFiles(Collection names) throws IOException { - in.deleteFiles(names); - for(String name : names) { - createdFileNames.remove(name); - } + public void deleteFile(String name) throws IOException { + in.deleteFile(name); + createdFileNames.remove(name); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java index 3be3d7f5d56..f6a62723169 100644 --- a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java +++ b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java @@ -38,10 +38,8 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Set; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -193,10 +191,12 @@ public final class IOUtils { * Note that the files should not be null. */ public static void deleteFilesIgnoringExceptions(Directory dir, Collection files) { - try { - dir.deleteFiles(files); - } catch (Throwable ignored) { - // ignore + for(String name : files) { + try { + dir.deleteFile(name); + } catch (Throwable ignored) { + // ignore + } } } @@ -215,15 +215,21 @@ public final class IOUtils { * @param names file names to delete */ public static void deleteFiles(Directory dir, Collection names) throws IOException { - Set nonNullNames = new HashSet<>(); - for(String name : names) { + Throwable th = null; + for (String name : names) { if (name != null) { - nonNullNames.add(name); + try { + dir.deleteFile(name); + } catch (Throwable t) { + addSuppressed(th, t); + if (th == null) { + th = t; + } + } } } - if (names.isEmpty() == false) { - dir.deleteFiles(names); - } + + reThrow(th); } public static void deleteFiles(Directory dir, String... files) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 328ea2e1575..e47830db94d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -22,7 +22,6 @@ import java.io.EOFException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -40,8 +39,8 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntroSorter; import org.apache.lucene.util.LongBitSet; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter; import org.apache.lucene.util.OfflineSorter; +import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter; import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.StringHelper; @@ -821,7 +820,7 @@ public class BKDWriter implements Closeable { //System.out.println("sort time: " + ((t1-t0)/1000000.0) + " msec"); if (tempInput != null) { - tempDir.deleteFiles(Collections.singleton(tempInput.getName())); + tempDir.deleteFile(tempInput.getName()); tempInput = null; } else { assert heapPointWriter != null; @@ -914,7 +913,7 @@ public class BKDWriter implements Closeable { try { tempInput.close(); } finally { - tempDir.deleteFiles(Collections.singleton(tempInput.getName())); + tempDir.deleteFile(tempInput.getName()); tempInput = null; } } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index 0751354a578..83ff90b564d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -18,7 +18,6 @@ package org.apache.lucene.util.bkd; */ import java.io.IOException; -import java.util.Collections; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -75,7 +74,7 @@ final class OfflinePointWriter implements PointWriter { @Override public void destroy() throws IOException { - tempDir.deleteFiles(Collections.singleton(out.getName())); + tempDir.deleteFile(out.getName()); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java index fa773fa1b8c..4ffc1f87c29 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java @@ -17,9 +17,6 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; -import java.util.Arrays; - import org.apache.lucene.document.Document; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; @@ -48,7 +45,9 @@ public class TestCodecHoldsOpenFiles extends LuceneTestCase { w.commit(); w.close(); - d.deleteFiles(Arrays.asList(d.listAll())); + for (String name : d.listAll()) { + d.deleteFile(name); + } for(LeafReaderContext cxt : r.leaves()) { TestUtil.checkReader(cxt.reader()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java index 6d7d1def170..f15f673d748 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java @@ -19,7 +19,6 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,9 +34,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.Version; @@ -297,7 +295,7 @@ public class TestDeletionPolicy extends LuceneTestCase { break; } - dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen))); + dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; } @@ -375,7 +373,7 @@ public class TestDeletionPolicy extends LuceneTestCase { while(gen > 0) { IndexReader reader = DirectoryReader.open(dir); reader.close(); - dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen))); + dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; if (gen > 0) { @@ -602,7 +600,7 @@ public class TestDeletionPolicy extends LuceneTestCase { } } if (i < N) { - dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen))); + dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); } gen--; } @@ -716,7 +714,7 @@ public class TestDeletionPolicy extends LuceneTestCase { } } if (i < N) { - dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen))); + dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); } gen--; } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java index 934ec732991..2040c7266de 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java @@ -19,7 +19,6 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -40,8 +39,8 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException; import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -708,7 +707,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { // Blow away the index: for(String fileName : dir.listAll()) { - dir.deleteFiles(Collections.singleton(fileName)); + dir.deleteFile(fileName); } w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); @@ -757,7 +756,9 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(dir); // Blow away the index: - dir.deleteFiles(Arrays.asList(dir.listAll())); + for(String name : dir.listAll()) { + dir.deleteFile(name); + } w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); doc = new Document(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java index 24ce405aeb3..705c68bdcb0 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java @@ -48,7 +48,6 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.StringHelper; -import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.Version; /** JUnit adaptation of an older test case DocTest. */ @@ -234,7 +233,9 @@ public class TestDoc extends LuceneTestCase { Collection filesToDelete = si.files(); codec.compoundFormat().write(dir, si, context); si.setUseCompoundFile(true); - si1.info.dir.deleteFiles(filesToDelete); + for(String name : filesToDelete) { + si1.info.dir.deleteFile(name); + } } return new SegmentCommitInfo(si, 0, -1L, -1L, -1L); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java index fa9f1a7e518..a5e3c7e5ff4 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java @@ -18,6 +18,9 @@ package org.apache.lucene.index; */ import java.io.*; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Path; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -26,6 +29,8 @@ import org.apache.lucene.codecs.simpletext.SimpleTextCodec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.mockfile.FilterPath; +import org.apache.lucene.mockfile.VirusCheckingFS; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -219,8 +224,14 @@ public class TestIndexFileDeleter extends LuceneTestCase { } public void testVirusScannerDoesntCorruptIndex() throws IOException { - MockDirectoryWrapper dir = newMockDirectory(); - dir.setPreventDoubleWrite(false); // we arent trying to test this + Path path = createTempDir(); + VirusCheckingFS fs = new VirusCheckingFS(path.getFileSystem(), random()); + FileSystem filesystem = fs.getFileSystem(URI.create("file:///")); + fs.disable(); + + Path path2 = new FilterPath(path, filesystem); + + Directory dir = newFSDirectory(path2); // add empty commit new IndexWriter(dir, new IndexWriterConfig(null)).close(); @@ -228,25 +239,12 @@ public class TestIndexFileDeleter extends LuceneTestCase { dir.createOutput("_0.si", IOContext.DEFAULT).close(); // start virus scanner - final AtomicBoolean stopScanning = new AtomicBoolean(); - dir.failOn(new MockDirectoryWrapper.Failure() { - @Override - public void eval(MockDirectoryWrapper dir) throws IOException { - if (stopScanning.get()) { - return; - } - for (StackTraceElement f : new Exception().getStackTrace()) { - if ("deleteFile".equals(f.getMethodName())) { - throw new IOException("temporarily cannot delete file"); - } - } - } - }); + fs.enable(); IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null)); iw.addDocument(new Document()); // stop virus scanner - stopScanning.set(true); + fs.disable(); iw.commit(); iw.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index db2253bf075..6daa3ca6a8f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -57,14 +57,14 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException; import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.TestUtil; @SuppressCodecs("SimpleText") // too slow here @@ -916,7 +916,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) { isCommit = true; } - if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFiles".equals(trace[i].getMethodName())) { + if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName())) { isDelete = true; } if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) { @@ -1205,7 +1205,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } in.close(); out.close(); - dir.deleteFiles(Collections.singleton(fileNameIn)); + dir.deleteFile(fileNameIn); IndexReader reader = null; try { @@ -1255,7 +1255,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { assertTrue(si.info.getUseCompoundFile()); List victims = new ArrayList(si.info.files()); Collections.shuffle(victims, random()); - dir.deleteFiles(Collections.singleton(victims.get(0))); + dir.deleteFile(victims.get(0)); corrupted = true; break; } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java b/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java index bb452bf993a..1d53c5acc74 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java @@ -18,7 +18,6 @@ package org.apache.lucene.index; */ import java.io.IOException; -import java.util.Arrays; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -26,9 +25,8 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.Constants; -import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; /** LUCENE-5574 */ @SuppressFileSystems("WindowsFS") // the bug doesn't happen on windows. @@ -62,7 +60,9 @@ public class TestNRTReaderCleanup extends LuceneTestCase { w.close(); // Blow away index and make a new writer: - dir.deleteFiles(Arrays.asList(dir.listAll())); + for(String name : dir.listAll()) { + dir.deleteFile(name); + } w = new RandomIndexWriter(random(), dir); w.addDocument(doc); diff --git a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java index 3b653892830..9c68b9e1b70 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java @@ -21,10 +21,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.Collections; import java.util.List; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -82,7 +80,7 @@ public class TestDirectory extends LuceneTestCase { } // delete with a different dir - dirs[(i+1)%dirs.length].deleteFiles(Collections.singleton(fname)); + dirs[(i+1)%dirs.length].deleteFile(fname); for (int j=0; j(in, outputs); in.close(); } else { - dir.deleteFiles(Collections.singleton("fst")); + dir.deleteFile("fst"); } } } @@ -205,7 +204,7 @@ public class Test2BFST extends LuceneTestCase { fst = new FST<>(in, outputs); in.close(); } else { - dir.deleteFiles(Collections.singleton("fst")); + dir.deleteFile("fst"); } } } @@ -289,7 +288,7 @@ public class Test2BFST extends LuceneTestCase { fst = new FST<>(in, outputs); in.close(); } else { - dir.deleteFiles(Collections.singleton("fst")); + dir.deleteFile("fst"); } } } diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java index e11cfeff9ed..bab144b7cfc 100644 --- a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java +++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java @@ -22,7 +22,6 @@ import java.nio.ByteBuffer; import java.nio.LongBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Random; @@ -841,7 +840,7 @@ public class TestPackedInts extends LuceneTestCase { assertEquals(mutable.get(i), reader.get(i)); } in.close(); - directory.deleteFiles(Collections.singleton("packed-ints.bin")); + directory.deleteFile("packed-ints.bin"); } directory.close(); } diff --git a/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java b/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java index 43763e21d85..51d7e92aa28 100644 --- a/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java +++ b/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java @@ -148,7 +148,6 @@ public class NativeUnixDirectory extends FSDirectory { if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect) { return delegate.createOutput(name, context); } else { - ensureCanWrite(name); return new NativeUnixIndexOutput(getDirectory().resolve(name), name, mergeBufferSize); } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java index cb2d38c2c1f..f54ad03c616 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java @@ -487,7 +487,7 @@ public class AnalyzingSuggester extends Lookup implements Accountable { tempSortedFileName = sorter.sort(tempInput.getName()); // Free disk space: - tempDir.deleteFiles(Collections.singleton(tempInput.getName())); + tempDir.deleteFile(tempInput.getName()); reader = new OfflineSorter.ByteSequencesReader(tempDir.openInput(tempSortedFileName, IOContext.READONCE)); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java index 132c630c801..7f907afe489 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java @@ -69,7 +69,7 @@ public class ExternalRefSorter implements BytesRefSorter, Closeable { success = true; } finally { if (success) { - sorter.getDirectory().deleteFiles(Collections.singleton(input.getName())); + sorter.getDirectory().deleteFile(input.getName()); } else { IOUtils.deleteFilesIgnoringExceptions(sorter.getDirectory(), input.getName()); } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java index d178aa4e0dd..24dd0f9e2df 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java @@ -204,7 +204,7 @@ public class FSTCompletionLookup extends Lookup implements Accountable { // We don't know the distribution of scores and we need to bucket them, so we'll sort // and divide into equal buckets. tempSortedFileName = sorter.sort(tempInput.getName()); - tempDir.deleteFiles(Collections.singleton(tempInput.getName())); + tempDir.deleteFile(tempInput.getName()); FSTCompletionBuilder builder = new FSTCompletionBuilder( buckets, externalSorter, sharedTailLength); diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java index de77e96326d..208eab9cb10 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java +++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java @@ -76,7 +76,7 @@ public class VocabularyAssert { /** Run a vocabulary test against a tab-separated data file inside a zip file */ public static void assertVocabulary(Analyzer a, Path zipFile, String vocOut) throws IOException { - Path tmp = LuceneTestCase.createTempDir(); + Path tmp = LuceneTestCase.createTempDir().resolve("unzipped"); try (InputStream in = Files.newInputStream(zipFile)) { TestUtil.unzip(in, tmp); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java index 0c40bb33e2c..7cee89e2343 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java @@ -247,7 +247,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT); Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT); try { - cfs.deleteFiles(Collections.singleton(testfile)); + cfs.deleteFile(testfile); fail("didn't get expected exception"); } catch (UnsupportedOperationException expected) { // expected UOE diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java index b30ce94982f..759a84046a8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java @@ -46,6 +46,10 @@ public class VirusCheckingFS extends FilterFileSystemProvider { this.random = new Random(random.nextLong()); } + public void enable() { + enabled = true; + } + public void disable() { enabled = false; } @@ -57,7 +61,7 @@ public class VirusCheckingFS extends FilterFileSystemProvider { && Files.exists(path) // important that we NOT delay a NoSuchFileException until later && path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file && random.nextInt(5) == 1) { - if (true || LuceneTestCase.VERBOSE) { + if (LuceneTestCase.VERBOSE) { System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path); } throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\""); diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index db8315318d9..ef8bf704d9f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -164,7 +164,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase { int count = dir.listAll().length; dir.createOutput("foo.txt", IOContext.DEFAULT).close(); assertEquals(count+1, dir.listAll().length); - dir.deleteFiles(Collections.singleton("foo.txt")); + dir.deleteFile("foo.txt"); assertEquals(count, dir.listAll().length); dir.close(); } @@ -751,7 +751,8 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase { } in2.close(); - dir.deleteFiles(Arrays.asList(new String[] {"test", "test2"})); + dir.deleteFile("test"); + dir.deleteFile("test2"); dir.close(); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 3bd231916a6..3c7941029af 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -43,7 +43,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoDeletionPolicy; -import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -279,7 +278,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { if (damage == 0) { action = "deleted"; - deleteFiles(Collections.singleton(name)); + deleteFile(name); } else if (damage == 1) { action = "zeroed"; // Zero out file entirely @@ -313,7 +312,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { ii.close(); // Delete original and copy bytes back: - deleteFiles(Collections.singleton(name)); + deleteFile(name); try(IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) { ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState)); @@ -328,14 +327,14 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { throw ioe; } } - deleteFiles(Collections.singleton(tempFileName)); + deleteFile(tempFileName); } else if (damage == 3) { // The file survived intact: action = "didn't change"; } else { action = "fully truncated"; // Totally truncate the file to zero bytes - deleteFiles(Collections.singleton(name)); + deleteFile(name); try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) { } catch (IOException ioe) { // VirusCheckingFS may have blocked the delete, at which point FSDir cannot overwrite here @@ -449,7 +448,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } @Override - public synchronized void deleteFiles(Collection names) throws IOException { + public synchronized void deleteFile(String name) throws IOException { maybeYield(); maybeThrowDeterministicException(); @@ -458,19 +457,17 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { throw new IOException("cannot delete after crash"); } - for(String name : names) { - if (openFiles.containsKey(name)) { - openFilesDeleted.add(name); - if (assertNoDeleteOpenFile) { - throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); - } - } else { - openFilesDeleted.remove(name); + if (openFiles.containsKey(name)) { + openFilesDeleted.add(name); + if (assertNoDeleteOpenFile) { + throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); } + } else { + openFilesDeleted.remove(name); } - unSyncedFiles.removeAll(names); - in.deleteFiles(names); + unSyncedFiles.remove(name); + in.deleteFile(name); } // sets the cause of the incoming ioe to be the stack diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java index 3800ff7f411..a2ace6195ba 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java @@ -317,7 +317,7 @@ public class FSTTester { fst = new FST<>(in, outputs); } finally { in.close(); - dir.deleteFiles(Collections.singleton("fst.bin")); + dir.deleteFile("fst.bin"); } }