LUCENE-7373: deprecate Directory.renameFile, which both renamed and fsync'd the directory, replacing it with separate rename and syncMetaData methods

This commit is contained in:
Mike McCandless 2016-07-11 11:15:22 -04:00
parent cf3a500542
commit df9efb8b6d
22 changed files with 118 additions and 30 deletions

View File

@ -41,6 +41,11 @@ New Features
analyzer for the Ukrainian language (Andriy Rysin via Mike analyzer for the Ukrainian language (Andriy Rysin via Mike
McCandless) McCandless)
* LUCENE-7373: Directory.renameFile, which did both renaming and fsync
of the directory metadata, has been deprecated; use the new separate
methods Directory.rename and Directory.syncMetaData instead (Robert Muir,
Uwe Schindler, Mike McCandless)
Bug Fixes Bug Fixes
* LUCENE-6662: Fixed potential resource leaks. (Rishabh Patel via Adrien Grand) * LUCENE-6662: Fixed potential resource leaks. (Rishabh Patel via Adrien Grand)

View File

@ -151,7 +151,10 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
public void deleteFile(String name) { throw new UnsupportedOperationException(); } public void deleteFile(String name) { throw new UnsupportedOperationException(); }
@Override @Override
public void renameFile(String source, String dest) { throw new UnsupportedOperationException(); } public void rename(String source, String dest) { throw new UnsupportedOperationException(); }
@Override
public void syncMetaData() { throw new UnsupportedOperationException(); }
@Override @Override
public Lock obtainLock(String name) { throw new UnsupportedOperationException(); } public Lock obtainLock(String name) { throw new UnsupportedOperationException(); }

View File

@ -168,10 +168,15 @@ final class Lucene50CompoundReader extends Directory {
/** Not implemented /** Not implemented
* @throws UnsupportedOperationException always: not supported by CFS */ * @throws UnsupportedOperationException always: not supported by CFS */
public void renameFile(String from, String to) { @Override
public void rename(String from, String to) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public void syncMetaData() {
}
/** Returns the length of a file in the directory. /** Returns the length of a file in the directory.
* @throws IOException if the file does not exist */ * @throws IOException if the file does not exist */
@Override @Override

View File

@ -788,7 +788,8 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
try { try {
final String src = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", generation); final String src = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", generation);
dest = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation); dest = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
dir.renameFile(src, dest); dir.rename(src, dest);
dir.syncMetaData();
success = true; success = true;
} finally { } finally {
if (!success) { if (!success) {

View File

@ -99,8 +99,32 @@ public abstract class Directory implements Closeable {
* both {@code source} and {@code dest} can be visible temporarily. * both {@code source} and {@code dest} can be visible temporarily.
* It is just important that the contents of {@code dest} appear * It is just important that the contents of {@code dest} appear
* atomically, or an exception is thrown. * atomically, or an exception is thrown.
*
* @deprecated Use {@link #rename} and {@link #syncMetaData} instead.
*/ */
public abstract void renameFile(String source, String dest) throws IOException; @Deprecated
public final void renameFile(String source, String dest) throws IOException {
rename(source, dest);
syncMetaData();
}
/**
* Renames {@code source} to {@code dest} as an atomic operation,
* where {@code dest} does not yet exist in the directory.
* <p>
* Notes: This method is used by IndexWriter to publish commits.
* It is ok if this operation is not truly atomic, for example
* both {@code source} and {@code dest} can be visible temporarily.
* It is just important that the contents of {@code dest} appear
* atomically, or an exception is thrown.
*/
public abstract void rename(String source, String dest) throws IOException;
/**
* Ensure that directory metadata, such as recent file renames, are made
* durable.
*/
public abstract void syncMetaData() throws IOException;
/** Returns a stream reading an existing file. /** Returns a stream reading an existing file.
* <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException} * <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException}

View File

@ -288,15 +288,20 @@ public abstract class FSDirectory extends BaseDirectory {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
ensureOpen(); ensureOpen();
if (pendingDeletes.contains(source)) { if (pendingDeletes.contains(source)) {
throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved"); throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
} }
pendingDeletes.remove(dest); pendingDeletes.remove(dest);
Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE); Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
// TODO: should we move directory fsync to a separate 'syncMetadata' method? maybeDeletePendingFiles();
// for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns }
@Override
public void syncMetaData() throws IOException {
// TODO: to improve listCommits(), IndexFileDeleter could call this after deleting segments_Ns
ensureOpen();
IOUtils.fsync(directory, true); IOUtils.fsync(directory, true);
maybeDeletePendingFiles(); maybeDeletePendingFiles();
} }

View File

@ -182,14 +182,20 @@ public class FileSwitchDirectory extends Directory {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
Directory sourceDir = getDirectory(source); Directory sourceDir = getDirectory(source);
// won't happen with standard lucene index files since pending and commit will // won't happen with standard lucene index files since pending and commit will
// always have the same extension ("") // always have the same extension ("")
if (sourceDir != getDirectory(dest)) { if (sourceDir != getDirectory(dest)) {
throw new AtomicMoveNotSupportedException(source, dest, "source and dest are in different directories"); throw new AtomicMoveNotSupportedException(source, dest, "source and dest are in different directories");
} }
sourceDir.renameFile(source, dest); sourceDir.rename(source, dest);
}
@Override
public void syncMetaData() throws IOException {
primaryDir.syncMetaData();
secondaryDir.syncMetaData();
} }
@Override @Override

View File

@ -84,8 +84,13 @@ public abstract class FilterDirectory extends Directory {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
in.renameFile(source, dest); in.rename(source, dest);
}
@Override
public void syncMetaData() throws IOException {
in.syncMetaData();
} }
@Override @Override

View File

@ -51,9 +51,15 @@ public final class LockValidatingDirectoryWrapper extends FilterDirectory {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
writeLock.ensureValid(); writeLock.ensureValid();
in.renameFile(source, dest); in.rename(source, dest);
}
@Override
public void syncMetaData() throws IOException {
writeLock.ensureValid();
in.syncMetaData();
} }
@Override @Override

View File

@ -170,12 +170,12 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
unCache(source); unCache(source);
if (cache.fileNameExists(dest)) { if (cache.fileNameExists(dest)) {
throw new IllegalArgumentException("target file " + dest + " already exists"); throw new IllegalArgumentException("target file " + dest + " already exists");
} }
in.renameFile(source, dest); in.rename(source, dest);
} }
@Override @Override

View File

@ -214,7 +214,7 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
ensureOpen(); ensureOpen();
RAMFile file = fileMap.get(source); RAMFile file = fileMap.get(source);
if (file == null) { if (file == null) {
@ -229,6 +229,11 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
fileMap.remove(source); fileMap.remove(source);
} }
@Override
public void syncMetaData() throws IOException {
// we are by definition not durable!
}
/** Returns a stream reading an existing file. */ /** Returns a stream reading an existing file. */
@Override @Override
public IndexInput openInput(String name, IOContext context) throws IOException { public IndexInput openInput(String name, IOContext context) throws IOException {

View File

@ -60,8 +60,8 @@ public final class TrackingDirectoryWrapper extends FilterDirectory {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
in.renameFile(source, dest); in.rename(source, dest);
synchronized (createdFileNames) { synchronized (createdFileNames) {
createdFileNames.add(dest); createdFileNames.add(dest);
createdFileNames.remove(source); createdFileNames.remove(source);

View File

@ -39,6 +39,7 @@ public class TestFilterDirectory extends BaseDirectoryTestCase {
Set<Method> exclude = new HashSet<>(); Set<Method> exclude = new HashSet<>();
exclude.add(Directory.class.getMethod("copyFrom", Directory.class, String.class, String.class, IOContext.class)); exclude.add(Directory.class.getMethod("copyFrom", Directory.class, String.class, String.class, IOContext.class));
exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class)); exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class));
exclude.add(Directory.class.getMethod("renameFile", String.class, String.class));
for (Method m : FilterDirectory.class.getMethods()) { for (Method m : FilterDirectory.class.getMethods()) {
if (m.getDeclaringClass() == Directory.class) { if (m.getDeclaringClass() == Directory.class) {
assertTrue("method " + m.getName() + " not overridden!", exclude.contains(m)); assertTrue("method " + m.getName() + " not overridden!", exclude.contains(m));

View File

@ -51,7 +51,7 @@ public class TestTrackingDirectoryWrapper extends BaseDirectoryTestCase {
TrackingDirectoryWrapper dir = new TrackingDirectoryWrapper(new RAMDirectory()); TrackingDirectoryWrapper dir = new TrackingDirectoryWrapper(new RAMDirectory());
dir.createOutput("foo", newIOContext(random())).close(); dir.createOutput("foo", newIOContext(random())).close();
assertEquals(asSet("foo"), dir.getCreatedFiles()); assertEquals(asSet("foo"), dir.getCreatedFiles());
dir.renameFile("foo", "bar"); dir.rename("foo", "bar");
assertEquals(asSet("bar"), dir.getCreatedFiles()); assertEquals(asSet("bar"), dir.getCreatedFiles());
} }

View File

@ -141,10 +141,12 @@ public class IndexAndTaxonomyReplicationHandler implements ReplicationHandler {
indexDir.sync(Collections.singletonList(indexPendingFile)); indexDir.sync(Collections.singletonList(indexPendingFile));
if (taxoSegmentsFile != null) { if (taxoSegmentsFile != null) {
taxoDir.renameFile(taxoPendingFile, taxoSegmentsFile); taxoDir.rename(taxoPendingFile, taxoSegmentsFile);
taxoDir.syncMetaData();
} }
indexDir.renameFile(indexPendingFile, indexSegmentsFile); indexDir.rename(indexPendingFile, indexSegmentsFile);
indexDir.syncMetaData();
success = true; success = true;
} finally { } finally {

View File

@ -233,7 +233,8 @@ public class IndexReplicationHandler implements ReplicationHandler {
// now copy and fsync segmentsFile as pending, then rename (simulating lucene commit) // now copy and fsync segmentsFile as pending, then rename (simulating lucene commit)
indexDir.copyFrom(clientDir, segmentsFile, pendingSegmentsFile, IOContext.READONCE); indexDir.copyFrom(clientDir, segmentsFile, pendingSegmentsFile, IOContext.READONCE);
indexDir.sync(Collections.singletonList(pendingSegmentsFile)); indexDir.sync(Collections.singletonList(pendingSegmentsFile));
indexDir.renameFile(pendingSegmentsFile, segmentsFile); indexDir.rename(pendingSegmentsFile, segmentsFile);
indexDir.syncMetaData();
success = true; success = true;
} finally { } finally {

View File

@ -145,7 +145,7 @@ class SimpleCopyJob extends CopyJob {
// NOTE: if this throws exception, then some files have been moved to their true names, and others are leftover .tmp files. I don't // NOTE: if this throws exception, then some files have been moved to their true names, and others are leftover .tmp files. I don't
// think heroic exception handling is necessary (no harm will come, except some leftover files), nor warranted here (would make the // think heroic exception handling is necessary (no harm will come, except some leftover files), nor warranted here (would make the
// code more complex, for the exceptional cases when something is wrong w/ your IO system): // code more complex, for the exceptional cases when something is wrong w/ your IO system):
dest.dir.renameFile(tmpFileName, fileName); dest.dir.rename(tmpFileName, fileName);
} }
copiedFiles.clear(); copiedFiles.clear();

View File

@ -272,7 +272,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT); si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT); Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
expectThrows(UnsupportedOperationException.class, () -> { expectThrows(UnsupportedOperationException.class, () -> {
cfs.renameFile(testfile, "bogus"); cfs.rename(testfile, "bogus");
}); });
cfs.close(); cfs.close();

View File

@ -110,7 +110,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
output.writeBytes(bytes, bytes.length); output.writeBytes(bytes, bytes.length);
output.close(); output.close();
dir.renameFile("foobar", "foobaz"); dir.rename("foobar", "foobaz");
IndexInput input = dir.openInput("foobaz", newIOContext(random())); IndexInput input = dir.openInput("foobaz", newIOContext(random()));
byte bytes2[] = new byte[numBytes]; byte bytes2[] = new byte[numBytes];
@ -1196,7 +1196,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
// Make sure rename fails: // Make sure rename fails:
expectThrows(NoSuchFileException.class, () -> { expectThrows(NoSuchFileException.class, () -> {
fsDir.renameFile(fileName, "file2"); fsDir.rename(fileName, "file2");
}); });
// Make sure delete fails: // Make sure delete fails:

View File

@ -208,7 +208,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
} }
@Override @Override
public synchronized void renameFile(String source, String dest) throws IOException { public synchronized void rename(String source, String dest) throws IOException {
maybeYield(); maybeYield();
maybeThrowDeterministicException(); maybeThrowDeterministicException();
@ -226,7 +226,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
boolean success = false; boolean success = false;
try { try {
in.renameFile(source, dest); in.rename(source, dest);
success = true; success = true;
} finally { } finally {
if (success) { if (success) {
@ -242,6 +242,16 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
} }
} }
@Override
public synchronized void syncMetaData() throws IOException {
maybeYield();
maybeThrowDeterministicException();
if (crashed) {
throw new IOException("cannot rename after crash");
}
in.syncMetaData();
}
public synchronized final long sizeInBytes() throws IOException { public synchronized final long sizeInBytes() throws IOException {
if (in instanceof RAMDirectory) if (in instanceof RAMDirectory)
return ((RAMDirectory) in).ramBytesUsed(); return ((RAMDirectory) in).ramBytesUsed();
@ -303,6 +313,10 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
} }
private synchronized void _corruptFiles(Collection<String> files) throws IOException { private synchronized void _corruptFiles(Collection<String> files) throws IOException {
// TODO: we should also mess with any recent file renames, file deletions, if
// syncMetaData was not called!!
// Must make a copy because we change the incoming unsyncedFiles // Must make a copy because we change the incoming unsyncedFiles
// when we create temp files, delete, etc., below: // when we create temp files, delete, etc., below:
final List<String> filesToCorrupt = new ArrayList<>(files); final List<String> filesToCorrupt = new ArrayList<>(files);

View File

@ -148,12 +148,17 @@ public class HdfsDirectory extends BaseDirectory {
} }
@Override @Override
public void renameFile(String source, String dest) throws IOException { public void rename(String source, String dest) throws IOException {
Path sourcePath = new Path(hdfsDirPath, source); Path sourcePath = new Path(hdfsDirPath, source);
Path destPath = new Path(hdfsDirPath, dest); Path destPath = new Path(hdfsDirPath, dest);
fileContext.rename(sourcePath, destPath); fileContext.rename(sourcePath, destPath);
} }
@Override
public void syncMetaData() throws IOException {
// TODO: how?
}
@Override @Override
public long fileLength(String name) throws IOException { public long fileLength(String name) throws IOException {
FileStatus fileStatus = fileSystem.getFileStatus(new Path(hdfsDirPath, name)); FileStatus fileStatus = fileSystem.getFileStatus(new Path(hdfsDirPath, name));

View File

@ -128,7 +128,7 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
IndexOutput output = directory.createOutput("testing.test", new IOContext()); IndexOutput output = directory.createOutput("testing.test", new IOContext());
output.writeInt(12345); output.writeInt(12345);
output.close(); output.close();
directory.renameFile("testing.test", "testing.test.renamed"); directory.rename("testing.test", "testing.test.renamed");
assertFalse(slowFileExists(directory, "testing.test")); assertFalse(slowFileExists(directory, "testing.test"));
assertTrue(slowFileExists(directory, "testing.test.renamed")); assertTrue(slowFileExists(directory, "testing.test.renamed"));
IndexInput input = directory.openInput("testing.test.renamed", new IOContext()); IndexInput input = directory.openInput("testing.test.renamed", new IOContext());