LUCENE-1468: switch Directory.list() to Directory.listAll(), which does no filtering of returned array

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@723789 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2008-12-05 17:03:13 +00:00
parent 5e3cd6711a
commit 4a0a5664dd
22 changed files with 290 additions and 81 deletions

View File

@ -38,6 +38,14 @@ API Changes
These methods can be used to avoid additional calls to doc().
(Michael Busch)
6. LUCENE-1468: Deprecate Directory.list(), which sometimes (in
FSDirectory) filters out files that don't look like index files, in
favor of new Directory.listAll(), which does no filtering. Also,
listAll() will never return null; instead, it throws an IOException
(or subclass). Specifically, FSDirectory.listAll() will throw the
newly added NoSuchDirectoryException if the directory does not
exist. (Marcel Reutegger, Mike McCandless)
Bug fixes
1. LUCENE-1415: MultiPhraseQuery has incorrect hashCode() and equals()

View File

@ -24,7 +24,6 @@ import java.io.BufferedReader;
import java.util.List;
import java.util.Iterator;
import org.apache.lucene.benchmark.byTask.Benchmark;
import org.apache.lucene.benchmark.byTask.feeds.DocData;
import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException;
import org.apache.lucene.benchmark.byTask.feeds.ReutersDocMaker;
@ -712,7 +711,7 @@ public class TestPerfTasksLogic extends TestCase {
ir.close();
// Make sure we have 3 segments:
final String[] files = benchmark.getRunData().getDirectory().list();
final String[] files = benchmark.getRunData().getDirectory().listAll();
int cfsCount = 0;
for(int i=0;i<files.length;i++)
if (files[i].endsWith(".cfs"))

View File

@ -480,9 +480,7 @@ abstract class DirectoryIndexReader extends IndexReader {
/** @see IndexReader#listCommits */
public static Collection listCommits(Directory dir) throws IOException {
final String[] files = dir.list();
if (files == null)
throw new IOException("cannot read directory " + dir + ": list() returned null");
final String[] files = dir.listAll();
Collection commits = new ArrayList();

View File

@ -146,9 +146,7 @@ final class IndexFileDeleter {
long currentGen = segmentInfos.getGeneration();
IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
String[] files = directory.list();
if (files == null)
throw new IOException("cannot read directory " + directory + ": list() returned null");
String[] files = directory.listAll();
CommitPoint currentCommitPoint = null;
@ -306,9 +304,7 @@ final class IndexFileDeleter {
* that segment.
*/
public void refresh(String segmentName) throws IOException {
String[] files = directory.list();
if (files == null)
throw new IOException("cannot read directory " + directory + ": list() returned null");
String[] files = directory.listAll();
IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
String segmentPrefix1;
String segmentPrefix2;

View File

@ -620,12 +620,11 @@ final class SegmentInfo {
else
prefix = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION;
int prefixLength = prefix.length();
String[] allFiles = dir.list();
if (allFiles == null)
throw new IOException("cannot read directory " + dir + ": list() returned null");
String[] allFiles = dir.listAll();
final IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
for(int i=0;i<allFiles.length;i++) {
String fileName = allFiles[i];
if (fileName.length() > prefixLength && Character.isDigit(fileName.charAt(prefixLength)) && fileName.startsWith(prefix)) {
if (filter.accept(null, fileName) && fileName.length() > prefixLength && Character.isDigit(fileName.charAt(prefixLength)) && fileName.startsWith(prefix)) {
files.add(fileName);
}
}

View File

@ -17,11 +17,13 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.ChecksumIndexOutput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.NoSuchDirectoryException;
import java.io.File;
import java.io.FileNotFoundException;
@ -129,8 +131,11 @@ final class SegmentInfos extends Vector {
* @param directory -- directory to search for the latest segments_N file
*/
public static long getCurrentSegmentGeneration(Directory directory) throws IOException {
String[] files = directory.list();
return getCurrentSegmentGeneration(files);
try {
return getCurrentSegmentGeneration(directory.listAll());
} catch (NoSuchDirectoryException nsde) {
return -1;
}
}
/**
@ -558,9 +563,9 @@ final class SegmentInfos extends Vector {
long genA = -1;
if (directory != null)
files = directory.list();
files = directory.listAll();
else
files = fileDirectory.list();
files = FSDirectory.listAll(fileDirectory);
if (files != null)
genA = getCurrentSegmentGeneration(files);

View File

@ -19,6 +19,8 @@ package org.apache.lucene.store;
import java.io.IOException;
import org.apache.lucene.index.IndexFileNameFilter;
/** A Directory is a flat list of files. Files may be written once, when they
* are created. Once a file is created it may only be opened for read, or
* deleted. Random access is permitted both when reading and writing.
@ -43,14 +45,29 @@ public abstract class Directory {
* this Directory instance). */
protected LockFactory lockFactory;
/** Returns an array of strings, one for each file in the
* directory. This method may return null (for example for
* {@link FSDirectory} if the underlying directory doesn't
* exist in the filesystem or there are permissions
* problems).*/
/** @deprecated For some Directory implementations ({@link
* FSDirectory}, and its subclasses), this method
* silently filters its results to include only index
* files. Please use {@link #listAll} instead, which
* does no filtering. */
public abstract String[] list()
throws IOException;
/** Returns an array of strings, one for each file in the
* directory. Unlike {@link #list} this method does no
* filtering of the contents in a directory, and it will
* never return null (throws IOException instead).
*
* Currently this method simply fallsback to {@link
* #list} for Directory impls outside of Lucene's core &
* contrib, but in 3.0 that method will be removed and
* this method will become abstract. */
public String[] listAll()
throws IOException
{
return list();
}
/** Returns true iff a file with the given name exists. */
public abstract boolean fileExists(String name)
throws IOException;
@ -173,48 +190,55 @@ public abstract class Directory {
* are undefined and you could easily hit a
* FileNotFoundException.
*
* <p><b>NOTE:</b> this method only copies files that look
* like index files (ie, have extensions matching the
* known extensions of index files).
*
* @param src source directory
* @param dest destination directory
* @param closeDirSrc if <code>true</code>, call {@link #close()} method on source directory
* @throws IOException
*/
public static void copy(Directory src, Directory dest, boolean closeDirSrc) throws IOException {
final String[] files = src.list();
final String[] files = src.listAll();
if (files == null)
throw new IOException("cannot read directory " + src + ": list() returned null");
IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.length; i++) {
IndexOutput os = null;
IndexInput is = null;
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.length; i++) {
if (!filter.accept(null, files[i]))
continue;
IndexOutput os = null;
IndexInput is = null;
try {
// create file in dest directory
os = dest.createOutput(files[i]);
// read current file
is = src.openInput(files[i]);
// and copy to dest directory
long len = is.length();
long readCount = 0;
while (readCount < len) {
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int)(len - readCount) : BufferedIndexOutput.BUFFER_SIZE;
is.readBytes(buf, 0, toRead);
os.writeBytes(buf, toRead);
readCount += toRead;
}
} finally {
// graceful cleanup
try {
// create file in dest directory
os = dest.createOutput(files[i]);
// read current file
is = src.openInput(files[i]);
// and copy to dest directory
long len = is.length();
long readCount = 0;
while (readCount < len) {
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int)(len - readCount) : BufferedIndexOutput.BUFFER_SIZE;
is.readBytes(buf, 0, toRead);
os.writeBytes(buf, toRead);
readCount += toRead;
}
if (os != null)
os.close();
} finally {
// graceful cleanup
try {
if (os != null)
os.close();
} finally {
if (is != null)
is.close();
}
if (is != null)
is.close();
}
}
if(closeDirSrc)
src.close();
}
if(closeDirSrc)
src.close();
}
/**

View File

@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.File;
import java.io.FilenameFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
@ -242,6 +243,7 @@ public class FSDirectory extends Directory {
return dir;
}
/** @deprecated */
private void create() throws IOException {
if (directory.exists()) {
String[] files = directory.list(IndexFileNameFilter.getFilter()); // clear old files
@ -265,9 +267,6 @@ public class FSDirectory extends Directory {
final void createDir() throws IOException {
if (!checked) {
if (directory.exists() && !directory.isDirectory())
throw new IOException(directory + " not a directory");
if (!directory.exists())
if (!directory.mkdirs())
throw new IOException("Cannot create directory: " + directory);
@ -304,6 +303,9 @@ public class FSDirectory extends Directory {
directory = path;
if (directory.exists() && !directory.isDirectory())
throw new NoSuchDirectoryException("file '" + directory + "' exists but is not a directory");
boolean doClearLockID = false;
if (lockFactory == null) {
@ -356,12 +358,46 @@ public class FSDirectory extends Directory {
}
}
/** Returns an array of strings, one for each Lucene index file in the directory. */
/** Lists all files (not subdirectories) in the
* directory. This method never returns null (throws
* {@link IOException} instead).
*
* @throws NoSuchDirectoryException if the directory
* does not exist, or does exist but is not a
* directory.
* @throws IOException if list() returns null */
public static String[] listAll(File dir) throws IOException {
if (!dir.exists())
throw new NoSuchDirectoryException("directory '" + dir + "' does not exist");
else if (!dir.isDirectory())
throw new NoSuchDirectoryException("file '" + dir + "' exists but is not a directory");
// Exclude subdirs
String[] result = dir.list(new FilenameFilter() {
public boolean accept(File dir, String file) {
return !new File(dir, file).isDirectory();
}
});
if (result == null)
throw new IOException("directory '" + dir + "' exists and is a directory, but cannot be listed: list() returned null");
return result;
}
public String[] list() {
ensureOpen();
return directory.list(IndexFileNameFilter.getFilter());
}
/** Lists all files (not subdirectories) in the
* directory.
* @see #listAll(File) */
public String[] listAll() throws IOException {
ensureOpen();
return listAll(directory);
}
/** Returns true iff a file with the given name exists. */
public boolean fileExists(String name) {
ensureOpen();

View File

@ -0,0 +1,31 @@
package org.apache.lucene.store;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileNotFoundException;
/**
* This exception is thrown when you try to list a
* non-existent directory.
*/
public class NoSuchDirectoryException extends FileNotFoundException {
public NoSuchDirectoryException(String message) {
super(message);
}
}

View File

@ -95,8 +95,11 @@ public class RAMDirectory extends Directory implements Serializable {
this(FSDirectory.getDirectory(dir), true);
}
/** Returns an array of strings, one for each file in the directory. */
public synchronized final String[] list() {
return listAll();
}
public synchronized final String[] listAll() {
ensureOpen();
Set fileNames = fileMap.keySet();
String[] result = new String[fileNames.size()];

View File

@ -447,7 +447,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase
"segments_3",
"segments.gen"};
String[] actual = dir.list();
String[] actual = dir.listAll();
Arrays.sort(expected);
Arrays.sort(actual);
if (!Arrays.equals(expected, actual)) {

View File

@ -333,10 +333,10 @@ public class TestDeletionPolicy extends LuceneTestCase
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.list().length;
int preCount = dir.listAll().length;
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.LIMITED);
writer.close();
int postCount = dir.list().length;
int postCount = dir.listAll().length;
assertTrue(postCount < preCount);
}
}

View File

@ -315,6 +315,9 @@ public class TestFieldsReader extends LuceneTestCase {
public String[] list() throws IOException {
return fsDir.list();
}
public String[] listAll() throws IOException {
return fsDir.listAll();
}
public boolean fileExists(String name) throws IOException {
return fsDir.fileExists(name);
}

View File

@ -65,7 +65,7 @@ public class TestIndexFileDeleter extends LuceneTestCase
// Now, artificially create an extra .del file & extra
// .s0 file:
String[] files = dir.list();
String[] files = dir.listAll();
/*
for(int j=0;j<files.length;j++) {
@ -142,14 +142,14 @@ public class TestIndexFileDeleter extends LuceneTestCase
// Create a bogus cfs file shadowing a non-cfs segment:
copyFile(dir, "_2.cfs", "_3.cfs");
String[] filesPre = dir.list();
String[] filesPre = dir.listAll();
// Open & close a writer: it should delete the above 4
// files and nothing more:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
writer.close();
String[] files2 = dir.list();
String[] files2 = dir.listAll();
dir.close();
Arrays.sort(files);

View File

@ -48,6 +48,7 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@ -962,11 +963,11 @@ public class TestIndexReader extends LuceneTestCase
// new IndexFileDeleter, have it delete
// unreferenced files, then verify that in fact
// no files were deleted:
String[] startFiles = dir.list();
String[] startFiles = dir.listAll();
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
String[] endFiles = dir.list();
String[] endFiles = dir.listAll();
Arrays.sort(startFiles);
Arrays.sort(endFiles);
@ -1520,6 +1521,7 @@ public class TestIndexReader extends LuceneTestCase
}
}
// LUCENE-1474
public void testIndexReader() throws Exception {
Directory dir = new RAMDirectory();
@ -1542,4 +1544,18 @@ public class TestIndexReader extends LuceneTestCase
doc.add(new Field("id", id, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
return doc;
}
// LUCENE-1468 -- make sure on attempting to open an
// IndexReader on a non-existent directory, you get a
// good exception
public void testNoDir() throws Throwable {
String tempDir = System.getProperty("java.io.tmpdir");
Directory dir = FSDirectory.getDirectory(new File(tempDir, "doesnotexist"), null);
try {
IndexReader.open(dir);
fail("did not hit expected exception");
} catch (NoSuchDirectoryException nsde) {
// expected
}
}
}

View File

@ -1051,7 +1051,7 @@ public class TestIndexReaderReopen extends LuceneTestCase {
dir1.close();
try {
dir1.list();
dir1.listAll();
fail("did not hit AlreadyClosedException");
} catch (AlreadyClosedException ace) {
// expected

View File

@ -170,7 +170,7 @@ public class TestIndexWriter extends LuceneTestCase
addDocWithIndex(writer, 25*i+j);
}
writer.close();
String[] files = dirs[i].list();
String[] files = dirs[i].listAll();
for(int j=0;j<files.length;j++) {
inputDiskUsage += dirs[i].fileLength(files[j]);
}
@ -207,11 +207,11 @@ public class TestIndexWriter extends LuceneTestCase
// succeed and index should show all documents were
// added.
// String[] files = startDir.list();
// String[] files = startDir.listAll();
long diskUsage = startDir.sizeInBytes();
long startDiskUsage = 0;
String[] files = startDir.list();
String[] files = startDir.listAll();
for(int i=0;i<files.length;i++) {
startDiskUsage += startDir.fileLength(files[i]);
}
@ -539,11 +539,11 @@ public class TestIndexWriter extends LuceneTestCase
}
public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
String[] startFiles = dir.list();
String[] startFiles = dir.listAll();
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
String[] endFiles = dir.list();
String[] endFiles = dir.listAll();
Arrays.sort(startFiles);
Arrays.sort(endFiles);
@ -708,7 +708,7 @@ public class TestIndexWriter extends LuceneTestCase
writer.close();
long startDiskUsage = 0;
String[] files = dir.list();
String[] files = dir.listAll();
for(int i=0;i<files.length;i++) {
startDiskUsage += dir.fileLength(files[i]);
}
@ -988,7 +988,7 @@ public class TestIndexWriter extends LuceneTestCase
long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
assertTrue("segment generation should be > 1 but got " + gen, gen > 1);
String[] files = dir.list();
String[] files = dir.listAll();
for(int i=0;i<files.length;i++) {
if (files[i].endsWith(".cfs")) {
dir.deleteFile(files[i]);
@ -1276,12 +1276,12 @@ public class TestIndexWriter extends LuceneTestCase
RAMDirectory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.setRAMBufferSizeMB(0.000001);
int lastNumFile = dir.list().length;
int lastNumFile = dir.listAll().length;
for(int j=0;j<9;j++) {
Document doc = new Document();
doc.add(new Field("field", "aaa" + j, Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
int numFile = dir.list().length;
int numFile = dir.listAll().length;
// Verify that with a tiny RAM buffer we see new
// segment after every doc
assertTrue(numFile > lastNumFile);
@ -4254,4 +4254,31 @@ public class TestIndexWriter extends LuceneTestCase
r.close();
dir.close();
}
// LUCENE-1468 -- make sure opening an IndexWriter with
// create=true does not remove non-index files
public void testOtherFiles() throws Throwable {
File indexDir = new File(System.getProperty("tempDir"), "otherfiles");
Directory dir = new FSDirectory(indexDir, null);
try {
// Create my own random file:
IndexOutput out = dir.createOutput("myrandomfile");
out.writeByte((byte) 42);
out.close();
new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED).close();
assertTrue(dir.fileExists("myrandomfile"));
// Make sure this does not copy myrandomfile:
Directory dir2 = new RAMDirectory(dir);
assertTrue(!dir2.fileExists("myrandomfile"));
} finally {
dir.close();
_TestUtil.rmDir(indexDir);
}
}
}

View File

@ -668,11 +668,11 @@ public class TestIndexWriterDelete extends LuceneTestCase {
}
}
String[] startFiles = dir.list();
String[] startFiles = dir.listAll();
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
String[] endFiles = dir.list();
String[] endFiles = dir.listAll();
if (!Arrays.equals(startFiles, endFiles)) {
fail("docswriter abort() failed to delete unreferenced files:\n before delete:\n "

View File

@ -239,7 +239,7 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
assertTrue(numSegments < mergeFactor);
}
String[] files = writer.getDirectory().list();
String[] files = writer.getDirectory().listAll();
int segmentCfsCount = 0;
for (int i = 0; i < files.length; i++) {
if (files[i].endsWith(".cfs")) {
@ -249,6 +249,7 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
assertEquals(segmentCount, segmentCfsCount);
}
/*
private void printSegmentDocCounts(IndexWriter writer) {
int segmentCount = writer.getSegmentCount();
System.out.println("" + segmentCount + " segments total");
@ -257,4 +258,5 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
+ " docs");
}
}
*/
}

View File

@ -197,7 +197,7 @@ public class TestOmitTf extends LuceneTestCase {
}
private void assertNoPrx(Directory dir) throws Throwable {
final String[] files = dir.list();
final String[] files = dir.listAll();
for(int i=0;i<files.length;i++)
assertFalse(files[i].endsWith(".prx"));
}

View File

@ -271,6 +271,11 @@ public class TestBufferedIndexInput extends LuceneTestCase {
{
return dir.list();
}
public String[] listAll()
throws IOException
{
return dir.listAll();
}
public long fileLength(String name) throws IOException {
return dir.fileLength(name);

View File

@ -21,6 +21,8 @@ import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
public class TestDirectory extends LuceneTestCase {
@ -128,5 +130,60 @@ public class TestDirectory extends LuceneTestCase {
_TestUtil.rmDir(path);
}
}
// LUCENE-1468
public void testRAMDirectoryFilter() throws IOException {
checkDirectoryFilter(new RAMDirectory());
}
// LUCENE-1468
public void testFSDirectoryFilter() throws IOException {
checkDirectoryFilter(FSDirectory.getDirectory("test"));
}
// LUCENE-1468
private void checkDirectoryFilter(Directory dir) throws IOException {
String name = "file";
try {
dir.createOutput(name).close();
assertTrue(dir.fileExists(name));
assertTrue(Arrays.asList(dir.listAll()).contains(name));
} finally {
dir.close();
}
}
// LUCENE-1468
public void testCopySubdir() throws Throwable {
File path = new File(System.getProperty("tempDir"), "testsubdir");
try {
path.mkdirs();
new File(path, "subdir").mkdirs();
Directory fsDir = new FSDirectory(path, null);
assertEquals(0, new RAMDirectory(fsDir).listAll().length);
} finally {
_TestUtil.rmDir(path);
}
}
// LUCENE-1468
public void testNotDirectory() throws Throwable {
File path = new File(System.getProperty("tempDir"), "testnotdir");
Directory fsDir = new FSDirectory(path, null);
try {
IndexOutput out = fsDir.createOutput("afile");
out.close();
assertTrue(fsDir.fileExists("afile"));
try {
new FSDirectory(new File(path, "afile"), null);
fail("did not hit expected exception");
} catch (NoSuchDirectoryException nsde) {
// Expected
}
} finally {
fsDir.close();
_TestUtil.rmDir(path);
}
}
}