LUCENE-6424: DirectoryStream doesn't wrap with FilterPath

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1673720 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2015-04-15 10:36:34 +00:00
parent 63073a01c2
commit d71933415d
9 changed files with 190 additions and 340 deletions

View File

@ -94,6 +94,9 @@ Bug Fixes
* LUCENE-6409: Fixed integer overflow in LongBitSet.ensureCapacity.
(Luc Vanlerberghe via Adrien Grand)
* LUCENE-6424: Fix many bugs with mockfs filesystems in the test-framework:
always consistently wrap Path, fix buggy behavior for globs, etc. (Ryan Ernst)
API Changes
* LUCENE-6377: SearcherFactory#newSearcher now accepts the previous reader

View File

@ -25,6 +25,7 @@ import java.net.URI;
import java.nio.channels.AsynchronousFileChannel;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
@ -260,4 +261,84 @@ public class TestMockFilesystems extends LuceneTestCase {
IOUtils.close(toClose);
}
public void testDirectoryStreamFiltered() throws IOException {
Path dir = FilterPath.unwrap(createTempDir());
FileSystem fs = new FilterFileSystemProvider("test://", dir.getFileSystem()).getFileSystem(URI.create("file:///"));
Path wrapped = new FilterPath(dir, fs);
OutputStream file = Files.newOutputStream(wrapped.resolve("file1"));
file.write(5);
file.close();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(wrapped)) {
int count = 0;
for (Path path : stream) {
assertTrue(path instanceof FilterPath);
if (!path.getFileName().toString().startsWith("extra")) {
count++;
}
}
assertEquals(1, count);
}
// check with LeakFS, a subclass of HandleTrackingFS which mucks with newDirectoryStream
dir = FilterPath.unwrap(createTempDir());
fs = new LeakFS(dir.getFileSystem()).getFileSystem(URI.create("file:///"));
wrapped = new FilterPath(dir, fs);
file = Files.newOutputStream(wrapped.resolve("file1"));
file.write(5);
file.close();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(wrapped)) {
int count = 0;
for (Path path : stream) {
assertTrue(path instanceof FilterPath);
if (!path.getFileName().toString().startsWith("extra")) {
count++;
}
}
assertEquals(1, count);
}
}
public void testDirectoryStreamGlobFiltered() throws IOException {
Path dir = FilterPath.unwrap(createTempDir());
FileSystem fs = new FilterFileSystemProvider("test://", dir.getFileSystem()).getFileSystem(URI.create("file:///"));
Path wrapped = new FilterPath(dir, fs);
OutputStream file = Files.newOutputStream(wrapped.resolve("foo"));
file.write(5);
file.close();
file = Files.newOutputStream(wrapped.resolve("bar"));
file.write(5);
file.close();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(wrapped, "f*")) {
int count = 0;
for (Path path : stream) {
assertTrue(path instanceof FilterPath);
++count;
}
assertEquals(1, count);
}
// check with LeakFS, a subclass of HandleTrackingFS which mucks with newDirectoryStream
dir = FilterPath.unwrap(createTempDir());
fs = new LeakFS(dir.getFileSystem()).getFileSystem(URI.create("file:///"));
wrapped = new FilterPath(dir, fs);
file = Files.newOutputStream(wrapped.resolve("foo"));
file.write(5);
file.close();
file = Files.newOutputStream(wrapped.resolve("bar"));
file.write(5);
file.close();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(wrapped, "f*")) {
int count = 0;
for (Path path : stream) {
assertTrue(path instanceof FilterPath);
++count;
}
assertEquals(1, count);
}
}
}

View File

@ -18,15 +18,12 @@ package org.apache.lucene.util;
*/
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessMode;
import java.nio.file.FileStore;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
@ -165,7 +162,6 @@ public class TestIOUtils extends LuceneTestCase {
* it will chroot /dev and /sys requests to root, so you can mock those too.
* <p>
* It is hacky by definition, so don't try putting it around a complex chain or anything.
* Use FilterPath.unwrap
*/
static class MockLinuxFileSystemProvider extends FilterFileSystemProvider {
final Map<String,FileStore> filesToStore;
@ -182,11 +178,11 @@ public class TestIOUtils extends LuceneTestCase {
@Override
public Path getPath(String first, String... more) {
return new MockLinuxPath(super.getPath(first, more), this);
return new MockLinuxPath(delegateInstance.getPath(first, more), this);
}
};
this.filesToStore = filesToStore;
this.root = root;
this.root = new MockLinuxPath(root, this.fileSystem);
}
@Override
@ -213,14 +209,8 @@ public class TestIOUtils extends LuceneTestCase {
}
@Override
public void checkAccess(Path path, AccessMode... modes) throws IOException {
// TODO: kinda screwed up how we do this, but it's easy to get lost. just unravel completely.
delegate.checkAccess(maybeChroot(FilterPath.unwrap(path)), modes);
}
@Override
public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
return super.newInputStream(maybeChroot(path), options);
protected Path toDelegate(Path path) {
return super.toDelegate(maybeChroot(path));
}
class MockLinuxPath extends FilterPath {

View File

@ -19,6 +19,8 @@ package org.apache.lucene.mockfile;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystem;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Objects;
@ -28,13 +30,18 @@ import java.util.Objects;
* source of data, possibly transforming the data along the
* way or providing additional functionality.
*/
public class FilterDirectoryStream<T> implements DirectoryStream<T> {
public class FilterDirectoryStream implements DirectoryStream<Path> {
/**
* The underlying {@code DirectoryStream} instance.
*/
protected final DirectoryStream<T> delegate;
protected final DirectoryStream<Path> delegate;
/**
* The underlying {@code FileSystem} instance.
*/
protected final FileSystem fileSystem;
/**
* Construct a {@code FilterDirectoryStream} based on
* the specified base stream.
@ -42,8 +49,9 @@ public class FilterDirectoryStream<T> implements DirectoryStream<T> {
* Note that base stream is closed if this stream is closed.
* @param delegate specified base stream.
*/
public FilterDirectoryStream(DirectoryStream<T> delegate) {
public FilterDirectoryStream(DirectoryStream<Path> delegate, FileSystem fileSystem) {
this.delegate = Objects.requireNonNull(delegate);
this.fileSystem = Objects.requireNonNull(fileSystem);
}
@Override
@ -52,7 +60,21 @@ public class FilterDirectoryStream<T> implements DirectoryStream<T> {
}
@Override
public Iterator<T> iterator() {
return delegate.iterator();
public Iterator<Path> iterator() {
final Iterator<Path> delegateIterator = delegate.iterator();
return new Iterator<Path>() {
@Override
public boolean hasNext() {
return delegateIterator.hasNext();
}
@Override
public Path next() {
return new FilterPath(delegateIterator.next(), fileSystem);
}
@Override
public void remove() {
delegateIterator.remove();
}
};
}
}

View File

@ -35,6 +35,7 @@ import java.nio.file.FileSystem;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.ProviderMismatchException;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.FileAttributeView;
@ -215,8 +216,14 @@ public class FilterFileSystemProvider extends FileSystemProvider {
}
@Override
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException {
return delegate.newDirectoryStream(toDelegate(dir), filter);
public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) throws IOException {
Filter<Path> wrappedFilter = new Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
return filter.accept(new FilterPath(entry, fileSystem));
}
};
return new FilterDirectoryStream(delegate.newDirectoryStream(toDelegate(dir), wrappedFilter), fileSystem);
}
@Override
@ -239,11 +246,16 @@ public class FilterFileSystemProvider extends FileSystemProvider {
return delegate.readSymbolicLink(toDelegate(link));
}
private Path toDelegate(Path path) {
protected Path toDelegate(Path path) {
if (path instanceof FilterPath) {
return ((FilterPath) path).delegate;
FilterPath fp = (FilterPath) path;
if (fp.fileSystem != fileSystem) {
throw new ProviderMismatchException("mismatch, expected: " + fileSystem.provider().getClass() + ", got: " + fp.fileSystem.provider().getClass());
}
return fp.delegate;
} else {
throw new ProviderMismatchException("mismatch, expected: FilterPath, got: " + path.getClass());
}
return path;
}
private URI toDelegate(URI uri) {

View File

@ -23,6 +23,7 @@ import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.ProviderMismatchException;
import java.nio.file.WatchEvent.Kind;
import java.nio.file.WatchEvent.Modifier;
import java.nio.file.WatchKey;
@ -120,11 +121,7 @@ public class FilterPath implements Path {
@Override
public boolean startsWith(Path other) {
if (other instanceof FilterPath) {
FilterPath f = (FilterPath) other;
return fileSystem == f.fileSystem && delegate.startsWith(f.delegate);
}
return false;
return delegate.startsWith(toDelegate(other));
}
@Override
@ -134,11 +131,7 @@ public class FilterPath implements Path {
@Override
public boolean endsWith(Path other) {
if (other instanceof FilterPath) {
FilterPath f = (FilterPath) other;
return fileSystem == f.fileSystem && delegate.endsWith(f.delegate);
}
return false;
return delegate.endsWith(toDelegate(other));
}
@Override
@ -153,10 +146,7 @@ public class FilterPath implements Path {
@Override
public Path resolve(Path other) {
if (other instanceof FilterPath) {
other = ((FilterPath)other).delegate;
}
return wrap(delegate.resolve(other));
return wrap(delegate.resolve(toDelegate(other)));
}
@Override
@ -166,10 +156,7 @@ public class FilterPath implements Path {
@Override
public Path resolveSibling(Path other) {
if (other instanceof FilterPath) {
other = ((FilterPath)other).delegate;
}
return wrap(delegate.resolveSibling(other));
return wrap(delegate.resolveSibling(toDelegate(other)));
}
@Override
@ -179,10 +166,7 @@ public class FilterPath implements Path {
@Override
public Path relativize(Path other) {
if (other instanceof FilterPath) {
other = ((FilterPath)other).delegate;
}
return wrap(delegate.relativize(other));
return wrap(delegate.relativize(toDelegate(other)));
}
// TODO: should these methods not expose delegate result directly?
@ -247,10 +231,7 @@ public class FilterPath implements Path {
@Override
public int compareTo(Path other) {
if (other instanceof FilterPath) {
other = ((FilterPath)other).delegate;
}
return delegate.compareTo(other);
return delegate.compareTo(toDelegate(other));
}
/**
@ -273,4 +254,19 @@ public class FilterPath implements Path {
protected Path wrap(Path other) {
return new FilterPath(other, fileSystem);
}
/** Override this to customize the unboxing of Path
* from various operations
*/
protected Path toDelegate(Path path) {
if (path instanceof FilterPath) {
FilterPath fp = (FilterPath) path;
if (fp.fileSystem != fileSystem) {
throw new ProviderMismatchException("mismatch, expected: " + fileSystem.provider().getClass() + ", got: " + fp.fileSystem.provider().getClass());
}
return fp.delegate;
} else {
throw new ProviderMismatchException("mismatch, expected: FilterPath, got: " + path.getClass());
}
}
}

View File

@ -1,98 +0,0 @@
package org.apache.lucene.mockfile;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.SecureDirectoryStream;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.FileAttributeView;
import java.util.Iterator;
import java.util.Set;
/**
* A {@code FilterSecureDirectoryStream} contains another
* {@code SecureDirectoryStream}, which it uses as its basic
* source of data, possibly transforming the data along the
* way or providing additional functionality.
*/
public class FilterSecureDirectoryStream<T> implements SecureDirectoryStream<T> {
/**
* The underlying {@code SecureDirectoryStream} instance.
*/
protected final SecureDirectoryStream<T> delegate;
/**
* Construct a {@code FilterSecureDirectoryStream} based on
* the specified base stream.
* <p>
* Note that base stream is closed if this stream is closed.
* @param delegate specified base stream.
*/
public FilterSecureDirectoryStream(SecureDirectoryStream<T> delegate) {
this.delegate = delegate;
}
@Override
public Iterator<T> iterator() {
return delegate.iterator();
}
@Override
public void close() throws IOException {
delegate.close();
}
@Override
public SecureDirectoryStream<T> newDirectoryStream(T path, LinkOption... options) throws IOException {
return delegate.newDirectoryStream(path, options);
}
@Override
public SeekableByteChannel newByteChannel(T path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
return delegate.newByteChannel(path, options, attrs);
}
@Override
public void deleteFile(T path) throws IOException {
delegate.deleteFile(path);
}
@Override
public void deleteDirectory(T path) throws IOException {
delegate.deleteDirectory(path);
}
@Override
public void move(T srcpath, SecureDirectoryStream<T> targetdir, T targetpath) throws IOException {
delegate.move(srcpath, targetdir, targetpath);
}
@Override
public <V extends FileAttributeView> V getFileAttributeView(Class<V> type) {
return delegate.getFileAttributeView(type);
}
@Override
public <V extends FileAttributeView> V getFileAttributeView(T path, Class<V> type, LinkOption... options) {
return delegate.getFileAttributeView(path, type, options);
}
}

View File

@ -26,10 +26,8 @@ import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystem;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.SecureDirectoryStream;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.attribute.FileAttribute;
import java.util.Set;
@ -126,7 +124,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
@Override
public OutputStream newOutputStream(final Path path, OpenOption... options) throws IOException {
OutputStream stream = new FilterOutputStream2(super.newOutputStream(path, options)) {
OutputStream stream = new FilterOutputStream2(delegate.newOutputStream(toDelegate(path), options)) {
boolean closed;
@ -160,7 +158,7 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
@Override
public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
FileChannel channel = new FilterFileChannel(super.newFileChannel(path, options, attrs)) {
FileChannel channel = new FilterFileChannel(delegate.newFileChannel(toDelegate(path), options, attrs)) {
boolean closed;
@ -262,117 +260,42 @@ public abstract class HandleTrackingFS extends FilterFileSystemProvider {
@Override
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException {
DirectoryStream<Path> stream = super.newDirectoryStream(dir, filter);
if (stream instanceof SecureDirectoryStream) {
stream = new TrackingSecureDirectoryStream((SecureDirectoryStream<Path>)stream, dir);
} else {
stream = new FilterDirectoryStream<Path>(stream) {
boolean closed;
@Override
public void close() throws IOException {
if (!closed) {
closed = true;
onClose(dir, this);
}
super.close();
Filter<Path> wrappedFilter = new Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
return filter.accept(new FilterPath(entry, fileSystem));
}
};
DirectoryStream<Path> stream = delegate.newDirectoryStream(toDelegate(dir), wrappedFilter);
stream = new FilterDirectoryStream(stream, fileSystem) {
boolean closed;
@Override
public void close() throws IOException {
if (!closed) {
closed = true;
onClose(dir, this);
}
@Override
public String toString() {
return "DirectoryStream(" + dir + ")";
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
@Override
public boolean equals(Object obj) {
return this == obj;
}
};
}
super.close();
}
@Override
public String toString() {
return "DirectoryStream(" + dir + ")";
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
@Override
public boolean equals(Object obj) {
return this == obj;
}
};
callOpenHook(dir, stream);
return stream;
}
/** You can also open various things from SecureDirectoryStream, so we ensure we track those */
class TrackingSecureDirectoryStream extends FilterSecureDirectoryStream<Path> {
final Path dir;
TrackingSecureDirectoryStream(SecureDirectoryStream<Path> delegate, Path dir) {
super(delegate);
this.dir = dir;
}
boolean closed;
@Override
public void close() throws IOException {
if (!closed) {
closed = true;
onClose(dir, this);
}
super.close();
}
@Override
public String toString() {
return "SecureDirectoryStream(" + dir + ")";
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
@Override
public boolean equals(Object obj) {
return this == obj;
}
@Override
public SecureDirectoryStream<Path> newDirectoryStream(Path path, LinkOption... options) throws IOException {
SecureDirectoryStream<Path> stream = new TrackingSecureDirectoryStream(super.newDirectoryStream(path, options), path);
callOpenHook(path, stream);
return stream;
}
@Override
public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
SeekableByteChannel channel = new FilterSeekableByteChannel(super.newByteChannel(path, options, attrs)) {
boolean closed;
@Override
public void close() throws IOException {
if (!closed) {
closed = true;
onClose(path, this);
}
super.close();
}
@Override
public String toString() {
return "SeekableByteChannel(" + path.toString() + ")";
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
@Override
public boolean equals(Object obj) {
return this == obj;
}
};
callOpenHook(path, channel);
return channel;
}
}
}

View File

@ -23,13 +23,10 @@ import java.nio.channels.AsynchronousFileChannel;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.FileSystem;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.SecureDirectoryStream;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.util.Arrays;
@ -244,80 +241,4 @@ public class VerboseFS extends FilterFileSystemProvider {
}
throw new AssertionError();
}
@Override
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException {
DirectoryStream<Path> stream = super.newDirectoryStream(dir, filter);
if (stream instanceof SecureDirectoryStream) {
stream = new VerboseSecureDirectoryStream((SecureDirectoryStream<Path>)stream, dir);
}
return stream;
}
class VerboseSecureDirectoryStream extends FilterSecureDirectoryStream<Path> {
final Path dir;
VerboseSecureDirectoryStream(SecureDirectoryStream<Path> delegate, Path dir) {
super(delegate);
this.dir = dir;
}
@Override
public SecureDirectoryStream<Path> newDirectoryStream(Path path, LinkOption... options) throws IOException {
return new VerboseSecureDirectoryStream(super.newDirectoryStream(path, options), path);
}
@Override
public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
Throwable exception = null;
try {
return super.newByteChannel(path, options, attrs);
} catch (Throwable t) {
exception = t;
} finally {
if (containsDestructive(options)) {
sop("newByteChannel[SECURE]" + options + ": " + path(path), exception);
} else {
IOUtils.reThrow(exception);
}
}
throw new AssertionError();
}
@Override
public void deleteFile(Path path) throws IOException {
Throwable exception = null;
try {
super.deleteFile(path);
} catch (Throwable t) {
exception = t;
} finally {
sop("deleteFile[SECURE]: " + path(path), exception);
}
}
@Override
public void deleteDirectory(Path path) throws IOException {
Throwable exception = null;
try {
super.deleteDirectory(path);
} catch (Throwable t) {
exception = t;
} finally {
sop("deleteDirectory[SECURE]: " + path(path), exception);
}
}
@Override
public void move(Path srcpath, SecureDirectoryStream<Path> targetdir, Path targetpath) throws IOException {
Throwable exception = null;
try {
super.move(srcpath, targetdir, targetpath);
} catch (Throwable t) {
exception = t;
} finally {
sop("move[SECURE]: " + path(srcpath) + " -> " + path(targetpath), exception);
}
}
}
}