From 284a1e58d734536a152d9abf0004e26be3e3cfb5 Mon Sep 17 00:00:00 2001 From: Virajith Jalaparti Date: Mon, 11 May 2020 22:18:40 -0700 Subject: [PATCH] HADOOP-15565. Add an inner FS cache to ViewFileSystem, separate from the global cache, to avoid file system leaks. Contributed by Jinglun. --- .../java/org/apache/hadoop/fs/FileSystem.java | 5 + .../hadoop/fs/viewfs/ChRootedFileSystem.java | 22 ++-- .../apache/hadoop/fs/viewfs/Constants.java | 7 ++ .../hadoop/fs/viewfs/ViewFileSystem.java | 101 +++++++++++++++++- .../org/apache/hadoop/fs/TestFileUtil.java | 7 ++ .../fs/viewfs/TestChRootedFileSystem.java | 28 ++++- .../viewfs/TestViewFileSystemDelegation.java | 40 ++++--- ...tViewFileSystemDelegationTokenSupport.java | 12 ++- .../fs/viewfs/ViewFileSystemBaseTest.java | 100 ++++++++++++++++- 9 files changed, 288 insertions(+), 34 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index e7f3624ae48..c32f4cd3d05 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -190,6 +190,11 @@ public abstract class FileSystem extends Configured implements Closeable { CACHE.map.put(new Cache.Key(uri, conf), fs); } + @VisibleForTesting + static int cacheSize() { + return CACHE.map.size(); + } + /** * Get a FileSystem instance based on the uri, the passed in * configuration and the user. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 3d8f03accea..61455e18b22 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -95,13 +95,12 @@ class ChRootedFileSystem extends FilterFileSystem { /** * Constructor - * @param uri base file system - * @param conf configuration + * @param fs base file system + * @param uri base uri * @throws IOException */ - public ChRootedFileSystem(final URI uri, Configuration conf) - throws IOException { - super(FileSystem.get(uri, conf)); + ChRootedFileSystem(final FileSystem fs, URI uri) throws IOException { + super(fs); String pathString = uri.getPath(); if (pathString.isEmpty()) { pathString = "/"; @@ -112,7 +111,18 @@ class ChRootedFileSystem extends FilterFileSystem { workingDir = getHomeDirectory(); // We don't use the wd of the myFs } - + + /** + * Constructor. + * @param uri base file system + * @param conf configuration + * @throws IOException + */ + public ChRootedFileSystem(final URI uri, Configuration conf) + throws IOException { + this(FileSystem.get(uri, conf), uri); + } + /** * Called after a new FileSystem instance is constructed. * @param name a uri whose authority section names the host, port, etc. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 9882a8e9338..7f9277cdd3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -68,4 +68,11 @@ public interface Constants { new FsPermission((short) 0555); String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy"; + + /** + * Enable ViewFileSystem to cache all children filesystems in inner cache. + */ + String CONFIG_VIEWFS_ENABLE_INNER_CACHE = "fs.viewfs.enable.inner.cache"; + + boolean CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT = true; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 4c73eaea189..c248a2781f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -18,16 +18,21 @@ package org.apache.hadoop.fs.viewfs; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.Map.Entry; @@ -85,7 +90,73 @@ public class ViewFileSystem extends FileSystem { final Path p) { return readOnlyMountTable(operation, p.toString()); } - + + /** + * Caching children filesystems. HADOOP-15565. + */ + static class InnerCache { + private Map map = new HashMap<>(); + + FileSystem get(URI uri, Configuration config) throws IOException { + Key key = new Key(uri); + if (map.get(key) == null) { + FileSystem fs = FileSystem.newInstance(uri, config); + map.put(key, fs); + return fs; + } else { + return map.get(key); + } + } + + void closeAll() { + for (FileSystem fs : map.values()) { + try { + fs.close(); + } catch (IOException e) { + LOG.info("Fail closing ViewFileSystem's child filesystem " + fs, e); + } + } + } + + InnerCache unmodifiableCache() { + map = Collections.unmodifiableMap(map); + return this; + } + + /** + * All the cached instances share the same UGI so there is no need to have a + * URI in the Key. Make the Key simple with just the scheme and authority. + */ + private static class Key { + private final String scheme; + private final String authority; + + Key(URI uri) { + scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(); + authority = + uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase(); + } + + @Override + public int hashCode() { + return Objects.hash(scheme, authority); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj != null && obj instanceof Key) { + Key that = (Key) obj; + return this.scheme.equals(that.scheme) && this.authority + .equals(that.authority); + } + return false; + } + } + } + static public class MountPoint { private Path src; // the src of the mount private URI[] targets; // target of the mount; Multiple targets imply mergeMount @@ -108,6 +179,8 @@ public class ViewFileSystem extends FileSystem { Configuration config; InodeTree fsState; // the fs state; ie the mount table Path homeDir = null; + private boolean enableInnerCache = false; + private InnerCache cache; // Default to rename within same mountpoint private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT; /** @@ -162,6 +235,9 @@ public class ViewFileSystem extends FileSystem { super.initialize(theUri, conf); setConf(conf); config = conf; + enableInnerCache = config.getBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, + CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT); + final InnerCache innerCache = new InnerCache(); // Now build client side view (i.e. client side mount table) from config. final String authority = theUri.getAuthority(); try { @@ -172,7 +248,13 @@ public class ViewFileSystem extends FileSystem { protected FileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, IOException { - return new ChRootedFileSystem(uri, config); + FileSystem fs; + if (enableInnerCache) { + fs = innerCache.get(uri, config); + } else { + fs = FileSystem.get(uri, config); + } + return new ChRootedFileSystem(fs, uri); } @Override @@ -190,6 +272,13 @@ public class ViewFileSystem extends FileSystem { // return MergeFs.createMergeFs(mergeFsURIList, config); } }; + + if (enableInnerCache) { + // All fs instances are created and cached on startup. The cache is + // readonly after the initialize() so the concurrent access of the cache + // is safe. + cache = innerCache.unmodifiableCache(); + } workingDir = this.getHomeDirectory(); renameStrategy = RenameStrategy.valueOf( conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, @@ -1130,4 +1219,12 @@ public class ViewFileSystem extends FileSystem { SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT, SAME_FILESYSTEM_ACROSS_MOUNTPOINT } + + @Override + public void close() throws IOException { + super.close(); + if (enableInnerCache && cache != null) { + cache.closeAll(); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index bbda2e45a47..b355729c394 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -1261,4 +1261,11 @@ public class TestFileUtil { assertEquals(FileUtil.compareFs(fs3,fs4),true); assertEquals(FileUtil.compareFs(fs1,fs6),false); } + + /* + * The size of FileSystem cache. + */ + public static int getCacheSize() { + return FileSystem.cacheSize(); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index e4a1ac9729a..92f266e6087 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.URI; import java.util.Collections; import java.util.List; +import java.util.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -32,7 +33,6 @@ import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; -import org.apache.hadoop.fs.viewfs.ChRootedFileSystem; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -396,7 +396,7 @@ public class TestChRootedFileSystem { } @Test - public void testListLocatedFileStatus() throws IOException { + public void testListLocatedFileStatus() throws Exception { final Path mockMount = new Path("mockfs://foo/user"); final Path mockPath = new Path("/usermock"); final Configuration conf = new Configuration(); @@ -404,17 +404,35 @@ public class TestChRootedFileSystem { ConfigUtil.addLink(conf, mockPath.toString(), mockMount.toUri()); FileSystem vfs = FileSystem.get(URI.create("viewfs:///"), conf); vfs.listLocatedStatus(mockPath); - final FileSystem mockFs = ((MockFileSystem)mockMount.getFileSystem(conf)) - .getRawFileSystem(); + final FileSystem mockFs = + ((MockFileSystem) getChildFileSystem((ViewFileSystem) vfs, + new URI("mockfs://foo/"))).getRawFileSystem(); verify(mockFs).listLocatedStatus(new Path(mockMount.toUri().getPath())); } + static FileSystem getChildFileSystem(ViewFileSystem viewFs, URI uri) { + for (FileSystem fs : viewFs.getChildFileSystems()) { + if (Objects.equals(fs.getUri().getScheme(), uri.getScheme()) && Objects + .equals(fs.getUri().getAuthority(), uri.getAuthority())) { + return fs; + } + } + return null; + } + static class MockFileSystem extends FilterFileSystem { + private URI uri; MockFileSystem() { super(mock(FileSystem.class)); } @Override - public void initialize(URI name, Configuration conf) throws IOException {} + public URI getUri() { + return uri; + } + @Override + public void initialize(URI name, Configuration conf) throws IOException { + uri = name; + } } @Test(timeout = 30000) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java index cb1f413bda4..d8c39f79d04 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.viewfs; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -31,6 +32,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.viewfs.TestChRootedFileSystem.MockFileSystem; import org.junit.*; + +import static org.apache.hadoop.fs.viewfs.TestChRootedFileSystem.getChildFileSystem; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -46,12 +49,16 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup { @BeforeClass public static void setup() throws Exception { conf = ViewFileSystemTestSetup.createConfig(); - fs1 = setupFileSystem(new URI("fs1:/"), FakeFileSystem.class); - fs2 = setupFileSystem(new URI("fs2:/"), FakeFileSystem.class); + setupFileSystem(new URI("fs1:/"), FakeFileSystem.class); + setupFileSystem(new URI("fs2:/"), FakeFileSystem.class); viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + fs1 = (FakeFileSystem) getChildFileSystem((ViewFileSystem) viewFs, + new URI("fs1:/")); + fs2 = (FakeFileSystem) getChildFileSystem((ViewFileSystem) viewFs, + new URI("fs2:/")); } - static FakeFileSystem setupFileSystem(URI uri, Class clazz) + static void setupFileSystem(URI uri, Class clazz) throws Exception { String scheme = uri.getScheme(); conf.set("fs."+scheme+".impl", clazz.getName()); @@ -59,22 +66,21 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup { assertEquals(uri, fs.getUri()); Path targetPath = new FileSystemTestHelper().getAbsoluteTestRootPath(fs); ConfigUtil.addLink(conf, "/mounts/"+scheme, targetPath.toUri()); - return fs; } - private static FileSystem setupMockFileSystem(Configuration conf, URI uri) + private static void setupMockFileSystem(Configuration config, URI uri) throws Exception { String scheme = uri.getScheme(); - conf.set("fs." + scheme + ".impl", MockFileSystem.class.getName()); - FileSystem fs = FileSystem.get(uri, conf); - ConfigUtil.addLink(conf, "/mounts/" + scheme, uri); - return ((MockFileSystem)fs).getRawFileSystem(); + config.set("fs." + scheme + ".impl", MockFileSystem.class.getName()); + ConfigUtil.addLink(config, "/mounts/" + scheme, uri); } @Test - public void testSanity() { - assertEquals("fs1:/", fs1.getUri().toString()); - assertEquals("fs2:/", fs2.getUri().toString()); + public void testSanity() throws URISyntaxException { + assertEquals(new URI("fs1:/").getScheme(), fs1.getUri().getScheme()); + assertEquals(new URI("fs1:/").getAuthority(), fs1.getUri().getAuthority()); + assertEquals(new URI("fs2:/").getScheme(), fs2.getUri().getScheme()); + assertEquals(new URI("fs2:/").getAuthority(), fs2.getUri().getAuthority()); } @Test @@ -91,9 +97,15 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup { @Test public void testAclMethods() throws Exception { Configuration conf = ViewFileSystemTestSetup.createConfig(); - FileSystem mockFs1 = setupMockFileSystem(conf, new URI("mockfs1:/")); - FileSystem mockFs2 = setupMockFileSystem(conf, new URI("mockfs2:/")); + setupMockFileSystem(conf, new URI("mockfs1:/")); + setupMockFileSystem(conf, new URI("mockfs2:/")); FileSystem viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + FileSystem mockFs1 = + ((MockFileSystem) getChildFileSystem((ViewFileSystem) viewFs, + new URI("mockfs1:/"))).getRawFileSystem(); + FileSystem mockFs2 = + ((MockFileSystem) getChildFileSystem((ViewFileSystem) viewFs, + new URI("mockfs2:/"))).getRawFileSystem(); Path viewFsPath1 = new Path("/mounts/mockfs1/a/b/c"); Path mockFsPath1 = new Path("/a/b/c"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java index 735dfcf3cf6..239f47d1da6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.viewfs; import static org.junit.Assert.*; +import static org.apache.hadoop.fs.viewfs.TestChRootedFileSystem.getChildFileSystem; import java.io.IOException; import java.net.URI; @@ -54,12 +55,16 @@ public class TestViewFileSystemDelegationTokenSupport { @BeforeClass public static void setup() throws Exception { conf = ViewFileSystemTestSetup.createConfig(); - fs1 = setupFileSystem(new URI("fs1:///"), FakeFileSystem.class); - fs2 = setupFileSystem(new URI("fs2:///"), FakeFileSystem.class); + setupFileSystem(new URI("fs1:///"), FakeFileSystem.class); + setupFileSystem(new URI("fs2:///"), FakeFileSystem.class); viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + fs1 = (FakeFileSystem) getChildFileSystem((ViewFileSystem) viewFs, + new URI("fs1:///")); + fs2 = (FakeFileSystem) getChildFileSystem((ViewFileSystem) viewFs, + new URI("fs2:///")); } - static FakeFileSystem setupFileSystem(URI uri, Class clazz) + static void setupFileSystem(URI uri, Class clazz) throws Exception { String scheme = uri.getScheme(); conf.set("fs."+scheme+".impl", clazz.getName()); @@ -67,7 +72,6 @@ public class TestViewFileSystemDelegationTokenSupport { // mount each fs twice, will later ensure 1 token/fs ConfigUtil.addLink(conf, "/mounts/"+scheme+"-one", fs.getUri()); ConfigUtil.addLink(conf, "/mounts/"+scheme+"-two", fs.getUri()); - return fs; } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 5ebcacb6fd9..eb48be1967f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -19,22 +19,25 @@ package org.apache.hadoop.fs.viewfs; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.List; - +import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.TestFileUtil; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import static org.apache.hadoop.fs.FileSystemTestHelper.*; import org.apache.hadoop.fs.permission.AclEntry; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.apache.hadoop.test.GenericTestUtils.*; +import static org.junit.Assert.*; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; @@ -962,4 +965,95 @@ abstract public class ViewFileSystemBaseTest { }); } + @Test + public void testViewFileSystemInnerCache() throws Exception { + ViewFileSystem.InnerCache cache = new ViewFileSystem.InnerCache(); + FileSystem fs = cache.get(fsTarget.getUri(), conf); + + // InnerCache caches filesystem. + assertSame(fs, cache.get(fsTarget.getUri(), conf)); + + // InnerCache and FileSystem.CACHE are independent. + assertNotSame(fs, FileSystem.get(fsTarget.getUri(), conf)); + + // close InnerCache. + cache.closeAll(); + try { + fs.exists(new Path("/")); + if (!(fs instanceof LocalFileSystem)) { + // Ignore LocalFileSystem because it can still be used after close. + fail("Expect Filesystem closed exception"); + } + } catch (IOException e) { + assertExceptionContains("Filesystem closed", e); + } + } + + @Test + public void testCloseChildrenFileSystem() throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + ConfigUtil.addLink(config, clusterName, "/user", + new Path(targetTestRoot, "user").toUri()); + config.setBoolean("fs.viewfs.impl.disable.cache", false); + URI uri = new URI("viewfs://" + clusterName + "/"); + + ViewFileSystem viewFs = (ViewFileSystem) FileSystem.get(uri, config); + assertTrue("viewfs should have at least one child fs.", + viewFs.getChildFileSystems().length > 0); + // viewFs is cached in FileSystem.CACHE + assertSame(viewFs, FileSystem.get(uri, config)); + + // child fs is not cached in FileSystem.CACHE + FileSystem child = viewFs.getChildFileSystems()[0]; + assertNotSame(child, FileSystem.get(child.getUri(), config)); + + viewFs.close(); + for (FileSystem childfs : viewFs.getChildFileSystems()) { + try { + childfs.exists(new Path("/")); + if (!(childfs instanceof LocalFileSystem)) { + // Ignore LocalFileSystem because it can still be used after close. + fail("Expect Filesystem closed exception"); + } + } catch (IOException e) { + assertExceptionContains("Filesystem closed", e); + } + } + } + + @Test + public void testChildrenFileSystemLeak() throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + ConfigUtil.addLink(config, clusterName, "/user", + new Path(targetTestRoot, "user").toUri()); + + final int cacheSize = TestFileUtil.getCacheSize(); + ViewFileSystem viewFs = (ViewFileSystem) FileSystem + .get(new URI("viewfs://" + clusterName + "/"), config); + assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); + viewFs.close(); + assertEquals(cacheSize, TestFileUtil.getCacheSize()); + } + + @Test + public void testDeleteOnExit() throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + ConfigUtil.addLink(config, clusterName, "/user", + new Path(targetTestRoot, "user").toUri()); + + Path testDir = new Path("/user/testDeleteOnExit"); + Path realTestPath = new Path(targetTestRoot, "user/testDeleteOnExit"); + ViewFileSystem viewFs = (ViewFileSystem) FileSystem + .get(new URI("viewfs://" + clusterName + "/"), config); + viewFs.mkdirs(testDir); + assertTrue(viewFs.exists(testDir)); + assertTrue(fsTarget.exists(realTestPath)); + + viewFs.deleteOnExit(testDir); + viewFs.close(); + assertFalse(fsTarget.exists(realTestPath)); + } }