diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d0c25d19223..547c34916f5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -146,6 +146,8 @@ Release 2.0.0 - UNRELEASED HADOOP-8152. Expand public APIs for security library classes. (atm via eli) + HADOOP-7549. Use JDK ServiceLoader mechanism to find FileSystem implementations. (tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 288a3033ed6..4b8279f5e3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.ServiceLoader; import java.util.Set; import java.util.Stack; import java.util.TreeSet; @@ -184,6 +185,17 @@ public void initialize(URI name, Configuration conf) throws IOException { statistics = getStatistics(name.getScheme(), getClass()); } + /** + * Return the protocol scheme for the FileSystem. + *

+ * This implementation throws an UnsupportedOperationException. + * + * @return the protocol scheme for the FileSystem. + */ + public String getScheme() { + throw new UnsupportedOperationException("Not implemented by the FileSystem implementation"); + } + /** Returns a URI whose scheme and authority identify this FileSystem.*/ public abstract URI getUri(); @@ -2078,9 +2090,45 @@ public void setTimes(Path p, long mtime, long atime ) throws IOException { } + // making it volatile to be able to do a double checked locking + private volatile static boolean FILE_SYSTEMS_LOADED = false; + + private static final Map> + SERVICE_FILE_SYSTEMS = new HashMap>(); + + private static void loadFileSystems() { + synchronized (FileSystem.class) { + if (!FILE_SYSTEMS_LOADED) { + ServiceLoader serviceLoader = ServiceLoader.load(FileSystem.class); + for (FileSystem fs : serviceLoader) { + SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); + } + FILE_SYSTEMS_LOADED = true; + } + } + } + + public static Class getFileSystemClass(String scheme, + Configuration conf) throws IOException { + if (!FILE_SYSTEMS_LOADED) { + loadFileSystems(); + } + Class clazz = null; + if (conf != null) { + clazz = (Class) conf.getClass("fs." + scheme + ".impl", null); + } + if (clazz == null) { + clazz = SERVICE_FILE_SYSTEMS.get(scheme); + } + if (clazz == null) { + throw new IOException("No FileSystem for scheme: " + scheme); + } + return clazz; + } + private static FileSystem createFileSystem(URI uri, Configuration conf ) throws IOException { - Class clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null); + Class clazz = getFileSystemClass(uri.getScheme(), conf); if (clazz == null) { throw new IOException("No FileSystem for scheme: " + uri.getScheme()); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 9a5b28381c3..48afb671602 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -71,7 +71,18 @@ public class HarFileSystem extends FilterFileSystem { */ public HarFileSystem() { } - + + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return har + */ + @Override + public String getScheme() { + return "har"; + } + /** * Constructor to create a HarFileSystem with an * underlying filesystem. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java index 4aff81114b9..ac9b25d972a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java @@ -39,7 +39,18 @@ public class LocalFileSystem extends ChecksumFileSystem { public LocalFileSystem() { this(new RawLocalFileSystem()); } - + + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return file + */ + @Override + public String getScheme() { + return "file"; + } + public FileSystem getRaw() { return getRawFileSystem(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java index f2072d9ae9c..1c19ce27fb8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java @@ -59,6 +59,17 @@ public class FTPFileSystem extends FileSystem { private URI uri; + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return ftp + */ + @Override + public String getScheme() { + return "ftp"; + } + @Override public void initialize(URI uri, Configuration conf) throws IOException { // get super.initialize(uri, conf); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java index 68de2afb77c..591eeaf5e24 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java @@ -57,6 +57,17 @@ public KosmosFileSystem() { this.kfsImpl = fsimpl; } + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return kfs + */ + @Override + public String getScheme() { + return "kfs"; + } + @Override public URI getUri() { return uri; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java index d8a9015235a..5a5d628adb0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java @@ -67,6 +67,17 @@ public S3FileSystem(FileSystemStore store) { this.store = store; } + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return s3 + */ + @Override + public String getScheme() { + return "s3"; + } + @Override public URI getUri() { return uri; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java index ea672c44e6d..af1c7a9885e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java @@ -251,7 +251,18 @@ public NativeS3FileSystem() { public NativeS3FileSystem(NativeFileSystemStore store) { this.store = store; } - + + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return s3n + */ + @Override + public String getScheme() { + return "s3n"; + } + @Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index c2bdaaad9da..20932ee278c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -149,6 +149,17 @@ public ViewFileSystem() throws IOException { creationTime = System.currentTimeMillis(); } + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return viewfs + */ + @Override + public String getScheme() { + return "viewfs"; + } + /** * Called after a new FileSystem instance is constructed. * @param theUri a uri whose authority section names the host, port, etc. for diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem new file mode 100644 index 00000000000..74f1607f1ff --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.fs.LocalFileSystem +org.apache.hadoop.fs.viewfs.ViewFileSystem +org.apache.hadoop.fs.s3.S3FileSystem +org.apache.hadoop.fs.s3native.NativeS3FileSystem +org.apache.hadoop.fs.kfs.KosmosFileSystem +org.apache.hadoop.fs.ftp.FTPFileSystem +org.apache.hadoop.fs.HarFileSystem diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index b6369b3c7ba..18a0ff1a2f9 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -352,25 +352,6 @@ - - fs.file.impl - org.apache.hadoop.fs.LocalFileSystem - The FileSystem for file: uris. - - - - fs.hdfs.impl - org.apache.hadoop.hdfs.DistributedFileSystem - The FileSystem for hdfs: uris. - - - - fs.viewfs.impl - org.apache.hadoop.fs.viewfs.ViewFileSystem - The FileSystem for view file system for viewfs: uris - (ie client side mount table:). - - fs.AbstractFileSystem.file.impl org.apache.hadoop.fs.local.LocalFs @@ -391,45 +372,6 @@ (ie client side mount table:). - - fs.s3.impl - org.apache.hadoop.fs.s3.S3FileSystem - The FileSystem for s3: uris. - - - - fs.s3n.impl - org.apache.hadoop.fs.s3native.NativeS3FileSystem - The FileSystem for s3n: (Native S3) uris. - - - - fs.kfs.impl - org.apache.hadoop.fs.kfs.KosmosFileSystem - The FileSystem for kfs: uris. - - - - fs.hftp.impl - org.apache.hadoop.hdfs.HftpFileSystem - - - - fs.hsftp.impl - org.apache.hadoop.hdfs.HsftpFileSystem - - - - fs.webhdfs.impl - org.apache.hadoop.hdfs.web.WebHdfsFileSystem - - - - fs.ftp.impl - org.apache.hadoop.fs.ftp.FTPFileSystem - The FileSystem for ftp: uris. - - fs.ftp.host 0.0.0.0 @@ -444,18 +386,6 @@ - - fs.har.impl - org.apache.hadoop.fs.HarFileSystem - The filesystem for Hadoop archives. - - - - fs.har.impl.disable.cache - true - Don't cache 'har' filesystem instances. - - fs.df.interval 60000 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java index 0ec8305a5e4..1e176a0a12f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCaching.java @@ -43,7 +43,7 @@ public class TestFileSystemCaching { @Test public void testCacheEnabled() throws Exception { Configuration conf = new Configuration(); - conf.set("fs.cachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); FileSystem fs1 = FileSystem.get(new URI("cachedfile://a"), conf); FileSystem fs2 = FileSystem.get(new URI("cachedfile://a"), conf); assertSame(fs1, fs2); @@ -84,7 +84,7 @@ public void run() { // wait for InitializeForeverFileSystem to start initialization InitializeForeverFileSystem.sem.acquire(); - conf.set("fs.cachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); FileSystem.get(new URI("cachedfile://a"), conf); t.interrupt(); t.join(); @@ -93,7 +93,7 @@ public void run() { @Test public void testCacheDisabled() throws Exception { Configuration conf = new Configuration(); - conf.set("fs.uncachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.uncachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); conf.setBoolean("fs.uncachedfile.impl.disable.cache", true); FileSystem fs1 = FileSystem.get(new URI("uncachedfile://a"), conf); FileSystem fs2 = FileSystem.get(new URI("uncachedfile://a"), conf); @@ -104,7 +104,7 @@ public void testCacheDisabled() throws Exception { @Test public void testCacheForUgi() throws Exception { final Configuration conf = new Configuration(); - conf.set("fs.cachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); UserGroupInformation ugiA = UserGroupInformation.createRemoteUser("foo"); UserGroupInformation ugiB = UserGroupInformation.createRemoteUser("bar"); FileSystem fsA = ugiA.doAs(new PrivilegedExceptionAction() { @@ -156,7 +156,7 @@ public FileSystem run() throws Exception { @Test public void testUserFS() throws Exception { final Configuration conf = new Configuration(); - conf.set("fs.cachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); FileSystem fsU1 = FileSystem.get(new URI("cachedfile://a"), conf, "bar"); FileSystem fsU2 = FileSystem.get(new URI("cachedfile://a"), conf, "foo"); @@ -166,7 +166,7 @@ public void testUserFS() throws Exception { @Test public void testFsUniqueness() throws Exception { final Configuration conf = new Configuration(); - conf.set("fs.cachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); // multiple invocations of FileSystem.get return the same object. FileSystem fs1 = FileSystem.get(conf); FileSystem fs2 = FileSystem.get(conf); @@ -183,7 +183,7 @@ public void testFsUniqueness() throws Exception { @Test public void testCloseAllForUGI() throws Exception { final Configuration conf = new Configuration(); - conf.set("fs.cachedfile.impl", conf.get("fs.file.impl")); + conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName()); UserGroupInformation ugiA = UserGroupInformation.createRemoteUser("foo"); FileSystem fsA = ugiA.doAs(new PrivilegedExceptionAction() { public FileSystem run() throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index 364f46d2a40..aeb926004b1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -165,7 +165,10 @@ public void primitiveMkdir(Path f, FsPermission absolutePermission, public Token getDelegationToken(String renewer) throws IOException { return null; } - + + public String getScheme() { + return "dontcheck"; + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 988a6e7ee3f..1c10044b044 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -88,6 +88,17 @@ public class DistributedFileSystem extends FileSystem { public DistributedFileSystem() { } + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return hdfs + */ + @Override + public String getScheme() { + return "hdfs"; + } + @Deprecated public DistributedFileSystem(InetSocketAddress namenode, Configuration conf) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index 82b321d2dd4..befa58c56ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -154,6 +154,17 @@ public String getCanonicalServiceName() { return SecurityUtil.buildTokenService(nnSecureUri).toString(); } + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return hftp + */ + @Override + public String getScheme() { + return "hftp"; + } + @Override public void initialize(final URI name, final Configuration conf) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java index 141ebb2bbd4..a5326f316c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java @@ -58,6 +58,17 @@ public class HsftpFileSystem extends HftpFileSystem { private static final long MM_SECONDS_PER_DAY = 1000 * 60 * 60 * 24; private volatile int ExpWarnDays = 0; + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return hsftp + */ + @Override + public String getScheme() { + return "hsftp"; + } + @Override public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 4e44e03e050..0b355ccaa98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -155,6 +155,17 @@ public static boolean isEnabled(final Configuration conf, final Log log) { } } + /** + * Return the protocol scheme for the FileSystem. + *

+ * + * @return webhdfs + */ + @Override + public String getScheme() { + return "webhdfs"; + } + @Override public synchronized void initialize(URI uri, Configuration conf ) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem new file mode 100644 index 00000000000..848a3bd019b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.hdfs.DistributedFileSystem +org.apache.hadoop.hdfs.HftpFileSystem +org.apache.hadoop.hdfs.HsftpFileSystem +org.apache.hadoop.hdfs.web.WebHdfsFileSystem