diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d0c25d19223..547c34916f5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -146,6 +146,8 @@ Release 2.0.0 - UNRELEASED HADOOP-8152. Expand public APIs for security library classes. (atm via eli) + HADOOP-7549. Use JDK ServiceLoader mechanism to find FileSystem implementations. (tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 288a3033ed6..4b8279f5e3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.ServiceLoader; import java.util.Set; import java.util.Stack; import java.util.TreeSet; @@ -184,6 +185,17 @@ public void initialize(URI name, Configuration conf) throws IOException { statistics = getStatistics(name.getScheme(), getClass()); } + /** + * Return the protocol scheme for the FileSystem. + *
+ * This implementation throws anUnsupportedOperationException
.
+ *
+ * @return the protocol scheme for the FileSystem.
+ */
+ public String getScheme() {
+ throw new UnsupportedOperationException("Not implemented by the FileSystem implementation");
+ }
+
/** Returns a URI whose scheme and authority identify this FileSystem.*/
public abstract URI getUri();
@@ -2078,9 +2090,45 @@ public void setTimes(Path p, long mtime, long atime
) throws IOException {
}
+ // making it volatile to be able to do a double checked locking
+ private volatile static boolean FILE_SYSTEMS_LOADED = false;
+
+ private static final Maphar
+ */
+ @Override
+ public String getScheme() {
+ return "har";
+ }
+
/**
* Constructor to create a HarFileSystem with an
* underlying filesystem.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
index 4aff81114b9..ac9b25d972a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
@@ -39,7 +39,18 @@ public class LocalFileSystem extends ChecksumFileSystem {
public LocalFileSystem() {
this(new RawLocalFileSystem());
}
-
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return file
+ */
+ @Override
+ public String getScheme() {
+ return "file";
+ }
+
public FileSystem getRaw() {
return getRawFileSystem();
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index f2072d9ae9c..1c19ce27fb8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -59,6 +59,17 @@ public class FTPFileSystem extends FileSystem {
private URI uri;
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return ftp
+ */
+ @Override
+ public String getScheme() {
+ return "ftp";
+ }
+
@Override
public void initialize(URI uri, Configuration conf) throws IOException { // get
super.initialize(uri, conf);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
index 68de2afb77c..591eeaf5e24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
@@ -57,6 +57,17 @@ public KosmosFileSystem() {
this.kfsImpl = fsimpl;
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return kfs
+ */
+ @Override
+ public String getScheme() {
+ return "kfs";
+ }
+
@Override
public URI getUri() {
return uri;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
index d8a9015235a..5a5d628adb0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
@@ -67,6 +67,17 @@ public S3FileSystem(FileSystemStore store) {
this.store = store;
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return s3
+ */
+ @Override
+ public String getScheme() {
+ return "s3";
+ }
+
@Override
public URI getUri() {
return uri;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
index ea672c44e6d..af1c7a9885e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
@@ -251,7 +251,18 @@ public NativeS3FileSystem() {
public NativeS3FileSystem(NativeFileSystemStore store) {
this.store = store;
}
-
+
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return s3n
+ */
+ @Override
+ public String getScheme() {
+ return "s3n";
+ }
+
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index c2bdaaad9da..20932ee278c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -149,6 +149,17 @@ public ViewFileSystem() throws IOException {
creationTime = System.currentTimeMillis();
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return viewfs
+ */
+ @Override
+ public String getScheme() {
+ return "viewfs";
+ }
+
/**
* Called after a new FileSystem instance is constructed.
* @param theUri a uri whose authority section names the host, port, etc. for
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
new file mode 100644
index 00000000000..74f1607f1ff
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.fs.LocalFileSystem
+org.apache.hadoop.fs.viewfs.ViewFileSystem
+org.apache.hadoop.fs.s3.S3FileSystem
+org.apache.hadoop.fs.s3native.NativeS3FileSystem
+org.apache.hadoop.fs.kfs.KosmosFileSystem
+org.apache.hadoop.fs.ftp.FTPFileSystem
+org.apache.hadoop.fs.HarFileSystem
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b6369b3c7ba..18a0ff1a2f9 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -352,25 +352,6 @@
-hdfs
+ */
+ @Override
+ public String getScheme() {
+ return "hdfs";
+ }
+
@Deprecated
public DistributedFileSystem(InetSocketAddress namenode,
Configuration conf) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
index 82b321d2dd4..befa58c56ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
@@ -154,6 +154,17 @@ public String getCanonicalServiceName() {
return SecurityUtil.buildTokenService(nnSecureUri).toString();
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return hftp
+ */
+ @Override
+ public String getScheme() {
+ return "hftp";
+ }
+
@Override
public void initialize(final URI name, final Configuration conf)
throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
index 141ebb2bbd4..a5326f316c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
@@ -58,6 +58,17 @@ public class HsftpFileSystem extends HftpFileSystem {
private static final long MM_SECONDS_PER_DAY = 1000 * 60 * 60 * 24;
private volatile int ExpWarnDays = 0;
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return hsftp
+ */
+ @Override
+ public String getScheme() {
+ return "hsftp";
+ }
+
@Override
public void initialize(URI name, Configuration conf) throws IOException {
super.initialize(name, conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 4e44e03e050..0b355ccaa98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -155,6 +155,17 @@ public static boolean isEnabled(final Configuration conf, final Log log) {
}
}
+ /**
+ * Return the protocol scheme for the FileSystem.
+ *
+ *
+ * @return webhdfs
+ */
+ @Override
+ public String getScheme() {
+ return "webhdfs";
+ }
+
@Override
public synchronized void initialize(URI uri, Configuration conf
) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
new file mode 100644
index 00000000000..848a3bd019b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.hdfs.DistributedFileSystem
+org.apache.hadoop.hdfs.HftpFileSystem
+org.apache.hadoop.hdfs.HsftpFileSystem
+org.apache.hadoop.hdfs.web.WebHdfsFileSystem