HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri

(cherry picked from commit 8652cce5b2)
(cherry picked from commit d984ba4fd7)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -765,8 +765,6 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12656. MiniKdc throws "address in use" BindException.
     (Wei-Chiu Chuang via Arpit Agarwal)
 
-  OPTIMIZATIONS
-
     HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
     over getMessage() in logging/span events. (Varun Saxena via stevel)
 
@@ -882,6 +880,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-10729. Add tests for PB RPC in case version mismatch of client and
     server. (Junping Du via wheat9)
 
+    HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems.
+    (Inigo Goiri via cdouglas)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -36,6 +36,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.Stack;
@@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -2708,8 +2710,20 @@ public abstract class FileSystem extends Configured implements Closeable {
     synchronized (FileSystem.class) {
       if (!FILE_SYSTEMS_LOADED) {
         ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
-        for (FileSystem fs : serviceLoader) {
+        Iterator<FileSystem> it = serviceLoader.iterator();
-          SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+        while (it.hasNext()) {
+          FileSystem fs = null;
+          try {
+            fs = it.next();
+            try {
+              SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+            } catch (Exception e) {
+              LOG.warn("Cannot load: " + fs + " from " +
+                  ClassUtil.findContainingJar(fs.getClass()), e);
+            }
+          } catch (ServiceConfigurationError ee) {
+            LOG.warn("Cannot load filesystem", ee);
+          }
         }
         FILE_SYSTEMS_LOADED = true;
       }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ServiceConfigurationError;
 
 import org.junit.Test;
 import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
       assertFalse(false);
     }
   }
+
+  @Test
+  public void testMissingLibraries() {
+    boolean catched = false;
+    try {
+      Configuration conf = new Configuration();
+      FileSystem.getFileSystemClass("s3a", conf);
+    } catch (Exception e) {
+      catched = true;
+    } catch (ServiceConfigurationError e) {
+      // S3A shouldn't find AWS SDK and fail
+      catched = true;
+    }
+    assertTrue(catched);
+  }
 }