Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri 

(cherry picked from commit 8652cce5b2)
This commit is contained in:
Chris Douglas 2015-12-18 18:21:52 -08:00
parent dd91cf9fa5
commit d984ba4fd7
3 changed files with 35 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -782,8 +782,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-12656. MiniKdc throws "address in use" BindException.
(Wei-Chiu Chuang via Arpit Agarwal)
OPTIMIZATIONS
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
over getMessage() in logging/span events. (Varun Saxena via stevel)
@ -899,6 +897,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10729. Add tests for PB RPC in case version mismatch of client and
server. (Junping Du via wheat9)
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems.
(Inigo Goiri via cdouglas)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

18
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
@ -2708,8 +2710,20 @@ public abstract class FileSystem extends Configured implements Closeable {
synchronized (FileSystem.class) {
if (!FILE_SYSTEMS_LOADED) {
ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
for (FileSystem fs : serviceLoader) {
SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
Iterator<FileSystem> it = serviceLoader.iterator();
while (it.hasNext()) {
FileSystem fs = null;
try {
fs = it.next();
try {
SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
} catch (Exception e) {
LOG.warn("Cannot load: " + fs + " from " +
ClassUtil.findContainingJar(fs.getClass()), e);
}
} catch (ServiceConfigurationError ee) {
LOG.warn("Cannot load filesystem", ee);
}
}
FILE_SYSTEMS_LOADED = true;
}

16
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
View File

@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.net.URL;
import java.util.ServiceConfigurationError;
import org.junit.Test;
import static org.junit.Assert.*;
@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
assertFalse(false);
}
}
@Test
public void testMissingLibraries() {
boolean catched = false;
try {
Configuration conf = new Configuration();
FileSystem.getFileSystemClass("s3a", conf);
} catch (Exception e) {
catched = true;
} catch (ServiceConfigurationError e) {
// S3A shouldn't find AWS SDK and fail
catched = true;
}
assertTrue(catched);
}
}