HADOOP-12304. Applications using FileContext fail with the default file system configured to be wasb/s3/etc. Contributed by Chris Nauroth.

(cherry picked from commit 3da0bedaec)
This commit is contained in:
cnauroth 2015-08-05 14:56:41 -07:00
parent 1466772827
commit b83d475252
4 changed files with 52 additions and 4 deletions

View File

@ -578,6 +578,9 @@ Release 2.7.2 - UNRELEASED
HDFS-8767. RawLocalFileSystem.listStatus() returns null for UNIX pipefile.
(kanaka kumar avvaru via wheat9)
HADOOP-12304. Applications using FileContext fail with the default file
system configured to be wasb/s3/etc. (cnauroth)
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES

View File

@ -46,12 +46,27 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem {
Configuration conf, String supportedScheme, boolean authorityRequired)
throws IOException, URISyntaxException {
super(theUri, supportedScheme, authorityRequired,
theFsImpl.getDefaultPort());
getDefaultPortIfDefined(theFsImpl));
fsImpl = theFsImpl;
fsImpl.initialize(theUri, conf);
fsImpl.statistics = getStatistics();
}
/**
* Returns the default port if the file system defines one.
* {@link FileSystem#getDefaultPort()} returns 0 to indicate the default port
* is undefined. However, the logic that consumes this value expects to
* receive -1 to indicate the port is undefined, which agrees with the
* contract of {@link URI#getPort()}.
*
* @param theFsImpl file system to check for default port
* @return default port, or -1 if default port is undefined
*/
private static int getDefaultPortIfDefined(FileSystem theFsImpl) {
int defaultPort = theFsImpl.getDefaultPort();
return defaultPort != 0 ? defaultPort : -1;
}
@Override
public Path getInitialWorkingDirectory() {
return fsImpl.getInitialWorkingDirectory();

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs.azure;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@ -35,6 +36,8 @@ import java.util.Date;
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
@ -362,7 +365,7 @@ public class TestWasbUriAndConfiguration {
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI(defaultScheme, authority, null, null, null);
conf.set("fs.default.name", defaultUri.toString());
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
// Add references to file system implementations for wasb and wasbs.
conf.addResource("azure-test.xml");
@ -385,11 +388,34 @@ public class TestWasbUriAndConfiguration {
// authority for the Azure file system should throw.
testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
conf.set("fs.default.name", "file:///");
conf.set(FS_DEFAULT_NAME_KEY, "file:///");
try {
FileSystem.get(new URI("wasb:///random/path"), conf);
fail("Should've thrown.");
} catch (IllegalArgumentException e) {
}
}
@Test
public void testWasbAsDefaultFileSystemHasNoPort() throws Exception {
try {
testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI("wasb", authority, null, null, null);
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
conf.addResource("azure-test.xml");
FileSystem fs = FileSystem.get(conf);
assertTrue(fs instanceof NativeAzureFileSystem);
assertEquals(-1, fs.getUri().getPort());
AbstractFileSystem afs = FileContext.getFileContext(conf)
.getDefaultFileSystem();
assertTrue(afs instanceof Wasb);
assertEquals(-1, afs.getUri().getPort());
} finally {
FileSystem.closeAll();
}
}
}

View File

@ -35,4 +35,8 @@
<value>true</value>
</property>
-->
<property>
<name>fs.AbstractFileSystem.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.Wasb</value>
</property>
</configuration>