HADOOP-12304. Applications using FileContext fail with the default file system configured to be wasb/s3/etc. Contributed by Chris Nauroth.
(cherry picked from commit 3da0bedaec
)
This commit is contained in:
parent
1466772827
commit
b83d475252
|
@ -578,6 +578,9 @@ Release 2.7.2 - UNRELEASED
|
||||||
HDFS-8767. RawLocalFileSystem.listStatus() returns null for UNIX pipefile.
|
HDFS-8767. RawLocalFileSystem.listStatus() returns null for UNIX pipefile.
|
||||||
(kanaka kumar avvaru via wheat9)
|
(kanaka kumar avvaru via wheat9)
|
||||||
|
|
||||||
|
HADOOP-12304. Applications using FileContext fail with the default file
|
||||||
|
system configured to be wasb/s3/etc. (cnauroth)
|
||||||
|
|
||||||
Release 2.7.1 - 2015-07-06
|
Release 2.7.1 - 2015-07-06
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -46,12 +46,27 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem {
|
||||||
Configuration conf, String supportedScheme, boolean authorityRequired)
|
Configuration conf, String supportedScheme, boolean authorityRequired)
|
||||||
throws IOException, URISyntaxException {
|
throws IOException, URISyntaxException {
|
||||||
super(theUri, supportedScheme, authorityRequired,
|
super(theUri, supportedScheme, authorityRequired,
|
||||||
theFsImpl.getDefaultPort());
|
getDefaultPortIfDefined(theFsImpl));
|
||||||
fsImpl = theFsImpl;
|
fsImpl = theFsImpl;
|
||||||
fsImpl.initialize(theUri, conf);
|
fsImpl.initialize(theUri, conf);
|
||||||
fsImpl.statistics = getStatistics();
|
fsImpl.statistics = getStatistics();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the default port if the file system defines one.
|
||||||
|
* {@link FileSystem#getDefaultPort()} returns 0 to indicate the default port
|
||||||
|
* is undefined. However, the logic that consumes this value expects to
|
||||||
|
* receive -1 to indicate the port is undefined, which agrees with the
|
||||||
|
* contract of {@link URI#getPort()}.
|
||||||
|
*
|
||||||
|
* @param theFsImpl file system to check for default port
|
||||||
|
* @return default port, or -1 if default port is undefined
|
||||||
|
*/
|
||||||
|
private static int getDefaultPortIfDefined(FileSystem theFsImpl) {
|
||||||
|
int defaultPort = theFsImpl.getDefaultPort();
|
||||||
|
return defaultPort != 0 ? defaultPort : -1;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Path getInitialWorkingDirectory() {
|
public Path getInitialWorkingDirectory() {
|
||||||
return fsImpl.getInitialWorkingDirectory();
|
return fsImpl.getInitialWorkingDirectory();
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
|
|
||||||
package org.apache.hadoop.fs.azure;
|
package org.apache.hadoop.fs.azure;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertNotNull;
|
import static org.junit.Assert.assertNotNull;
|
||||||
|
@ -35,6 +36,8 @@ import java.util.Date;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.AbstractFileSystem;
|
||||||
|
import org.apache.hadoop.fs.FileContext;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
|
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
|
||||||
|
@ -362,7 +365,7 @@ public class TestWasbUriAndConfiguration {
|
||||||
Configuration conf = testAccount.getFileSystem().getConf();
|
Configuration conf = testAccount.getFileSystem().getConf();
|
||||||
String authority = testAccount.getFileSystem().getUri().getAuthority();
|
String authority = testAccount.getFileSystem().getUri().getAuthority();
|
||||||
URI defaultUri = new URI(defaultScheme, authority, null, null, null);
|
URI defaultUri = new URI(defaultScheme, authority, null, null, null);
|
||||||
conf.set("fs.default.name", defaultUri.toString());
|
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
|
||||||
|
|
||||||
// Add references to file system implementations for wasb and wasbs.
|
// Add references to file system implementations for wasb and wasbs.
|
||||||
conf.addResource("azure-test.xml");
|
conf.addResource("azure-test.xml");
|
||||||
|
@ -385,11 +388,34 @@ public class TestWasbUriAndConfiguration {
|
||||||
// authority for the Azure file system should throw.
|
// authority for the Azure file system should throw.
|
||||||
testAccount = AzureBlobStorageTestAccount.createMock();
|
testAccount = AzureBlobStorageTestAccount.createMock();
|
||||||
Configuration conf = testAccount.getFileSystem().getConf();
|
Configuration conf = testAccount.getFileSystem().getConf();
|
||||||
conf.set("fs.default.name", "file:///");
|
conf.set(FS_DEFAULT_NAME_KEY, "file:///");
|
||||||
try {
|
try {
|
||||||
FileSystem.get(new URI("wasb:///random/path"), conf);
|
FileSystem.get(new URI("wasb:///random/path"), conf);
|
||||||
fail("Should've thrown.");
|
fail("Should've thrown.");
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWasbAsDefaultFileSystemHasNoPort() throws Exception {
|
||||||
|
try {
|
||||||
|
testAccount = AzureBlobStorageTestAccount.createMock();
|
||||||
|
Configuration conf = testAccount.getFileSystem().getConf();
|
||||||
|
String authority = testAccount.getFileSystem().getUri().getAuthority();
|
||||||
|
URI defaultUri = new URI("wasb", authority, null, null, null);
|
||||||
|
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
|
||||||
|
conf.addResource("azure-test.xml");
|
||||||
|
|
||||||
|
FileSystem fs = FileSystem.get(conf);
|
||||||
|
assertTrue(fs instanceof NativeAzureFileSystem);
|
||||||
|
assertEquals(-1, fs.getUri().getPort());
|
||||||
|
|
||||||
|
AbstractFileSystem afs = FileContext.getFileContext(conf)
|
||||||
|
.getDefaultFileSystem();
|
||||||
|
assertTrue(afs instanceof Wasb);
|
||||||
|
assertEquals(-1, afs.getUri().getPort());
|
||||||
|
} finally {
|
||||||
|
FileSystem.closeAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,4 +35,8 @@
|
||||||
<value>true</value>
|
<value>true</value>
|
||||||
</property>
|
</property>
|
||||||
-->
|
-->
|
||||||
|
<property>
|
||||||
|
<name>fs.AbstractFileSystem.wasb.impl</name>
|
||||||
|
<value>org.apache.hadoop.fs.azure.Wasb</value>
|
||||||
|
</property>
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
Loading…
Reference in New Issue