Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI 

(cherry picked from commit 9a8edb0aed)
This commit is contained in:
Da Zhou 2019-10-16 13:14:15 -07:00 committed by DadanielZ
parent 6cb9ee32e1
commit 7f71473888
No known key found for this signature in database
GPG Key ID: 5C91E270F172ADFB
2 changed files with 67 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
View File

@ -640,6 +640,20 @@ public class NativeAzureFileSystem extends FileSystem {
return "wasb";
}
/**
* If fs.azure.override.canonical.service.name is set as true, return URI of
* the WASB filesystem, otherwise use the default implementation.
*
* @return a service string that uniquely identifies this file system
*/
@Override
public String getCanonicalServiceName() {
if (returnUriAsCanonicalServiceName) {
return getUri().toString();
}
return super.getCanonicalServiceName();
}
/**
* <p>
@ -723,6 +737,11 @@ public class NativeAzureFileSystem extends FileSystem {
*/
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support";
/*
* Property to override canonical service name with filesystem's URI.
*/
public static final String RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME = "fs.azure.override.canonical.service.name";
/**
* The configuration property to set number of threads to be used for rename operation.
*/
@ -1198,6 +1217,7 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
private boolean returnUriAsCanonicalServiceName = false;
private DelegationTokenAuthenticatedURL authURL;
private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
private String credServiceUrl;
@ -1395,6 +1415,8 @@ public class NativeAzureFileSystem extends FileSystem {
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf);
}
this.returnUriAsCanonicalServiceName = conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
}
@Override

61
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
View File

@ -18,30 +18,16 @@
package org.apache.hadoop.fs.azure;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.junit.Assume.assumeNotNull;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.Date;
import java.util.EnumSet;
import java.io.File;
import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
@ -49,10 +35,24 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlockBlob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import static org.junit.Assume.assumeNotNull;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME;
public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
private static final int FILE_SIZE = 4096;
@ -581,4 +581,33 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
FileSystem.closeAll();
}
}
@Test
public void testCanonicalServiceName() throws Exception {
AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI("wasbs", authority, null, null, null);
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
try (FileSystem fs = FileSystem.get(conf)){
// Default getCanonicalServiceName() will try to resolve the host to IP,
// because the mock container does not exist, this call is expected to fail.
fs.getCanonicalServiceName();
Assert.assertTrue("Excepting exception", false);
} catch (IllegalArgumentException ex) {
if (!ex.getMessage().contains("java.net.UnknownHostException")) {
throw ex;
}
}
conf.setBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, true);
try (FileSystem fs = FileSystem.newInstance(defaultUri, conf)) {
Assert.assertEquals("getCanonicalServiceName() should return URI",
fs.getUri().toString(), fs.getCanonicalServiceName());
} finally {
testAccount.cleanup();
FileSystem.closeAll();
}
}
}