From 7f71473888f1c06b22083038e6c398252fdf42d0 Mon Sep 17 00:00:00 2001 From: Da Zhou Date: Wed, 16 Oct 2019 13:14:15 -0700 Subject: [PATCH] HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI (cherry picked from commit 9a8edb0aeddd7787b2654f6e2a8465c325e048a2) --- .../fs/azure/NativeAzureFileSystem.java | 22 +++++++ .../azure/ITestWasbUriAndConfiguration.java | 61 ++++++++++++++----- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 11a110b1414..41b13068a72 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -640,6 +640,20 @@ public String getScheme() { return "wasb"; } + /** + * If fs.azure.override.canonical.service.name is set as true, return URI of + * the WASB filesystem, otherwise use the default implementation. + * + * @return a service string that uniquely identifies this file system + */ + @Override + public String getCanonicalServiceName() { + if (returnUriAsCanonicalServiceName) { + return getUri().toString(); + } + return super.getCanonicalServiceName(); + } + /** *

@@ -723,6 +737,11 @@ public String getScheme() { */ public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME = "fs.azure.enable.append.support"; + /* + * Property to override canonical service name with filesystem's URI. + */ + public static final String RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME = "fs.azure.override.canonical.service.name"; + /** * The configuration property to set number of threads to be used for rename operation. */ @@ -1198,6 +1217,7 @@ private void restoreKey() throws IOException { // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); private boolean appendSupportEnabled = false; + private boolean returnUriAsCanonicalServiceName = false; private DelegationTokenAuthenticatedURL authURL; private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); private String credServiceUrl; @@ -1395,6 +1415,8 @@ public void initialize(URI uri, Configuration conf) if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) { this.wasbDelegationTokenManager = new RemoteWasbDelegationTokenManager(conf); } + + this.returnUriAsCanonicalServiceName = conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false); } @Override diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java index 7783684e9d2..62a751dc44d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java @@ -18,30 +18,16 @@ package org.apache.hadoop.fs.azure; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; -import static org.junit.Assume.assumeNotNull; - import java.io.ByteArrayInputStream; import java.io.DataInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.util.Date; import java.util.EnumSet; -import java.io.File; -import org.apache.hadoop.fs.azure.integration.AzureTestUtils; -import org.apache.hadoop.security.ProviderUtils; -import org.apache.hadoop.security.alias.CredentialProvider; -import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.AbstractFileSystem; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; import org.junit.After; import org.junit.Assert; import org.junit.Assume; @@ -49,10 +35,24 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; +import org.apache.hadoop.fs.azure.integration.AzureTestUtils; +import org.apache.hadoop.security.ProviderUtils; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; + +import static org.junit.Assume.assumeNotNull; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME; + public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout { private static final int FILE_SIZE = 4096; @@ -581,4 +581,33 @@ public void testUserAgentConfig() throws Exception { FileSystem.closeAll(); } } + + @Test + public void testCanonicalServiceName() throws Exception { + AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount.createMock(); + Configuration conf = testAccount.getFileSystem().getConf(); + String authority = testAccount.getFileSystem().getUri().getAuthority(); + URI defaultUri = new URI("wasbs", authority, null, null, null); + conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); + try (FileSystem fs = FileSystem.get(conf)){ + // Default getCanonicalServiceName() will try to resolve the host to IP, + // because the mock container does not exist, this call is expected to fail. + fs.getCanonicalServiceName(); + Assert.assertTrue("Excepting exception", false); + } catch (IllegalArgumentException ex) { + if (!ex.getMessage().contains("java.net.UnknownHostException")) { + throw ex; + } + } + conf.setBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, true); + + try (FileSystem fs = FileSystem.newInstance(defaultUri, conf)) { + Assert.assertEquals("getCanonicalServiceName() should return URI", + fs.getUri().toString(), fs.getCanonicalServiceName()); + } finally { + testAccount.cleanup(); + FileSystem.closeAll(); + } + } + }