Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15663. ABFS: Simplify configuration. 

Contributed by Da Zhou.
This commit is contained in:
Thomas Marquardt 2018-08-31 03:24:42 +00:00
parent df57c6c3b1
commit 81dc4a995c
24 changed files with 431 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1618,6 +1618,18 @@
</property>
<!-- Azure file system properties -->
<property>
<name>fs.AbstractFileSystem.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.Wasb</value>
<description>AbstractFileSystem implementation class of wasb://</description>
</property>
<property>
<name>fs.AbstractFileSystem.wasbs.impl</name>
<value>org.apache.hadoop.fs.azure.Wasbs</value>
<description>AbstractFileSystem implementation class of wasbs://</description>
</property>
<property>
<name>fs.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>

4
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View File

@ -173,10 +173,6 @@ public class AbfsConfiguration{
}
}
public boolean isEmulator() {
return this.getConfiguration().getBoolean(FS_AZURE_EMULATOR_ENABLED, false);
}
public boolean isSecureMode() {
return isSecure;
}

24
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
View File

@ -77,6 +77,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@ -86,6 +87,7 @@ import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT;
import static org.apache.hadoop.util.Time.now;
/**
@ -146,7 +148,27 @@ public class AzureBlobFileSystemStore {
final URIBuilder uriBuilder = new URIBuilder();
uriBuilder.setScheme(scheme);
uriBuilder.setHost(hostName);
// For testing purposes, an IP address and port may be provided to override
// the host specified in the FileSystem URI. Also note that the format of
// the Azure Storage Service URI changes from
// http[s]://[account][domain-suffix]/[filesystem] to
// http[s]://[ip]:[port]/[account]/[filesystem].
String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT);
if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) {
uriBuilder.setHost(hostName);
return uriBuilder;
}
// Split ip and port
String[] data = endPoint.split(AbfsHttpConstants.COLON);
if (data.length != 2) {
throw new RuntimeException(String.format("ABFS endpoint is not set correctly : %s, "
+ "Do not specify scheme when using {IP}:{PORT}", endPoint));
}
uriBuilder.setHost(data[0].trim());
uriBuilder.setPort(Integer.parseInt(data[1].trim()));
uriBuilder.setPath("/" + UriUtils.extractAccountNameFromHostName(hostName));
return uriBuilder;
}

5
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
View File

@ -37,9 +37,6 @@ public final class ConfigurationKeys {
public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval";
public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries";
// Remove this and use common azure storage emulator property for public release.
public static final String FS_AZURE_EMULATOR_ENABLED = "fs.azure.abfs.emulator.enabled";
// Read and write buffer sizes defined by the user
public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size";
public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size";
@ -60,6 +57,8 @@ public final class ConfigurationKeys {
public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider.";
public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
/** End point of ABFS account: {@value}. */
public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint";
/** Prefix for auth type properties: {@value}. */
public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type.";
/** Prefix for oauth token provider type: {@value}. */

15
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
View File

@ -41,20 +41,21 @@ public final class UriUtils {
}
/**
* Extracts the raw account name from account name.
* @param accountName to extract the raw account name.
* @return extracted raw account name.
* Extracts the account name from the host name.
* @param hostName the fully-qualified domain name of the storage service
* endpoint (e.g. {account}.dfs.core.windows.net.
* @return the storage service account name.
*/
public static String extractRawAccountFromAccountName(final String accountName) {
if (accountName == null || accountName.isEmpty()) {
public static String extractAccountNameFromHostName(final String hostName) {
if (hostName == null || hostName.isEmpty()) {
return null;
}
if (!containsAbfsUrl(accountName)) {
if (!containsAbfsUrl(hostName)) {
return null;
}
String[] splitByDot = accountName.split("\\.");
String[] splitByDot = hostName.split("\\.");
if (splitByDot.length == 0) {
return null;
}

233
hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
View File

@ -90,7 +90,7 @@ For example:
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.azure.test.account.name</name>
<name>fs.azure.wasb.account.name</name>
<value>{ACCOUNTNAME}.blob.core.windows.net</value>
</property>
<property>
@ -126,7 +126,7 @@ Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml`
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.azure.test.account.name</name>
<name>fs.azure.wasb.account.name</name>
<value>{ACCOUNTNAME}.blob.core.windows.net</value>
</property>
<property>
@ -576,77 +576,172 @@ This will delete the containers; the output log of the test run will
provide the details and summary of the operation.
## Testing ABFS
## Testing the Azure ABFS Client
The ABFS Connector tests share the same account as the wasb tests; this is
needed for cross-connector compatibility tests.
Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to
big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS
schemes target the ADLS Gen 2 REST API, and the WASB and WASBS schemes target
the Azure Blob Storage REST API. ADLS Gen 2 offers better performance and
scalability. ADLS Gen 2 also offers authentication and authorization compatible
with the Hadoop Distributed File System permissions model when hierarchical
namespace is enabled for the storage account. Furthermore, the metadata and data
produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa.
In order to test ABFS, please add the following configuration to your
`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
compatibility tests which require WASB credentials, in addition to the ABFS
credentials.
This makes for a somewhat complex set of configuration options.
```xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<property>
<name>fs.azure.abfs.account.name</name>
<value>{ACCOUNT_NAME}.dfs.core.windows.net</value>
</property>
Here are the settings for an account `ACCOUNTNAME`
<property>
<name>fs.azure.account.key.{ACCOUNT_NAME}.dfs.core.windows.net</name>
<value>{ACCOUNT_ACCESS_KEY}</value>
</property>
<property>
<name>fs.azure.wasb.account.name</name>
<value>{ACCOUNT_NAME}.blob.core.windows.net</value>
</property>
<property>
<name>fs.azure.account.key.{ACCOUNT_NAME}.blob.core.windows.net</name>
<value>{ACCOUNT_ACCESS_KEY}</value>
</property>
<property>
<name>fs.contract.test.fs.abfs</name>
<value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
<description>A file system URI to be used by the contract tests.</description>
</property>
<property>
<name>fs.contract.test.fs.wasb</name>
<value>wasb://{CONTAINER_NAME}@{ACCOUNT_NAME}.blob.core.windows.net</value>
<description>A file system URI to be used by the contract tests.</description>
</property>
</configuration>
```
To run OAuth and ACL test cases you must use a storage account with the
hierarchical namespace enabled, and set the following configuration settings:
```xml
<!--=========================== AUTHENTICATION OPTIONS ===================-->
<!--ATTENTION:
TO RUN ABFS & WASB COMPATIBILITY TESTS, YOU MUST SET AUTH TYPE AS SharedKey.
OAUTH IS INTRODUCED TO ABFS ONLY.-->
<property>
<name>fs.azure.account.auth.type.{YOUR_ABFS_ACCOUNT_NAME}</name>
<value>{AUTH TYPE}</value>
<description>The authorization type can be SharedKey, OAuth, or Custom. The
default is SharedKey.</description>
</property>
<!--============================= FOR OAUTH ===========================-->
<!--IF AUTH TYPE IS SET AS OAUTH, FOLLOW THE STEPS BELOW-->
<!--NOTICE: AAD client and tenant related properties can be obtained through Azure Portal-->
<!--1. UNCOMMENT BELOW AND CHOOSE YOUR OAUTH PROVIDER TYPE -->
<!--
<property>
<name>fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}</name>
<value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
<description>The full name of token provider class name.</description>
</property>
-->
<!--2. UNCOMMENT BELOW AND SET CREDENTIALS ACCORDING TO THE PROVIDER TYPE-->
<!--2.1. If "ClientCredsTokenProvider" is set as key provider, uncomment below and
set auth endpoint, client id and secret below-->
<!--
<property>
<name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>Token end point, this can be found through Azure portal</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}</name>
<value>{client secret}</value>
</property>
-->
<!--2.2. If "UserPasswordTokenProvider" is set as key provider, uncomment below and
set auth endpoint, use name and password-->
<!--
<property>
<name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>Token end point, this can be found through Azure portal</description>
</property>
<property>
<name>fs.azure.account.oauth2.user.name.{ABFS_ACCOUNT_NAME}</name>
<value>{user name}</value>
</property>
<property>
<name>fs.azure.account.oauth2.user.password.{ABFS_ACCOUNT_NAME}</name>
<value>{user password}</value>
</property>
-->
<!--2.3. If "MsiTokenProvider" is set as key provider, uncomment below and
set tenantGuid and client id.-->
<!--
<property>
<name>fs.azure.account.oauth2.msi.tenant.{ABFS_ACCOUNT_NAME}</name>
<value>{tenantGuid}</value>
<description>msi tenantGuid.</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
-->
<!--2.4. If "RefreshTokenBasedTokenProvider" is set as key provider, uncomment below and
set refresh token and client id.-->
<!--
<property>
<name>fs.azure.account.oauth2.refresh.token.{ABFS_ACCOUNT_NAME}</name>
<value>{refresh token}</value>
<description>refresh token.</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
-->
```
If running tests against an endpoint that uses the URL format
http[s]://[ip]:[port]/[account]/[filesystem] instead of
http[s]://[account][domain-suffix]/[filesystem], please use the following:
```xml
<property>
<name>abfs.account.name</name>
<value>ACCOUNTNAME</value>
<name>fs.azure.abfs.endpoint</name>
<value>{IP}:{PORT}</value>
</property>
<property>
<name>abfs.account.full.name</name>
<value>${abfs.account.name}.dfs.core.windows.net</value>
</property>
<property>
<name>abfs.account.key</name>
<value>SECRETKEY==</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
<value>${abfs.account.key}</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
<value>${abfs.account.key}</value>
</property>
<property>
<name>fs.azure.test.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
<value>${abfs.account.key}</value>
</property>
<property>
<name>fs.azure.test.account.key.ACCOUNTNAME.blob.core.windows.net</name>
<value>${abfs.account.key}</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME</name>
<value>${abfs.account.key}</value>
</property>
<property>
<name>fs.azure.test.account.key.ACCOUNTNAME</name>
<value>${abfs.account.key}</value>
</property>
<property>
<name>fs.azure.test.account.name</name>
<value>${abfs.account.full.name}</value>
</property>
<property>
<name>fs.contract.test.fs.abfs</name>
<value>abfs://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net</value>
<description>Container for contract tests</description>
</property>
<property>
<name>fs.contract.test.fs.abfss</name>
<value>abfss://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net</value>
<description>Container for contract tests</description>
</property>
```
```

22
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
View File

@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
/**
* Helper class to create WASB file systems backed by either a mock in-memory
@ -58,11 +59,14 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
private static final Logger LOG = LoggerFactory.getLogger(
AzureBlobStorageTestAccount.class);
private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name";
public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name";
public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net";
public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net";
public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net";
public static final String MOCK_CONTAINER_NAME = "mockContainer";
public static final String WASB_AUTHORITY_DELIMITER = "@";
public static final String WASB_SCHEME = "wasb";
@ -379,7 +383,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
containerName);
container.create();
String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
String accountName = verifyWasbAccountNameInConfig(conf);
// Ensure that custom throttling is disabled and tolerate concurrent
// out-of-band appends.
@ -525,7 +529,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
static CloudStorageAccount createTestAccount(Configuration conf)
throws URISyntaxException, KeyProviderException {
String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
String testAccountName = verifyWasbAccountNameInConfig(conf);
if (testAccountName == null) {
LOG.warn("Skipping live Azure test because of missing test account");
return null;
@ -570,16 +574,16 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
String containerName = useContainerSuffixAsContainerName
? containerNameSuffix
: String.format(
"wasbtests-%s-%tQ%s",
"wasbtests-%s-%s%s",
System.getProperty("user.name"),
new Date(),
UUID.randomUUID().toString(),
containerNameSuffix);
container = account.createCloudBlobClient().getContainerReference(
containerName);
if (createOptions.contains(CreateOptions.CreateContainer)) {
container.createIfNotExists();
}
String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
String accountName = verifyWasbAccountNameInConfig(conf);
if (createOptions.contains(CreateOptions.UseSas)) {
String sas = generateSAS(container,
createOptions.contains(CreateOptions.Readonly));
@ -741,7 +745,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
CloudBlobClient blobClient = account.createCloudBlobClient();
// Capture the account URL and the account name.
String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
String accountName = verifyWasbAccountNameInConfig(conf);
configureSecureModeTestSettings(conf);
@ -814,7 +818,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
CloudBlobClient blobClient = account.createCloudBlobClient();
// Capture the account URL and the account name.
String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
String accountName = verifyWasbAccountNameInConfig(conf);
configureSecureModeTestSettings(conf);

3
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
View File

@ -29,6 +29,7 @@ import com.microsoft.azure.storage.CloudStorageAccount;
import org.junit.Test;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
/**
* Test for error messages coming from SDK.
@ -46,7 +47,7 @@ public class ITestFileSystemOperationExceptionMessage
AzureBlobStorageTestAccount.createTestAccount(conf);
AzureTestUtils.assume("No test account", account != null);
String testStorageAccount = conf.get("fs.azure.test.account.name");
String testStorageAccount = verifyWasbAccountNameInConfig(conf);
conf = new Configuration();
conf.set("fs.AbstractFileSystem.wasb.impl",
"org.apache.hadoop.fs.azure.Wasb");

26
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
View File

@ -499,32 +499,6 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
}
}
@Test
public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception {
try {
testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI("wasbs", authority, null, null, null);
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
FileSystem fs = FileSystem.get(conf);
assertTrue(fs instanceof NativeAzureFileSystem);
assertEquals("wasbs", fs.getScheme());
// should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified
try{
FileContext.getFileContext(conf).getDefaultFileSystem();
fail("Should've thrown.");
}catch(UnsupportedFileSystemException e){
}
} finally {
testAccount.cleanup();
FileSystem.closeAll();
}
}
@Test
public void testCredentialProviderPathExclusions() throws Exception {
String providerPath =

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
View File

@ -143,12 +143,10 @@ public interface AzureTestConstants {
String ACCOUNT_KEY_PROPERTY_NAME
= "fs.azure.account.key.";
String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
String ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
String SAS_PROPERTY_NAME = "fs.azure.sas.";
String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
String TEST_ACCOUNT_NAME_PROPERTY_NAME
= "fs.azure.test.account.name";
String MOCK_ACCOUNT_NAME
= "mockAccount.blob.core.windows.net";
String MOCK_CONTAINER_NAME = "mockContainer";

18
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
View File

@ -36,10 +36,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX;
import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN;
import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assume.assumeTrue;
/**
* Utilities for the Azure tests. Based on {@code S3ATestUtils}, so
@ -476,4 +480,18 @@ public final class AzureTestUtils extends Assert {
+ KEY_SCALE_TESTS_ENABLED,
enabled);
}
/**
* Check the account name for WASB tests is set correctly and return.
*/
public static String verifyWasbAccountNameInConfig(Configuration conf) {
String accountName = conf.get(ACCOUNT_NAME_PROPERTY_NAME);
if (accountName == null) {
accountName = conf.get(WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN);
}
assumeTrue("Account for WASB is missing or it is not in correct format",
accountName != null && !accountName.endsWith(WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX));
return accountName;
}
}

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java
View File

@ -18,8 +18,8 @@
package org.apache.hadoop.fs.azure.metrics;
import static org.junit.Assert.*;
import org.junit.*;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
public class TestRollingWindowAverage {
/**

73
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
View File

@ -35,7 +35,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@ -45,12 +44,12 @@ import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.io.IOUtils;
import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.assumeTrue;
import static org.hamcrest.CoreMatchers.*;
/**
* Base for AzureBlobFileSystem Integration tests.
@ -58,12 +57,12 @@ import static org.hamcrest.CoreMatchers.*;
* <I>Important: This is for integration tests only.</I>
*/
public abstract class AbstractAbfsIntegrationTest extends
AbstractWasbTestWithTimeout {
AbstractAbfsTestWithTimeout {
private static final Logger LOG =
LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class);
private boolean isEmulator;
private boolean isIPAddress;
private NativeAzureFileSystem wasb;
private AzureBlobFileSystem abfs;
private String abfsScheme;
@ -75,27 +74,28 @@ public abstract class AbstractAbfsIntegrationTest extends
private AuthType authType;
protected AbstractAbfsIntegrationTest() {
fileSystemName = ABFS_TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
configuration = new Configuration();
configuration.addResource(ABFS_TEST_RESOURCE_XML);
this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME);
configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME);
if (accountName == null) {
// check if accountName is set using different config key
accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME);
}
assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME,
accountName != null && !accountName.isEmpty());
authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME
+ accountName, AuthType.SharedKey);
abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME
: FileSystemUriSchemes.ABFS_SECURE_SCHEME;
String accountName = configuration.get(FS_AZURE_TEST_ACCOUNT_NAME, "");
assumeTrue("Not set: " + FS_AZURE_TEST_ACCOUNT_NAME,
!accountName.isEmpty());
assertThat("The key in " + FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
+ " is not bound to an ABFS account",
accountName, containsString("dfs.core.windows.net"));
String fullKey = FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
+ accountName;
if (authType == AuthType.SharedKey) {
assumeTrue("Not set: " + fullKey, configuration.get(fullKey) != null);
String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName;
assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null);
// Update credentials
} else {
String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName;
assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null);
@ -113,7 +113,17 @@ public abstract class AbstractAbfsIntegrationTest extends
this.testUrl = defaultUri.toString();
configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
this.isEmulator = this.configuration.getBoolean(FS_AZURE_EMULATOR_ENABLED, false);
// For testing purposes, an IP address and port may be provided to override
// the host specified in the FileSystem URI. Also note that the format of
// the Azure Storage Service URI changes from
// http[s]://[account][domain-suffix]/[filesystem] to
// http[s]://[ip]:[port]/[account]/[filesystem].
String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) {
this.isIPAddress = true;
} else {
this.isIPAddress = false;
}
}
@ -122,13 +132,22 @@ public abstract class AbstractAbfsIntegrationTest extends
//Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
createFileSystem();
if (!isEmulator && authType == AuthType.SharedKey) {
if (!isIPAddress && authType == AuthType.SharedKey) {
final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
final AzureNativeFileSystemStore azureNativeFileSystemStore =
new AzureNativeFileSystemStore();
// update configuration with wasb credentials
String accountNameWithoutDomain = accountName.split("\\.")[0];
String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName;
if (configuration.get(keyProperty) == null) {
configuration.set(keyProperty, getAccountKey());
}
azureNativeFileSystemStore.initialize(
wasbUri,
getConfiguration(),
configuration,
new AzureFileSystemInstrumentation(getConfiguration()));
wasb = new NativeAzureFileSystem(azureNativeFileSystemStore);
@ -201,7 +220,9 @@ public abstract class AbstractAbfsIntegrationTest extends
}
protected String getHostName() {
return configuration.get(FS_AZURE_TEST_HOST_NAME);
// READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC
String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
return endPoint.split(":")[0];
}
protected void setTestUrl(String testUrl) {
@ -220,21 +241,21 @@ public abstract class AbstractAbfsIntegrationTest extends
}
protected String getAccountName() {
return configuration.get(FS_AZURE_TEST_ACCOUNT_NAME);
return this.accountName;
}
protected String getAccountKey() {
return configuration.get(
FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
+ getAccountName());
FS_AZURE_ACCOUNT_KEY_PREFIX
+ accountName);
}
protected Configuration getConfiguration() {
return configuration;
}
protected boolean isEmulator() {
return isEmulator;
protected boolean isIPAddress() {
return isIPAddress;
}
protected AuthType getAuthType() {

70
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java Normal file
View File

@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.azurebfs;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TestName;
import org.junit.rules.Timeout;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT;
/**
* Base class for any ABFS test with timeouts & named threads.
* This class does not attempt to bind to Azure.
*/
public class AbstractAbfsTestWithTimeout extends Assert {
/**
* The name of the current method.
*/
@Rule
public TestName methodName = new TestName();
/**
* Set the timeout for every test.
* This is driven by the value returned by {@link #getTestTimeoutMillis()}.
*/
@Rule
public Timeout testTimeout = new Timeout(getTestTimeoutMillis());
/**
* Name the junit thread for the class. This will overridden
* before the individual test methods are run.
*/
@BeforeClass
public static void nameTestThread() {
Thread.currentThread().setName("JUnit");
}
/**
* Name the thread to the current test method.
*/
@Before
public void nameThread() {
Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
}
/**
* Override point: the test timeout in milliseconds.
* @return a timeout in milliseconds
*/
protected int getTestTimeoutMillis() {
return TEST_TIMEOUT;
}
}

2
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
View File

@ -69,7 +69,7 @@ public class ITestAzureBlobFileSystemBackCompat extends
private String getBlobConnectionString() {
String connectionString;
if (isEmulator()) {
if (isIPAddress()) {
connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://"
+ this.getHostName() + ":8880/" + this.getAccountName().split("\\.") [0]
+ ";AccountName=" + this.getAccountName().split("\\.")[0]

2
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
View File

@ -50,7 +50,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
LoggerFactory.getLogger(ITestWasbAbfsCompatibility.class);
public ITestWasbAbfsCompatibility() throws Exception {
Assume.assumeFalse("Emulator is not supported", isEmulator());
Assume.assumeFalse("Emulator is not supported", isIPAddress());
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}

3
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
View File

@ -23,6 +23,7 @@ import java.lang.reflect.Field;
import org.apache.commons.codec.Charsets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.BooleanConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
@ -98,7 +99,7 @@ public class TestAbfsConfigurationFieldsValidation {
this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
Configuration configuration = new Configuration();
configuration.addResource("azure-bfs-test.xml");
configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);
configuration.set(INT_KEY, "1234565");
configuration.set(LONG_KEY, "4194304");
configuration.set(STRING_KEY, "stringValue");

13
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
View File

@ -22,10 +22,9 @@ package org.apache.hadoop.fs.azurebfs.constants;
* Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file.
*/
public final class TestConfigurationKeys {
public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name";
public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key.";
public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name";
public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port";
public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name";
public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name";
public static final String FS_AZURE_ACCOUNT_KEY_PREFIX = "fs.azure.account.key.";
public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id";
@ -34,9 +33,9 @@ public final class TestConfigurationKeys {
public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_ID = "fs.azure.account.oauth2.reader.client.id";
public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET = "fs.azure.account.oauth2.reader.client.secret";
public static final String ABFS_TEST_RESOURCE_XML = "azure-bfs-test.xml";
public static final String ABFS_TEST_CONTAINER_PREFIX = "abfs-testcontainer-";
public static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
public static final String TEST_CONTAINER_PREFIX = "abfs-testcontainer-";
public static final int TEST_TIMEOUT = 10 * 60 * 1000;
private TestConfigurationKeys() {}
}

3
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java
View File

@ -26,6 +26,7 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.junit.Assume;
/**
* Bind ABFS contract tests to the Azure test setup/teardown.
@ -42,6 +43,8 @@ public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest {
if (useExistingFileSystem) {
Configuration configuration = getConfiguration();
String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI);
Assume.assumeTrue("Contract tests are skipped because of missing config property :"
+ TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI, testUrl != null);
if (getAuthType() != AuthType.SharedKey) {
testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME);

1
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
View File

@ -32,7 +32,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.M
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE;
import static org.junit.Assert.assertEquals;
/**
* Test configuration validators.

49
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java → hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java
View File

@ -22,56 +22,59 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import org.junit.Assume;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS_TEST_CONTAINER_PREFIX;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX;
/**
* If unit tests were interrupted and crushed accidentally, the test containers won't be deleted.
* In that case, dev can use this tool to list and delete all test containers.
* By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-"
*/
public class CleanUpAbfsTestContainer extends AbstractAbfsIntegrationTest{
/*
* Some Utils for ABFS tests.
* */
public final class AbfsTestUtils extends AbstractAbfsIntegrationTest{
private static final Logger LOG =
LoggerFactory.getLogger(AbfsTestUtils.class);
/**
* If unit tests were interrupted and crushed accidentally, the test containers won't be deleted.
* In that case, dev can use this tool to list and delete all test containers.
* By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-"
*/
public CleanUpAbfsTestContainer() {
public void checkContainers() throws Throwable {
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
@Test
public void testEnumContainers() throws Throwable {
int count = 0;
CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
Iterable<CloudBlobContainer> containers
= blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX);
= blobClient.listContainers(TEST_CONTAINER_PREFIX);
for (CloudBlobContainer container : containers) {
count++;
System.out.println(String.format("Container %s URI %s",
LOG.info("Container {}, URI {}",
container.getName(),
container.getUri()));
container.getUri());
}
System.out.println(String.format("Found %d test containers", count));
LOG.info("Found {} test containers", count);
}
@Test
public void testDeleteContainers() throws Throwable {
public void deleteContainers() throws Throwable {
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
int count = 0;
CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
Iterable<CloudBlobContainer> containers
= blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX);
= blobClient.listContainers(TEST_CONTAINER_PREFIX);
for (CloudBlobContainer container : containers) {
System.out.println(String.format("Container %s URI %s",
LOG.info("Container {} URI {}",
container.getName(),
container.getUri()));
container.getUri());
if (container.deleteIfExists()) {
count++;
}
}
System.out.println(String.format("Deleted %s test containers", count));
LOG.info("Deleted {} test containers", count);
}
}

12
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java
View File

@ -38,11 +38,11 @@ public final class TestUriUtils {
@Test
public void testExtractRawAccountName() throws Exception {
Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.core.windows.net"));
Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.preprod.core.windows.net"));
Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net"));
Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(""));
Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(null));
Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net"));
Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.core.windows.net"));
Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.preprod.core.windows.net"));
Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net"));
Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(""));
Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(null));
Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net"));
}
}

188
hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml
View File

@ -1,188 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<!--
<property>
<name>fs.azure.test.account.name</name>
<value>{YOURACCOUNT}</value>
</property>
<property>
<name>fs.azure.account.auth.type.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{AUTH TYPE}</value>
<description>The auth type can be : SharedKey, OAuth, Custom. By default "SharedKey" is used.</description>
</property>
<property>
<name>fs.contract.test.fs.abfs</name>
<value>abfs://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value>
<description>The name of the azure file system for testing.</description>
</property>
<property>
<name>fs.contract.test.fs.abfss</name>
<value>abfss://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value>
<description>The name of the azure file system for testing.</description>
</property>
-->
<!--If auth type is "SharedKey", provide SharedKey credentials below -->
<!--
<property>
<name>fs.azure.account.key.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{ACCOUNTKEY}</value>
</property>
<property>
<name>fs.azure.test.account.key.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{ACCOUNTKEY}</value>
</property>
-->
<!--If auth type is "OAuth", set below properties, AAD client and tenant related properties can be obtained through Azure Portal-->
<!--
<property>
<name>fs.azure.account.oauth.provider.type.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
<description>The full name of token provider class name.</description>
</property>
-->
<!--If "ClientCredsTokenProvider" is set as key provider, set auth endpoint, client id and secret below-->
<!--
<property>
<name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>Token end point, this can be found through Azure portal</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.secret.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{client secret}</value>
</property>
-->
<!--If "UserPasswordTokenProvider" is set as key provider, set auth endpoint, use name and password below-->
<!--
<property>
<name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>Token end point, this can be found through Azure portal</description>
</property>
<property>
<name>fs.azure.account.oauth2.user.name.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{user name}</value>
</property>
<property>
<name>fs.azure.account.oauth2.user.password.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{user password}</value>
</property>
-->
<!--If "MsiTokenProvider" is set as key provider, set tenantGuid and client id below-->
<!--
<property>
<name>fs.azure.account.oauth2.msi.tenant.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{tenantGuid}</value>
<description>msi tenantGuid.</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
-->
<!--If "RefreshTokenBasedTokenProvider" is set as key provider, set refresh token and client id below-->
<!--
<property>
<name>fs.azure.account.oauth2.refresh.token.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{refresh token}</value>
<description>refresh token.</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
-->
<!--Below four configure properties are provided for Blob Data Contributor a
nd Blob Data Reader OAuth access test "ITestAzureBlobFileSystemOauth",
using"ClientCredsTokenProvider" as key
provider. These are for test only.-->
<!--
<property>
<name>fs.azure.account.oauth2.contributor.client.id</name>
<value>{ID of client which has Data Contributor role}</value>
</property>
<property>
<name>fs.azure.account.oauth2.contributor.client.secret</name>
<value>{secret of client which has Data Contributor role}</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.id</name>
<value>{ID of client which has Data Reader role}</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.secret</name>
<value>{Secret of client which has Data Reader role}</value>
</property>
-->
<!-- Save above configuration properties in a separate file named -->
<!-- azure-bfs-auth-keys.xml in the same directory as this file. -->
<!-- DO NOT ADD azure-bfs-auth-keys.xml TO REVISION CONTROL. The keys to your -->
<!-- Azure Storage account are a secret and must not be shared. -->
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-bfs-auth-keys.xml">
<fallback />
</include>
<!--<property>-->
<!--<name>fs.azure.test.host.name</name>-->
<!--<value>{HOSTNAME}</value>-->
<!--</property>-->
<!--<property>-->
<!--<name>fs.azure.test.host.port</name>-->
<!--<value>{PORT}</value>-->
<!--</property>-->
<!--<property>-->
<!--<name>fs.azure.abfs.emulator.enabled</name>-->
<!--<value>false</value>-->
<!--</property>-->
</configuration>

56
hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
View File

@ -16,49 +16,39 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<!-- For tests against live azure, provide the following account information -->
<!--==================== WASB CONFIGURATION ====================-->
<!-- For tests against azure-emulator -->
<property>
<name>fs.azure.test.emulator</name>
<value>false</value>
</property>
<!--
<property>
<name>fs.azure.test.account.name</name>
<value>{ACCOUNTNAME}.blob.core.windows.net</value>
</property>
<property>
<name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
<value>{ACCOUNTKEY}</value>
</property>
-->
<property>
<name>fs.azure.secure.mode</name>
<value>false</value>
</property>
<!-- uncomment to test in Azure secure mode -->
<!--
<property>
<name>fs.azure.secure.mode</name>
<value>true</value>
</property>
-->
<!--==================== ABFS CONFIGURATION ====================-->
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
<!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
<property>
<name>fs.azure.user.agent.prefix</name>
<value>MSFT</value>
</property>
<!-- Save the above configuration properties in a separate file named -->
<!-- azure-auth-keys.xml in the same directory as this file. -->
<!-- DO NOT ADD azure-auth-keys.xml TO REVISION CONTROL. The keys to your -->
<!-- Azure Storage account are a secret and must not be shared. -->
<!--====================== IMPORTANT!! ========================-->
<!--WARNING!!
PLEASE CREATE A FILE NAMED "azure-auth-keys.xml"
AND SEE "site/markdown/testing_azure.md", THEN
STORE THE CONFIGURATION PROPERTIES WITHIN IT.
TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
IT IN PATCHES OR COMMITS. -->
<!--=============================================================-->
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
<fallback />
</include>
<!-- For tests against azure-emulator -->
<!--
<property>
<name>fs.azure.test.emulator</name>
<value>true</value>
</property>
-->
<property>
<name>fs.AbstractFileSystem.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.Wasb</value>
</property>
</configuration>