diff --git a/.gitignore b/.gitignore index eb1fc967416..2a82ba73461 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ target build +# Filesystem contract test options and credentials +auth-keys.xml +azure-auth-keys.xml + # External tool builders */.externalToolBuilders */maven-eclipse.xml @@ -22,12 +26,6 @@ build hadoop-common-project/hadoop-kms/downloads/ hadoop-hdfs-project/hadoop-hdfs/downloads hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads -hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml -hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox yarnregistry.pdf -hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml -hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml -hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml -hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml patchprocess/ diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 3d410252456..303b7bcbdfc 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -72,7 +72,7 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.*; @InterfaceAudience.Public @InterfaceStability.Evolving public class AdlFileSystem extends FileSystem { - static final String SCHEME = "adl"; + public static final String SCHEME = "adl"; static final int DEFAULT_PORT = 443; private URI uri; private String userName; diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index ced5cff88e3..5037db61f6b 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -224,7 +224,9 @@ commands demonstrate access to a storage account named `youraccount`. ## Testing the azure-datalake-store Module The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage. -A selection of tests can run against the Azure Data Lake Storage. To run tests against Adl storage. Please configure contract-test-options.xml with Adl account information mentioned in the above sections. Also turn on contract test execution flag to trigger tests against Azure Data Lake Storage. +A selection of tests can run against the Azure Data Lake Storage. To run these +tests, please create `src/test/resources/auth-keys.xml` with Adl account +information mentioned in the above sections and the following properties. dfs.adl.test.contract.enable diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java index f7f0b710e13..7d6153d5c26 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs.adl.live; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.adl.AdlFileSystem; +import org.apache.hadoop.util.ReflectionUtils; import java.io.IOException; import java.net.URI; @@ -30,15 +32,19 @@ import java.net.URISyntaxException; * Configure Adl storage file system. */ public final class AdlStorageConfiguration { + static final String CONTRACT_XML = "adls.xml"; + private static final String CONTRACT_ENABLE_KEY = "dfs.adl.test.contract.enable"; + private static final boolean CONTRACT_ENABLE_DEFAULT = false; - private static final String TEST_CONFIGURATION_FILE_NAME = - "contract-test-options.xml"; - private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME = - "adls.xml"; - private static final String KEY_FILE_SYSTEM_IMPL = "fs.contract.test.fs"; - private static final String KEY_FILE_SYSTEM = "test.fs.adl.name"; + private static final String FILE_SYSTEM_KEY = + String.format("test.fs.%s.name", AdlFileSystem.SCHEME); + + private static final String FILE_SYSTEM_IMPL_KEY = + String.format("fs.%s.impl", AdlFileSystem.SCHEME); + private static final Class FILE_SYSTEM_IMPL_DEFAULT = + AdlFileSystem.class; private static boolean isContractTestEnabled = false; private static Configuration conf = null; @@ -48,8 +54,7 @@ public final class AdlStorageConfiguration { public synchronized static Configuration getConfiguration() { Configuration newConf = new Configuration(); - newConf.addResource(TEST_CONFIGURATION_FILE_NAME); - newConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME); + newConf.addResource(CONTRACT_XML); return newConf; } @@ -58,7 +63,8 @@ public final class AdlStorageConfiguration { conf = getConfiguration(); } - isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false); + isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, + CONTRACT_ENABLE_DEFAULT); return isContractTestEnabled; } @@ -72,23 +78,15 @@ public final class AdlStorageConfiguration { return null; } - String fileSystem = conf.get(KEY_FILE_SYSTEM); + String fileSystem = conf.get(FILE_SYSTEM_KEY); if (fileSystem == null || fileSystem.trim().length() == 0) { throw new IOException("Default file system not configured."); } - String fileSystemImpl = conf.get(KEY_FILE_SYSTEM_IMPL); - if (fileSystemImpl == null || fileSystemImpl.trim().length() == 0) { - throw new IOException( - "Configuration " + KEY_FILE_SYSTEM_IMPL + "does not exist."); - } - FileSystem fs = null; - try { - fs = (FileSystem) Class.forName(fileSystemImpl).newInstance(); - } catch (Exception e) { - throw new IOException("Could not instantiate the filesystem."); - } - fs.initialize(new URI(conf.get(KEY_FILE_SYSTEM)), conf); + Class clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY, + FILE_SYSTEM_IMPL_DEFAULT); + FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf); + fs.initialize(new URI(fileSystem), conf); return fs; } } diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml index 76a78b0129e..5bbdd6fbb86 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml @@ -12,6 +12,17 @@ limitations under the License. See accompanying LICENSE file. --> + + + + + fs.contract.test.root-tests-enabled true diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml deleted file mode 100644 index 646922ecbb1..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - dfs.adls.oauth2.refresh.url - - - - - - dfs.adls.oauth2.credential - - - - - dfs.adls.oauth2.client.id - - - - - dfs.adls.oauth2.access.token.provider.type - ClientCredential - - Supported provider type: - - "ClientCredential" : Client id and client credentials(Provided - through configuration file) flow. - - "RefreshToken" : Client id and refresh token(Provided - through configuration file)flow. - - "Custom" : Custom AAD token management. - - - - - dfs.adl.test.contract.enable - false - - - - test.fs.adl.name - - - - - fs.contract.test.fs - org.apache.hadoop.fs.adl.AdlFileSystem - -