Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13929. ADLS connector should not check in contract-test-options.xml. (John Zhuge via lei)

This commit is contained in:
Lei Xu 2017-02-13 13:33:13 -08:00
parent 4ed33e9ca3
commit 71c23c9fc9
6 changed files with 39 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.gitignore vendored
View File

@ -17,6 +17,10 @@ target
build
dependency-reduced-pom.xml
# Filesystem contract test options and credentials
auth-keys.xml
azure-auth-keys.xml
# External tool builders
*/.externalToolBuilders
*/maven-eclipse.xml
@ -24,8 +28,6 @@ dependency-reduced-pom.xml
hadoop-common-project/hadoop-kms/downloads/
hadoop-hdfs-project/hadoop-hdfs/downloads
hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml
hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/dist
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tmp
@ -41,10 +43,4 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/testem.log
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/dist
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/tmp
yarnregistry.pdf
hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
patchprocess/
hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml

2
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
View File

@ -72,7 +72,7 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.*;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class AdlFileSystem extends FileSystem {
static final String SCHEME = "adl";
public static final String SCHEME = "adl";
static final int DEFAULT_PORT = 443;
private URI uri;
private String userName;

4
hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
View File

@ -224,7 +224,9 @@ commands demonstrate access to a storage account named `youraccount`.
## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
A selection of tests can run against the Azure Data Lake Storage. To run tests against Adl storage. Please configure contract-test-options.xml with Adl account information mentioned in the above sections. Also turn on contract test execution flag to trigger tests against Azure Data Lake Storage.
A selection of tests can run against the Azure Data Lake Storage. To run these
tests, please create `src/test/resources/auth-keys.xml` with Adl account
information mentioned in the above sections and the following properties.
<property>
<name>dfs.adl.test.contract.enable</name>

42
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
View File

@ -21,6 +21,8 @@ package org.apache.hadoop.fs.adl.live;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.util.ReflectionUtils;
import java.io.IOException;
import java.net.URI;
@ -30,15 +32,19 @@ import java.net.URISyntaxException;
* Configure Adl storage file system.
*/
public final class AdlStorageConfiguration {
static final String CONTRACT_XML = "adls.xml";
private static final String CONTRACT_ENABLE_KEY =
"dfs.adl.test.contract.enable";
private static final boolean CONTRACT_ENABLE_DEFAULT = false;
private static final String TEST_CONFIGURATION_FILE_NAME =
"contract-test-options.xml";
private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME =
"adls.xml";
private static final String KEY_FILE_SYSTEM_IMPL = "fs.contract.test.fs";
private static final String KEY_FILE_SYSTEM = "test.fs.adl.name";
private static final String FILE_SYSTEM_KEY =
String.format("test.fs.%s.name", AdlFileSystem.SCHEME);
private static final String FILE_SYSTEM_IMPL_KEY =
String.format("fs.%s.impl", AdlFileSystem.SCHEME);
private static final Class<?> FILE_SYSTEM_IMPL_DEFAULT =
AdlFileSystem.class;
private static boolean isContractTestEnabled = false;
private static Configuration conf = null;
@ -48,8 +54,7 @@ public final class AdlStorageConfiguration {
public synchronized static Configuration getConfiguration() {
Configuration newConf = new Configuration();
newConf.addResource(TEST_CONFIGURATION_FILE_NAME);
newConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME);
newConf.addResource(CONTRACT_XML);
return newConf;
}
@ -58,7 +63,8 @@ public final class AdlStorageConfiguration {
conf = getConfiguration();
}
isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false);
isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY,
CONTRACT_ENABLE_DEFAULT);
return isContractTestEnabled;
}
@ -72,23 +78,15 @@ public final class AdlStorageConfiguration {
return null;
}
String fileSystem = conf.get(KEY_FILE_SYSTEM);
String fileSystem = conf.get(FILE_SYSTEM_KEY);
if (fileSystem == null || fileSystem.trim().length() == 0) {
throw new IOException("Default file system not configured.");
}
String fileSystemImpl = conf.get(KEY_FILE_SYSTEM_IMPL);
if (fileSystemImpl == null || fileSystemImpl.trim().length() == 0) {
throw new IOException(
"Configuration " + KEY_FILE_SYSTEM_IMPL + "does not exist.");
}
FileSystem fs = null;
try {
fs = (FileSystem) Class.forName(fileSystemImpl).newInstance();
} catch (Exception e) {
throw new IOException("Could not instantiate the filesystem.");
}
fs.initialize(new URI(conf.get(KEY_FILE_SYSTEM)), conf);
Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
FILE_SYSTEM_IMPL_DEFAULT);
FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
fs.initialize(new URI(fileSystem), conf);
return fs;
}
}

11
hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
View File

@ -12,6 +12,17 @@
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<!--
To run live tests:
# Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
# Add property test.fs.adl.name to point to an ADLS filesystem URL
# Add property dfs.adl.test.contract.enable with value set to true
# Add the credentials for the service you are testing against
-->
<include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
<fallback/>
</include>
<property>
<name>fs.contract.test.root-tests-enabled</name>
<value>true</value>

61
hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
View File

@ -1,61 +0,0 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<property>
<name>dfs.adls.oauth2.refresh.url</name>
<value>
</value>
</property>
<property>
<name>dfs.adls.oauth2.credential</name>
<value></value>
</property>
<property>
<name>dfs.adls.oauth2.client.id</name>
<value></value>
</property>
<property>
<name>dfs.adls.oauth2.access.token.provider.type</name>
<value>ClientCredential</value>
<description>
Supported provider type:
"ClientCredential" : Client id and client credentials(Provided
through configuration file) flow.
"RefreshToken" : Client id and refresh token(Provided
through configuration file)flow.
"Custom" : Custom AAD token management.
</description>
</property>
<property>
<name>dfs.adl.test.contract.enable</name>
<value>false</value>
</property>
<property>
<name>test.fs.adl.name</name>
<value></value>
</property>
<property>
<name>fs.contract.test.fs</name>
<value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
</property>
</configuration>