HADOOP-17191. ABFS: Run the tests with various combinations of configurations and publish a consolidated results (#2597)

Contributed by Bilahari T H and Sumangala Patki
This commit is contained in:
sumangala-patki 2021-03-10 23:55:41 +05:30 committed by GitHub
parent 0e82d42d2c
commit cdaa64458d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 529 additions and 1 deletions

View File

@ -1,2 +1,5 @@
.checkstyle
bin/
bin/
src/test/resources/combinationConfigFiles
src/test/resources/abfs-combination-test-configs.xml
dev-support/testlogs

View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# shellcheck disable=SC2034
# unused variables are global in nature and used in testsupport.sh
set -eo pipefail
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# shellcheck disable=SC1091
. dev-support/testrun-scripts/testsupport.sh
begin
### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
combination=HNS-OAuth
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
"fs.azure.account.auth.type")
values=("{account name}.dfs.core.windows.net" "true" "OAuth")
generateconfigs
combination=HNS-SharedKey
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
generateconfigs
combination=NonHNS-SharedKey
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
generateconfigs
### DO NOT EDIT THE LINES BELOW.
runtests "$@"

View File

@ -0,0 +1,242 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
testresourcesdir=src/test/resources
combconfsdir=$testresourcesdir/combinationConfigFiles
combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
logdir=dev-support/testlogs
testresultsregex="Results:(\n|.)*?Tests run:"
testresultsfilename=
starttime=
threadcount=
defaultthreadcount=8
properties=
values=
validate() {
if [ -z "$threadcount" ] ; then
threadcount=$defaultthreadcount
fi
numberegex='^[0-9]+$'
if ! [[ $threadcount =~ $numberegex ]] ; then
echo "Exiting. The script param (threadcount) should be a number"
exit -1
fi
if [ -z "$combination" ]; then
echo "Exiting. combination cannot be empty"
exit -1
fi
propertiessize=${#properties[@]}
valuessize=${#values[@]}
if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
exit -1
fi
for filename in "${combinations[@]}"; do
if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
exit -1
fi
done
}
checkdependencies() {
if ! [ "$(command -v pcregrep)" ]; then
echo "Exiting. pcregrep is required to run the script."
exit -1
fi
if ! [ "$(command -v xmlstarlet)" ]; then
echo "Exiting. xmlstarlet is required to run the script."
exit -1
fi
}
cleancombinationconfigs() {
rm -rf $combconfsdir
mkdir -p $combconfsdir
}
generateconfigs() {
combconffile="$combconfsdir/$combination.xml"
rm -rf "$combconffile"
cat > "$combconffile" << ENDOFFILE
<configuration>
</configuration>
ENDOFFILE
propertiessize=${#properties[@]}
valuessize=${#values[@]}
if [ "$propertiessize" -ne "$valuessize" ]; then
echo "Exiting. Number of properties and values differ for $combination"
exit -1
fi
for ((i = 0; i < propertiessize; i++)); do
key=${properties[$i]}
val=${values[$i]}
changeconf "$key" "$val"
done
formatxml "$combconffile"
}
formatxml() {
xmlstarlet fo -s 2 "$1" > "$1.tmp"
mv "$1.tmp" "$1"
}
setactiveconf() {
if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
exit -1
fi
rm -rf $combtestfile
cat > $combtestfile << ENDOFFILE
<configuration>
</configuration>
ENDOFFILE
xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
formatxml $combtestfile
}
changeconf() {
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
then
echo "Exiting. Changing config property failed."
exit -1
fi
}
summary() {
{
echo ""
echo "$combination"
echo "========================"
pcregrep -M "$testresultsregex" "$testlogfilename"
} >> "$testresultsfilename"
printf "\n----- Test results -----\n"
pcregrep -M "$testresultsregex" "$testlogfilename"
secondstaken=$((ENDTIME - STARTTIME))
mins=$((secondstaken / 60))
secs=$((secondstaken % 60))
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
echo "Find test logs for the combination ($combination) in: $testlogfilename"
echo "Find consolidated test results in: $testresultsfilename"
echo "----------"
}
init() {
checkdependencies
if ! mvn clean install -DskipTests
then
echo ""
echo "Exiting. Build failed."
exit -1
fi
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
mkdir -p "$logdir"
testresultsfilename="$logdir/$starttime/Test-Results.txt"
if [[ -z "$combinations" ]]; then
combinations=( $( ls $combconfsdir/*.xml ))
fi
}
runtests() {
parseoptions "$@"
validate
if [ -z "$starttime" ]; then
init
fi
shopt -s nullglob
for combconffile in "${combinations[@]}"; do
STARTTIME=$(date +%s)
combination=$(basename "$combconffile" .xml)
mkdir -p "$logdir/$starttime"
testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
printf "\nRunning the combination: %s..." "$combination"
setactiveconf
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
ENDTIME=$(date +%s)
summary
done
}
begin() {
cleancombinationconfigs
}
parseoptions() {
runactivate=0
runtests=0
while getopts ":c:a:t:" option; do
case "${option}" in
a)
if [[ "$runactivate" -eq "1" ]]; then
echo "-a Option is not multivalued"
exit 1
fi
runactivate=1
combination=$(basename "$OPTARG" .xml)
;;
c)
runtests=1
combination=$(basename "$OPTARG" .xml)
combinations+=("$combination")
;;
t)
threadcount=$OPTARG
;;
*|?|h)
# shellcheck disable=SC2128
if [[ -z "$combinations" ]]; then
combinations=( $( ls $combconfsdir/*.xml ))
fi
combstr=""
for combconffile in "${combinations[@]}"; do
combname=$(basename "$combconffile" .xml)
combstr="${combname}, ${combstr}"
done
combstr=${combstr:0:-2}
echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
echo ""
echo "Where:"
echo " -a COMBINATION_NAME Specify the combination name which needs to be activated."
echo " Configured combinations: ${combstr}"
echo " -c COMBINATION_NAME Specify the combination name for test runs"
echo " -t THREAD_COUNT Specify the thread count"
exit 1
;;
esac
done
if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
exit 1
fi
if [[ "$runactivate" -eq "1" ]]; then
setactiveconf
exit 0
fi
}

View File

@ -592,6 +592,61 @@ with the Hadoop Distributed File System permissions model when hierarchical
namespace is enabled for the storage account. Furthermore, the metadata and data
produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa.
## Generating test run configurations and test triggers over various config combinations
To simplify the testing across various authentication and features combinations
that are mandatory for a PR, script `dev-support/testrun-scripts/runtests.sh`
should be used. Once the script is updated with relevant config settings for
various test combinations, it will:
1. Auto-generate configs specific to each test combinations
2. Run tests for all combinations
3. Summarize results across all the test combination runs.
As a pre-requiste step, fill config values for test accounts and credentials
needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
and rename as `src/test/resources/azure-auth-keys.xml`.
**To add a new test combination:** Templates for mandatory test combinations
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
If a new one needs to be added, add a combination set within
`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
and
1. Provide a new combination name
2. Update properties and values array which need to be effective for the test
combination
3. Call generateconfigs
**To run PR validation:** Running command
* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
each of the combinations defined and run tests for all the combinations.
* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
combinations can be provided with -c option. If combinations are provided
with -c option, tests for only those combinations will be run.
**Test logs:** Test runs will create a folder within dev-support/testlogs to
save the test logs. Folder name will be the test start timestamp. The mvn verify
command line logs for each combination will be saved into a file as
Test-Logs-$combination.txt into this folder. In case of any failures, this file
will have the failure exception stack. At the end of the test run, the
consolidated results of all the combination runs will be saved into a file as
Test-Results.log in the same folder. When run for PR validation, the
consolidated test results needs to be pasted into the PR comment section.
**To generate config for use in IDE:** Running command with -a (activate) option
`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
the effective config relevant for the specific test combination. Hence the same
config files used by the mvn test runs can be used for IDE without any manual
updates needed within config file.
**Other command line options:**
* -a <COMBINATION_NAME> Specify the combination name which needs to be
activated. This is to be used to generate config for use in IDE.
* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
config is specified, tests for only the specified combinations will run. All
combinations of tests will be running if this config is not specified.
* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
In order to test ABFS, please add the following configuration to your
`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
compatibility tests which require WASB credentials, in addition to the ABFS

View File

@ -0,0 +1,174 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<!--====================== IMPORTANT!! ========================-->
<!--
1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
2. UPDATE runtests.sh
A. "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
B. PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
DIFFERENT VARIANTS OF CONFIGS.
3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
-->
<!--=============== Auth type ===============-->
<property>
<name>fs.azure.account.auth.type</name>
<value>SharedKey</value>
</property>
<!--=============== Auth related accounts ===============-->
<!-- This set of configs needs to be provided for all the accounts with which
the tests needs to be ran. -->
<property>
<name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
<value>{ACCOUNT_ACCESS_KEY}</value>
<description>Account access key</description>
</property>
<property>
<name>
fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
<description>OAuth token provider implementation class</description>
</property>
<property>
<name>
fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>Token end point, this can be found through Azure portal
</description>
</property>
<property>
<name>
fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
<property>
<name>
fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>{client secret}</value>
<description>AAD client secret</description>
</property>
<!--=============== Configs for hadoop contract tests for ABFS ===============-->
<property>
<name>fs.contract.test.fs.abfs</name>
<value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
</property>
<property>
<name>fs.contract.test.fs.abfss</name>
<value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
</property>
<!--=============== WASB Configs ===============-->
<property>
<name>fs.azure.wasb.account.name</name>
<value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
</property>
<property>
<name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
<value>WASB account key</value>
</property>
<property>
<name>fs.contract.test.fs.wasb</name>
<value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
</value>
</property>
<!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
<property>
<name>fs.azure.account.oauth2.contributor.client.id</name>
<value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
</property>
<property>
<name>fs.azure.account.oauth2.contributor.client.secret</name>
<value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.id</name>
<value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.secret</name>
<value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
</property>
<!--=========================== FOR CheckAccess =========================-->
<!-- To run ABFS CheckAccess tests, you must register an app, with no role
assignments, and set the configuration discussed below:
1) Register a new app with no RBAC
2) As part of the test configs you need to provide the guid for the above
created app. Please follow the below steps to fetch the guid.
a) Get an access token with the above created app. Please refer the
following documentation for the same. https://docs.microsoft
.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
b) Decode the token fetched with the above step. You may use https
://jwt.ms/ to decode the token
d) The oid field in the decoded string is the guid.
3) Set the following configurations:
-->
<property>
<name>fs.azure.account.test.oauth2.client.id</name>
<value>{client id}</value>
<description>The client id(app id) for the app created on step 1
</description>
</property>
<property>
<name>fs.azure.account.test.oauth2.client.secret</name>
<value>{client secret}</value>
<description>
The client secret(application's secret) for the app created on step 1
</description>
</property>
<property>
<name>fs.azure.check.access.testuser.guid</name>
<value>{guid}</value>
<description>The guid fetched on step 2</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
.windows.net</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>
Token end point. This can be found through Azure portal. As part of CheckAccess
test cases. The access will be tested for an FS instance created with the
above mentioned client credentials. So this configuration is necessary to
create the test FS instance.
</description>
</property>
<!--========== Append blob related configs ===========-->
<property>
<name>fs.azure.test.appendblob.enabled</name>
<value>false</value>
<description>If made true, tests will be running under the assumption that
append blob is enabled and the root directory and contract test root
directory will be part of the append blob directories.
</description>
</property>
</configuration>

View File

@ -66,4 +66,8 @@
<fallback />
</include>
<include xmlns="http://www.w3.org/2001/XInclude" href="abfs-combination-test-configs.xml">
<fallback />
</include>
</configuration>