HADOOP-17767. ABFS: Update test scripts (#3124)
Contributed by Sneha Vijayarajan
This commit is contained in:
parent
9adf0ca089
commit
a996d889ec
|
@ -1,5 +1,6 @@
|
|||
.checkstyle
|
||||
bin/
|
||||
src/test/resources/combinationConfigFiles
|
||||
src/test/resources/abfs-combination-test-configs.xml
|
||||
dev-support/testlogs
|
||||
src/test/resources/accountSettings/*
|
||||
!src/test/resources/accountSettings/accountName_settings.xml.template
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# shellcheck disable=SC2034
|
||||
# unused variables are global in nature and used in testsupport.sh
|
||||
|
||||
test
|
||||
set -eo pipefail
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
|
@ -22,36 +22,154 @@ set -eo pipefail
|
|||
|
||||
# shellcheck disable=SC1091
|
||||
. dev-support/testrun-scripts/testsupport.sh
|
||||
init
|
||||
|
||||
begin
|
||||
resourceDir=src/test/resources/
|
||||
logdir=dev-support/testlogs/
|
||||
azureTestXml=azure-auth-keys.xml
|
||||
azureTestXmlPath=$resourceDir$azureTestXml
|
||||
processCount=8
|
||||
|
||||
### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
|
||||
### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
|
||||
## SECTION: TEST COMBINATION METHODS
|
||||
|
||||
runHNSOAuthTest()
|
||||
{
|
||||
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||
PROPERTIES=("fs.azure.account.auth.type")
|
||||
VALUES=("OAuth")
|
||||
triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||
}
|
||||
|
||||
combination=HNS-OAuth
|
||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
|
||||
"fs.azure.account.auth.type")
|
||||
values=("{account name}.dfs.core.windows.net" "true" "OAuth")
|
||||
generateconfigs
|
||||
runHNSSharedKeyTest()
|
||||
{
|
||||
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||
PROPERTIES=("fs.azure.account.auth.type")
|
||||
VALUES=("SharedKey")
|
||||
triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||
}
|
||||
|
||||
combination=AppendBlob-HNS-OAuth
|
||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
|
||||
"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
|
||||
values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
|
||||
generateconfigs
|
||||
runNonHNSSharedKeyTest()
|
||||
{
|
||||
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||
PROPERTIES=("fs.azure.account.auth.type")
|
||||
VALUES=("SharedKey")
|
||||
triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||
}
|
||||
|
||||
combination=HNS-SharedKey
|
||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
|
||||
values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
|
||||
generateconfigs
|
||||
runAppendBlobHNSOAuthTest()
|
||||
{
|
||||
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
|
||||
VALUES=("OAuth" "true")
|
||||
triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||
}
|
||||
|
||||
combination=NonHNS-SharedKey
|
||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
|
||||
values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
|
||||
generateconfigs
|
||||
runTest=false
|
||||
cleanUpTestContainers=false
|
||||
echo 'Ensure below are complete before running script:'
|
||||
echo '1. Account specific settings file is present.'
|
||||
echo ' Copy accountName_settings.xml.template to accountName_settings.xml'
|
||||
echo ' where accountName in copied file name should be the test account name without domain'
|
||||
echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
|
||||
echo ' folder. New account settings file to be added to same folder.)'
|
||||
echo ' Follow instructions in the template to populate settings correctly for the account'
|
||||
echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
|
||||
echo ' where accountNames should be the test account names without domain'
|
||||
echo ' '
|
||||
echo ' '
|
||||
echo 'Choose action:'
|
||||
echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
|
||||
select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
|
||||
do
|
||||
case $scriptMode in
|
||||
SET_ACTIVE_TEST_CONFIG)
|
||||
runTest=false
|
||||
break
|
||||
;;
|
||||
RUN_TEST)
|
||||
runTest=true
|
||||
read -r -p "Enter parallel test run process count [default - 8]: " processCount
|
||||
processCount=${processCount:-8}
|
||||
break
|
||||
;;
|
||||
CLEAN_UP_OLD_TEST_CONTAINERS)
|
||||
runTest=false
|
||||
cleanUpTestContainers=true
|
||||
break
|
||||
;;
|
||||
SET_OR_CHANGE_TEST_ACCOUNT)
|
||||
runTest=false
|
||||
cleanUpTestContainers=false
|
||||
accountSettingsFile="src/test/resources/azure-auth-keys.xml"
|
||||
if [[ ! -f "$accountSettingsFile" ]];
|
||||
then
|
||||
logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
|
||||
cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
|
||||
fi
|
||||
|
||||
vi $accountSettingsFile
|
||||
exit 0
|
||||
break
|
||||
;;
|
||||
PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
|
||||
runTest=false
|
||||
cleanUpTestContainers=false
|
||||
logFilePaths=/tmp/logPaths
|
||||
find target/ -name "*output.txt" > $logFilePaths
|
||||
logOutput "$(cat $logFilePaths)"
|
||||
rm $logFilePaths
|
||||
exit 0
|
||||
break
|
||||
;;
|
||||
*) logOutput "ERROR: Invalid selection"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
### DO NOT EDIT THE LINES BELOW.
|
||||
## SECTION: COMBINATION DEFINITIONS AND TRIGGER
|
||||
|
||||
runtests "$@"
|
||||
echo ' '
|
||||
echo 'Set the active test combination to run the action:'
|
||||
select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
|
||||
do
|
||||
case $combo in
|
||||
HNS-OAuth)
|
||||
runHNSOAuthTest
|
||||
break
|
||||
;;
|
||||
HNS-SharedKey)
|
||||
runHNSSharedKeyTest
|
||||
break
|
||||
;;
|
||||
nonHNS-SharedKey)
|
||||
runNonHNSSharedKeyTest
|
||||
break
|
||||
;;
|
||||
AppendBlob-HNS-OAuth)
|
||||
runAppendBlobHNSOAuthTest
|
||||
break
|
||||
;;
|
||||
AllCombinationsTestRun)
|
||||
if [ $runTest == false ]
|
||||
then
|
||||
logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
|
||||
break
|
||||
fi
|
||||
runHNSOAuthTest
|
||||
runHNSSharedKeyTest
|
||||
runNonHNSSharedKeyTest
|
||||
runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
|
||||
break
|
||||
;;
|
||||
Quit)
|
||||
exit 0
|
||||
;;
|
||||
*) logOutput "ERROR: Invalid selection"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ $runTest == true ]
|
||||
then
|
||||
printAggregate
|
||||
fi
|
||||
|
|
|
@ -15,117 +15,88 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
testresourcesdir=src/test/resources
|
||||
combconfsdir=$testresourcesdir/combinationConfigFiles
|
||||
combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
|
||||
resourceDir=src/test/resources/
|
||||
accountSettingsFolderName=accountSettings
|
||||
combtestfile=$resourceDir
|
||||
combtestfile+=abfs-combination-test-configs.xml
|
||||
logdir=dev-support/testlogs/
|
||||
|
||||
logdir=dev-support/testlogs
|
||||
testresultsregex="Results:(\n|.)*?Tests run:"
|
||||
testresultsfilename=
|
||||
starttime=
|
||||
threadcount=
|
||||
defaultthreadcount=8
|
||||
accountConfigFileSuffix="_settings.xml"
|
||||
testOutputLogFolder=$logdir
|
||||
testlogfilename=combinationTestLogFile
|
||||
|
||||
properties=
|
||||
values=
|
||||
fullRunStartTime=$(date +%s)
|
||||
STARTTIME=$(date +%s)
|
||||
ENDTIME=$(date +%s)
|
||||
|
||||
validate() {
|
||||
if [ -z "$threadcount" ] ; then
|
||||
threadcount=$defaultthreadcount
|
||||
fi
|
||||
numberegex='^[0-9]+$'
|
||||
if ! [[ $threadcount =~ $numberegex ]] ; then
|
||||
echo "Exiting. The script param (threadcount) should be a number"
|
||||
exit -1
|
||||
fi
|
||||
if [ -z "$combination" ]; then
|
||||
echo "Exiting. combination cannot be empty"
|
||||
exit -1
|
||||
fi
|
||||
propertiessize=${#properties[@]}
|
||||
valuessize=${#values[@]}
|
||||
if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
|
||||
echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
for filename in "${combinations[@]}"; do
|
||||
if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
|
||||
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
|
||||
exit -1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
checkdependencies() {
|
||||
if ! [ "$(command -v pcregrep)" ]; then
|
||||
echo "Exiting. pcregrep is required to run the script."
|
||||
exit -1
|
||||
fi
|
||||
if ! [ "$(command -v xmlstarlet)" ]; then
|
||||
echo "Exiting. xmlstarlet is required to run the script."
|
||||
exit -1
|
||||
fi
|
||||
}
|
||||
|
||||
cleancombinationconfigs() {
|
||||
rm -rf $combconfsdir
|
||||
mkdir -p $combconfsdir
|
||||
}
|
||||
|
||||
generateconfigs() {
|
||||
combconffile="$combconfsdir/$combination.xml"
|
||||
rm -rf "$combconffile"
|
||||
cat > "$combconffile" << ENDOFFILE
|
||||
<configuration>
|
||||
|
||||
</configuration>
|
||||
ENDOFFILE
|
||||
|
||||
propertiessize=${#properties[@]}
|
||||
valuessize=${#values[@]}
|
||||
if [ "$propertiessize" -ne "$valuessize" ]; then
|
||||
echo "Exiting. Number of properties and values differ for $combination"
|
||||
exit -1
|
||||
fi
|
||||
for ((i = 0; i < propertiessize; i++)); do
|
||||
key=${properties[$i]}
|
||||
val=${values[$i]}
|
||||
changeconf "$key" "$val"
|
||||
done
|
||||
formatxml "$combconffile"
|
||||
}
|
||||
|
||||
formatxml() {
|
||||
xmlstarlet fo -s 2 "$1" > "$1.tmp"
|
||||
mv "$1.tmp" "$1"
|
||||
}
|
||||
|
||||
setactiveconf() {
|
||||
if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
|
||||
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
|
||||
exit -1
|
||||
outputFormatOn="\033[0;95m"
|
||||
outputFormatOff="\033[0m"
|
||||
|
||||
triggerRun()
|
||||
{
|
||||
echo ' '
|
||||
combination=$1
|
||||
accountName=$2
|
||||
runTest=$3
|
||||
processcount=$4
|
||||
cleanUpTestContainers=$5
|
||||
|
||||
if [ -z "$accountName" ]; then
|
||||
logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account."
|
||||
exit 1;
|
||||
fi
|
||||
accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix
|
||||
rm -rf $combtestfile
|
||||
cat > $combtestfile << ENDOFFILE
|
||||
<configuration>
|
||||
|
||||
</configuration>
|
||||
ENDOFFILE
|
||||
propertiessize=${#PROPERTIES[@]}
|
||||
valuessize=${#VALUES[@]}
|
||||
if [ "$propertiessize" -ne "$valuessize" ]; then
|
||||
logOutput "Exiting. Number of properties and values differ for $combination"
|
||||
exit 1
|
||||
fi
|
||||
for ((i = 0; i < propertiessize; i++)); do
|
||||
key=${PROPERTIES[$i]}
|
||||
val=${VALUES[$i]}
|
||||
echo "Combination specific property setting: [ key=$key , value=$val ]"
|
||||
changeconf "$key" "$val"
|
||||
done
|
||||
formatxml "$combtestfile"
|
||||
xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
|
||||
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
|
||||
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile
|
||||
xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
|
||||
formatxml $combtestfile
|
||||
}
|
||||
echo ' '
|
||||
echo "Activated [$combtestfile] - for account: $accountName for combination $combination"
|
||||
testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt"
|
||||
touch "$testlogfilename"
|
||||
|
||||
changeconf() {
|
||||
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
|
||||
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
|
||||
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
|
||||
if [ "$runTest" == true ]
|
||||
then
|
||||
echo "Exiting. Changing config property failed."
|
||||
exit -1
|
||||
STARTTIME=$(date +%s)
|
||||
echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]"
|
||||
logOutput "Test run report can be seen in $testlogfilename"
|
||||
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true
|
||||
ENDTIME=$(date +%s)
|
||||
summary
|
||||
fi
|
||||
|
||||
if [ "$cleanUpTestContainers" == true ]
|
||||
then
|
||||
mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true
|
||||
if grep -q "There are test failures" "$testlogfilename";
|
||||
then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename"
|
||||
pcregrep -M "$testresultsregex" "$testlogfilename"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
logOutput "Delete test containers - complete. Test run logs in - $testlogfilename"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
summary() {
|
||||
|
@ -134,17 +105,42 @@ summary() {
|
|||
echo "$combination"
|
||||
echo "========================"
|
||||
pcregrep -M "$testresultsregex" "$testlogfilename"
|
||||
} >> "$testresultsfilename"
|
||||
} >> "$aggregatedTestResult"
|
||||
printf "\n----- Test results -----\n"
|
||||
pcregrep -M "$testresultsregex" "$testlogfilename"
|
||||
|
||||
secondstaken=$((ENDTIME - STARTTIME))
|
||||
mins=$((secondstaken / 60))
|
||||
secs=$((secondstaken % 60))
|
||||
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
|
||||
echo "Find test logs for the combination ($combination) in: $testlogfilename"
|
||||
echo "Find consolidated test results in: $testresultsfilename"
|
||||
echo "----------"
|
||||
echo "Find test result for the combination ($combination) in: $testlogfilename"
|
||||
logOutput "Consolidated test result is saved in: $aggregatedTestResult"
|
||||
echo "------------------------"
|
||||
}
|
||||
|
||||
checkdependencies() {
|
||||
if ! [ "$(command -v pcregrep)" ]; then
|
||||
logOutput "Exiting. pcregrep is required to run the script."
|
||||
exit 1
|
||||
fi
|
||||
if ! [ "$(command -v xmlstarlet)" ]; then
|
||||
logOutput "Exiting. xmlstarlet is required to run the script."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
formatxml() {
|
||||
xmlstarlet fo -s 2 "$1" > "$1.tmp"
|
||||
mv "$1.tmp" "$1"
|
||||
}
|
||||
|
||||
changeconf() {
|
||||
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile"
|
||||
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile"
|
||||
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile"
|
||||
then
|
||||
logOutput "Exiting. Changing config property failed."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
init() {
|
||||
|
@ -153,89 +149,24 @@ init() {
|
|||
then
|
||||
echo ""
|
||||
echo "Exiting. Build failed."
|
||||
exit -1
|
||||
fi
|
||||
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
mkdir -p "$logdir"
|
||||
testresultsfilename="$logdir/$starttime/Test-Results.txt"
|
||||
if [[ -z "$combinations" ]]; then
|
||||
combinations=( $( ls $combconfsdir/*.xml ))
|
||||
fi
|
||||
}
|
||||
|
||||
runtests() {
|
||||
parseoptions "$@"
|
||||
validate
|
||||
if [ -z "$starttime" ]; then
|
||||
init
|
||||
fi
|
||||
shopt -s nullglob
|
||||
for combconffile in "${combinations[@]}"; do
|
||||
STARTTIME=$(date +%s)
|
||||
combination=$(basename "$combconffile" .xml)
|
||||
mkdir -p "$logdir/$starttime"
|
||||
testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
|
||||
printf "\nRunning the combination: %s..." "$combination"
|
||||
setactiveconf
|
||||
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
|
||||
ENDTIME=$(date +%s)
|
||||
summary
|
||||
done
|
||||
}
|
||||
|
||||
begin() {
|
||||
cleancombinationconfigs
|
||||
}
|
||||
|
||||
parseoptions() {
|
||||
runactivate=0
|
||||
runtests=0
|
||||
while getopts ":c:a:t:" option; do
|
||||
case "${option}" in
|
||||
a)
|
||||
if [[ "$runactivate" -eq "1" ]]; then
|
||||
echo "-a Option is not multivalued"
|
||||
exit 1
|
||||
fi
|
||||
runactivate=1
|
||||
combination=$(basename "$OPTARG" .xml)
|
||||
;;
|
||||
c)
|
||||
runtests=1
|
||||
combination=$(basename "$OPTARG" .xml)
|
||||
combinations+=("$combination")
|
||||
;;
|
||||
t)
|
||||
threadcount=$OPTARG
|
||||
;;
|
||||
*|?|h)
|
||||
if [[ -z "$combinations" ]]; then
|
||||
combinations=( $( ls $combconfsdir/*.xml ))
|
||||
fi
|
||||
combstr=""
|
||||
for combconffile in "${combinations[@]}"; do
|
||||
combname=$(basename "$combconffile" .xml)
|
||||
combstr="${combname}, ${combstr}"
|
||||
done
|
||||
combstr=${combstr:0:-2}
|
||||
|
||||
echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
|
||||
echo ""
|
||||
echo "Where:"
|
||||
echo " -a COMBINATION_NAME Specify the combination name which needs to be activated."
|
||||
echo " Configured combinations: ${combstr}"
|
||||
echo " -c COMBINATION_NAME Specify the combination name for test runs"
|
||||
echo " -t THREAD_COUNT Specify the thread count"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
|
||||
echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$runactivate" -eq "1" ]]; then
|
||||
setactiveconf
|
||||
exit 0
|
||||
fi
|
||||
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
testOutputLogFolder+=$starttime
|
||||
mkdir -p "$testOutputLogFolder"
|
||||
aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
|
||||
}
|
||||
|
||||
printAggregate() {
|
||||
echo :::: AGGREGATED TEST RESULT ::::
|
||||
cat "$aggregatedTestResult"
|
||||
fullRunEndTime=$(date +%s)
|
||||
fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
|
||||
mins=$((fullRunTimeInSecs / 60))
|
||||
secs=$((fullRunTimeInSecs % 60))
|
||||
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
|
||||
}
|
||||
|
||||
logOutput() {
|
||||
echo -e "$outputFormatOn" "$1" "$outputFormatOff"
|
||||
}
|
||||
|
|
|
@ -602,26 +602,76 @@ various test combinations, it will:
|
|||
2. Run tests for all combinations
|
||||
3. Summarize results across all the test combination runs.
|
||||
|
||||
As a pre-requisite step, fill config values for test accounts and credentials
|
||||
needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
|
||||
and rename as `src/test/resources/azure-auth-keys.xml`.
|
||||
Below are the pre-requiste steps to follow:
|
||||
1. Copy
|
||||
|
||||
**To add a new test combination:** Templates for mandatory test combinations
|
||||
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
|
||||
If a new one needs to be added, add a combination set within
|
||||
`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
|
||||
and
|
||||
1. Provide a new combination name
|
||||
2. Update properties and values array which need to be effective for the test
|
||||
combination
|
||||
3. Call generateconfigs
|
||||
./src/test/resources/azure-auth-keys.xml.template
|
||||
TO
|
||||
./src/test/resources/azure-auth-keys.xml
|
||||
Update account names that should be used in the test run for HNS and non-HNS
|
||||
combinations in the 2 properties present in the xml (account name should be
|
||||
without domain part), namely
|
||||
|
||||
fs.azure.hnsTestAccountName
|
||||
fs.azure.nonHnsTestAccountName
|
||||
azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
|
||||
|
||||
```
|
||||
XInclude is supported, so for extra security secrets may be
|
||||
kept out of the source tree then referenced through an an XInclude element:
|
||||
|
||||
<include xmlns="http://www.w3.org/2001/XInclude"
|
||||
href="/users/self/.secrets/auth-keys.xml" />
|
||||
```
|
||||
|
||||
2. Create account config files (one config file per account) in folder:
|
||||
|
||||
./src/test/resources/accountSettings/
|
||||
Follow the instruction in the start of the template file
|
||||
|
||||
accountName_settings.xml.template
|
||||
within accountSettings folder while creating account config file.
|
||||
New files created in folder accountSettings is listed in .gitignore to
|
||||
prevent accidental cred leaks.
|
||||
|
||||
**To run PR validation:** Running command
|
||||
* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
|
||||
each of the combinations defined and run tests for all the combinations.
|
||||
* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
|
||||
combinations can be provided with -c option. If combinations are provided
|
||||
with -c option, tests for only those combinations will be run.
|
||||
* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
|
||||
```bash
|
||||
Choose action:
|
||||
[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
|
||||
1) SET_ACTIVE_TEST_CONFIG 4) SET_OR_CHANGE_TEST_ACCOUNT
|
||||
2) RUN_TEST 5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
|
||||
3) CLEAN_UP_OLD_TEST_CONTAINERS
|
||||
#? 2
|
||||
```
|
||||
Enter 1: for setting active combination for IDE test run/single mvn test class runs.
|
||||
|
||||
Enter 2: for choosing the combination to choose for mvn full test suite.
|
||||
|
||||
Enter 3: For clean-up of any abruptly ending test leaving auto generated test
|
||||
container on the account.
|
||||
|
||||
Enter 4: To create/modify the config file that decides the account to use for specific test combination.
|
||||
|
||||
Enter 5: To print the log4j paths the last test runs.
|
||||
|
||||
On next prompt, current list of combinations to choose are provided.
|
||||
Sample for Run_TEST action:
|
||||
```bash
|
||||
Enter parallel test run process count [default - 8]: 4
|
||||
Set the active test combination to run the action:
|
||||
1) HNS-OAuth 3) nonHNS-SharedKey 5) AllCombinationsTestRun
|
||||
2) HNS-SharedKey 4) AppendBlob-HNS-OAuth 6) Quit
|
||||
#? 1
|
||||
|
||||
Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
|
||||
|
||||
Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
|
||||
Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
|
||||
Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
|
||||
````
|
||||
|
||||
Provide the option for the action chosen first.
|
||||
|
||||
**Test logs:** Test runs will create a folder within dev-support/testlogs to
|
||||
save the test logs. Folder name will be the test start timestamp. The mvn verify
|
||||
|
@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
|
|||
Test-Results.log in the same folder. When run for PR validation, the
|
||||
consolidated test results needs to be pasted into the PR comment section.
|
||||
|
||||
**To generate config for use in IDE:** Running command with -a (activate) option
|
||||
`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
|
||||
the effective config relevant for the specific test combination. Hence the same
|
||||
config files used by the mvn test runs can be used for IDE without any manual
|
||||
updates needed within config file.
|
||||
**To add a new test combination:** Templates for mandatory test combinations
|
||||
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
|
||||
If a new one needs to be added, add a combination to
|
||||
`dev-support/testrun-scripts/runtests.sh`.
|
||||
(Refer to current active combinations within
|
||||
`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
|
||||
`SECTION: TEST COMBINATION METHODS` in the script).
|
||||
|
||||
**Other command line options:**
|
||||
* -a <COMBINATION_NAME> Specify the combination name which needs to be
|
||||
activated. This is to be used to generate config for use in IDE.
|
||||
* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
|
||||
config is specified, tests for only the specified combinations will run. All
|
||||
combinations of tests will be running if this config is not specified.
|
||||
* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
|
||||
are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
|
||||
**Test Configuration Details:**
|
||||
|
||||
In order to test ABFS, please add the following configuration to your
|
||||
`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
|
||||
compatibility tests which require WASB credentials, in addition to the ABFS
|
||||
credentials.
|
||||
Note that the ABFS tests include compatibility tests which require WASB
|
||||
credentials, in addition to the ABFS credentials.
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
|
@ -0,0 +1,74 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.azurebfs.utils;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||
import com.microsoft.azure.storage.StorageCredentials;
|
||||
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
|
||||
|
||||
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
|
||||
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
|
||||
|
||||
/**
|
||||
* This looks like a test, but it is really a command to invoke to
|
||||
* clean up containers created in other test runs.
|
||||
*
|
||||
*/
|
||||
public class CleanupTestContainers extends AbstractAbfsIntegrationTest {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class);
|
||||
private static final String CONTAINER_PREFIX = "abfs-testcontainer-";
|
||||
|
||||
public CleanupTestContainers() throws Exception {
|
||||
}
|
||||
|
||||
@org.junit.Test
|
||||
public void testDeleteContainers() throws Throwable {
|
||||
int count = 0;
|
||||
AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration();
|
||||
String accountName = abfsConfig.getAccountName().split("\\.")[0];
|
||||
LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName());
|
||||
|
||||
String accountKey = abfsConfig.getStorageAccountKey();
|
||||
if ((accountKey == null) || (accountKey.isEmpty())) {
|
||||
LOG.debug("Clean up not possible. Account ket not present in config");
|
||||
}
|
||||
final StorageCredentials credentials;
|
||||
credentials = new StorageCredentialsAccountAndKey(
|
||||
accountName, accountKey);
|
||||
CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
|
||||
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
|
||||
Iterable<CloudBlobContainer> containers
|
||||
= blobClient.listContainers(CONTAINER_PREFIX);
|
||||
for (CloudBlobContainer container : containers) {
|
||||
LOG.info("Container {} URI {}",
|
||||
container.getName(),
|
||||
container.getUri());
|
||||
if (container.deleteIfExists()) {
|
||||
count++;
|
||||
LOG.info("Current deleted test containers count - #{}", count);
|
||||
}
|
||||
}
|
||||
LOG.info("Summary: Deleted {} test containers", count);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,185 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
NOTE:
|
||||
Each config file will hold configs for one storage account.
|
||||
ABFS Driver tests run with different Auth settings, OAuth and SharedKey, on same test account.
|
||||
Below template collects information needed for both Auth settings on the account.
|
||||
|
||||
OAuth test mode works with token provider type - ClientCredsTokenProvider
|
||||
For specific details on test config or pre-requisite, refer to
|
||||
https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
|
||||
|
||||
Pre-requisite before configuring:
|
||||
1. An ABFS account
|
||||
2. A container within account
|
||||
3. A Service principal with RBAC - Storage Blob Data Owner (superuser) role on account
|
||||
4. A Service principal with no RBAC roles assigned on account. This is needed for checkaccess tests (noRBACUser)
|
||||
5. A Service principal with Storage Blob Data Contributor RBAC roles assigned on account. This is needed for RBAC tests (contributorRBACUser)
|
||||
6. A Service principal with Storage Blob Data Reader RBAC roles assigned on account. This is needed for RBAC tests (readerRBACUser)
|
||||
|
||||
Replace below strings with actual value:
|
||||
## ACCOUNT SETTINGS ##
|
||||
1. ACCOUNTNAME -> to account name without domain
|
||||
2. IS_NAMESPACE_ENABLED -> to true or false
|
||||
3. CONTAINER_NAME -> name of an existing container
|
||||
4. ACCOUNT_KEY -> Account access key
|
||||
## IDENTITY SETTINGS ##
|
||||
5. SUPERUSER_TENANT_ID -> AAD tenant ID of Superuser Service principal
|
||||
6. SUPERUSER_CLIENT_ID -> SuperUser Service principal's client ID
|
||||
7. SUPERUSER_CLIENT_SECRET -> SuperUser Service principal's client secret
|
||||
|
||||
8. NO_RBAC_USER_CLIENT_ID -> noRBACUser Service principal's client ID
|
||||
9. NO_RBAC_USER_CLIENT_SECRET -> noRBACUser Service principal's client secret
|
||||
10. NO_RBAC_USER_OID -> noRBACUser Service principal's OID
|
||||
|
||||
11. CONTRIBUTOR_RBAC_USER_CLIENT_ID -> contributorRBACUser Service principal's client ID
|
||||
12. CONTRIBUTOR_RBAC_USER_CLIENT_SECRET -> contributorRBACUser Service principal's client secret
|
||||
|
||||
13. READER_RBAC_USER_CLIENT_ID -> readerRBACUser Service principal's client ID
|
||||
14. READER_RBAC_USER_CLIENT_SECRET -> readerRBACUser Service principal's client secret
|
||||
-->
|
||||
|
||||
<configuration>
|
||||
|
||||
<!-- ACCOUNT SETTINGS -->
|
||||
<property>
|
||||
<name>fs.azure.abfs.account.name</name>
|
||||
<value>ACCOUNTNAME.dfs.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.contract.test.fs.abfs</name>
|
||||
<value>abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.contract.test.fs.abfss</name>
|
||||
<value>abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.contract.test.fs.wasb</name>
|
||||
<value>wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.wasb.account.name</name>
|
||||
<value>ACCOUNTNAME.blob.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.scale.test.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>IS_NAMESPACE_ENABLED</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.test.namespace.enabled</name>
|
||||
<value>IS_NAMESPACE_ENABLED</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>IS_NAMESPACE_ENABLED</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.hns.enabled</name>
|
||||
<value>IS_NAMESPACE_ENABLED</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>ACCOUNT_KEY</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
|
||||
<value>ACCOUNT_KEY</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>ACCOUNT_KEY</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
|
||||
<value>ACCOUNT_KEY</value>
|
||||
</property>
|
||||
|
||||
<!-- SUPERUSER SETTINGS -->
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.endpoint</name>
|
||||
<value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth.provider.type</name>
|
||||
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>SUPERUSER_CLIENT_ID</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.id</name>
|
||||
<value>SUPERUSER_CLIENT_ID</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||
<value>SUPERUSER_CLIENT_SECRET</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.secret</name>
|
||||
<value>SUPERUSER_CLIENT_SECRET</value>
|
||||
</property>
|
||||
|
||||
<!-- NO RBAC USER SETTINGS -->
|
||||
<property>
|
||||
<name>fs.azure.enable.check.access</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.test.oauth2.client.id</name>
|
||||
<value>NO_RBAC_USER_CLIENT_ID</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.test.oauth2.client.secret</name>
|
||||
<value>NO_RBAC_USER_CLIENT_SECRET</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.check.access.testuser.guid</name>
|
||||
<value>NO_RBAC_USER_OID</value>
|
||||
</property>
|
||||
|
||||
<!-- CONTRIBUTOR RBAC USER SETTINGS -->
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.contributor.client.id</name>
|
||||
<value>CONTRIBUTOR_RBAC_USER_CLIENT_ID</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.contributor.client.secret</name>
|
||||
<value>CONTRIBUTOR_RBAC_USER_CLIENT_SECRET</value>
|
||||
</property>
|
||||
|
||||
<!-- READER RBAC USER SETTINGS -->
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.reader.client.id</name>
|
||||
<value>READER_RBAC_USER_CLIENT_ID</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.reader.client.secret</name>
|
||||
<value>READER_RBAC_USER_CLIENT_ID</value>
|
||||
</property>
|
||||
</configuration>
|
|
@ -14,162 +14,16 @@
|
|||
-->
|
||||
|
||||
<configuration>
|
||||
|
||||
<!--====================== IMPORTANT!! ========================-->
|
||||
<!--
|
||||
1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
|
||||
2. UPDATE runtests.sh
|
||||
A. "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
|
||||
ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
|
||||
SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
|
||||
B. PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
|
||||
DIFFERENT VARIANTS OF CONFIGS.
|
||||
3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
|
||||
4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
|
||||
-->
|
||||
|
||||
<!--=============== Auth type ===============-->
|
||||
<property>
|
||||
<name>fs.azure.account.auth.type</name>
|
||||
<value>SharedKey</value>
|
||||
</property>
|
||||
|
||||
<!--=============== Auth related accounts ===============-->
|
||||
<!-- This set of configs needs to be provided for all the accounts with which
|
||||
the tests needs to be ran. -->
|
||||
<property>
|
||||
<name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
|
||||
<value>{ACCOUNT_ACCESS_KEY}</value>
|
||||
<description>Account access key</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>
|
||||
fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
||||
</name>
|
||||
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
|
||||
<description>OAuth token provider implementation class</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>
|
||||
fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
||||
</name>
|
||||
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
|
||||
<description>Token end point, this can be found through Azure portal
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>
|
||||
fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
||||
</name>
|
||||
<value>{client id}</value>
|
||||
<description>AAD client id.</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>
|
||||
fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
||||
</name>
|
||||
<value>{client secret}</value>
|
||||
<description>AAD client secret</description>
|
||||
</property>
|
||||
|
||||
<!--=============== Configs for hadoop contract tests for ABFS ===============-->
|
||||
<property>
|
||||
<name>fs.contract.test.fs.abfs</name>
|
||||
<value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.contract.test.fs.abfss</name>
|
||||
<value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
|
||||
</property>
|
||||
|
||||
<!--=============== WASB Configs ===============-->
|
||||
<property>
|
||||
<name>fs.azure.wasb.account.name</name>
|
||||
<value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
|
||||
<value>WASB account key</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.contract.test.fs.wasb</name>
|
||||
<value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
|
||||
</value>
|
||||
</property>
|
||||
|
||||
<!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.contributor.client.id</name>
|
||||
<value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.contributor.client.secret</name>
|
||||
<value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.reader.client.id</name>
|
||||
<value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.reader.client.secret</name>
|
||||
<value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
|
||||
</property>
|
||||
|
||||
<!--=========================== FOR CheckAccess =========================-->
|
||||
<!-- To run ABFS CheckAccess tests, you must register an app, with no role
|
||||
assignments, and set the configuration discussed below:
|
||||
|
||||
1) Register a new app with no RBAC
|
||||
2) As part of the test configs you need to provide the guid for the above
|
||||
created app. Please follow the below steps to fetch the guid.
|
||||
a) Get an access token with the above created app. Please refer the
|
||||
following documentation for the same. https://docs.microsoft
|
||||
.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
|
||||
b) Decode the token fetched with the above step. You may use https
|
||||
://jwt.ms/ to decode the token
|
||||
d) The oid field in the decoded string is the guid.
|
||||
3) Set the following configurations:
|
||||
-->
|
||||
<!-- Account names without domain -->
|
||||
|
||||
<property>
|
||||
<name>fs.azure.account.test.oauth2.client.id</name>
|
||||
<value>{client id}</value>
|
||||
<description>The client id(app id) for the app created on step 1
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.test.oauth2.client.secret</name>
|
||||
<value>{client secret}</value>
|
||||
<description>
|
||||
The client secret(application's secret) for the app created on step 1
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.check.access.testuser.guid</name>
|
||||
<value>{guid}</value>
|
||||
<description>The guid fetched on step 2</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
|
||||
.windows.net</name>
|
||||
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
|
||||
<description>
|
||||
Token end point. This can be found through Azure portal. As part of CheckAccess
|
||||
test cases. The access will be tested for an FS instance created with the
|
||||
above mentioned client credentials. So this configuration is necessary to
|
||||
create the test FS instance.
|
||||
</description>
|
||||
<name>fs.azure.hnsTestAccountName</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
<!--========== Append blob related configs ===========-->
|
||||
<property>
|
||||
<name>fs.azure.test.appendblob.enabled</name>
|
||||
<value>false</value>
|
||||
<description>If made true, tests will be running under the assumption that
|
||||
append blob is enabled and the root directory and contract test root
|
||||
directory will be part of the append blob directories. Should be false for
|
||||
non-HNS accounts.
|
||||
</description>
|
||||
<name>fs.azure.nonHnsTestAccountName</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
|
|
|
@ -28,11 +28,8 @@
|
|||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.azure.test.namespace.enabled</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<!--==================== ABFS CONFIGURATION ====================-->
|
||||
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
|
||||
<property>
|
||||
<name>fs.azure.abfs.latency.track</name>
|
||||
<value>false</value>
|
||||
|
@ -43,9 +40,6 @@
|
|||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<!--==================== ABFS CONFIGURATION ====================-->
|
||||
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
|
||||
|
||||
<!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
|
||||
<property>
|
||||
<name>fs.azure.user.agent.prefix</name>
|
||||
|
@ -59,7 +53,40 @@
|
|||
STORE THE CONFIGURATION PROPERTIES WITHIN IT.
|
||||
TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
|
||||
THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
|
||||
IT IN PATCHES OR COMMITS. -->
|
||||
IT IN PATCHES OR COMMITS.
|
||||
|
||||
TEST SCRIPT RUNS:
|
||||
================
|
||||
FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE
|
||||
TRIGGERED OVER SCRIPT:
|
||||
./dev-support/testrun-scripts/runtests.sh
|
||||
(FROM hadoop-azure ROOT PROJECT PATH)
|
||||
|
||||
TO USE THE TEST SCRIPT,
|
||||
1. COPY
|
||||
./src/test/resources/azure-auth-keys.xml.template
|
||||
TO
|
||||
./src/test/resources/azure-auth-keys.xml
|
||||
UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN
|
||||
FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES
|
||||
PRESENT IN THE XML, NAMELY
|
||||
fs.azure.hnsTestAccountName and
|
||||
fs.azure.nonHnsTestAccountName
|
||||
(ACCOUNT NAME SHOULD BE WITHOUT DOMAIN)
|
||||
|
||||
azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY
|
||||
ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED.
|
||||
|
||||
2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE
|
||||
PER ACCOUNT) IN FOLDER:
|
||||
./src/test/resources/accountSettings/
|
||||
|
||||
FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE
|
||||
accountName_settings.xml.template
|
||||
WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE.
|
||||
|
||||
NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore
|
||||
TO PREVENT ACCIDENTAL CRED LEAKS. -->
|
||||
<!--=============================================================-->
|
||||
|
||||
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
|
||||
|
|
Loading…
Reference in New Issue