HADOOP-17767. ABFS: Update test scripts (#3124)
Contributed by Sneha Vijayarajan
This commit is contained in:
Sneha Vijayarajan
GitHub
parent
9adf0ca089
commit
a996d889ec
|
|
@ -1,5 +1,6 @@
|
||||||
.checkstyle
|
.checkstyle
|
||||||
bin/
|
bin/
|
||||||
src/test/resources/combinationConfigFiles
|
|
||||||
src/test/resources/abfs-combination-test-configs.xml
|
src/test/resources/abfs-combination-test-configs.xml
|
||||||
dev-support/testlogs
|
dev-support/testlogs
|
||||||
|
src/test/resources/accountSettings/*
|
||||||
|
!src/test/resources/accountSettings/accountName_settings.xml.template
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
# shellcheck disable=SC2034
|
# shellcheck disable=SC2034
|
||||||
# unused variables are global in nature and used in testsupport.sh
|
# unused variables are global in nature and used in testsupport.sh
|
||||||
|
test
|
||||||
set -eo pipefail
|
set -eo pipefail
|
||||||
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
|
@ -22,36 +22,154 @@ set -eo pipefail
|
||||||
|
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
. dev-support/testrun-scripts/testsupport.sh
|
. dev-support/testrun-scripts/testsupport.sh
|
||||||
|
init
|
||||||
|
|
||||||
begin
|
resourceDir=src/test/resources/
|
||||||
|
logdir=dev-support/testlogs/
|
||||||
|
azureTestXml=azure-auth-keys.xml
|
||||||
|
azureTestXmlPath=$resourceDir$azureTestXml
|
||||||
|
processCount=8
|
||||||
|
|
||||||
### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
|
## SECTION: TEST COMBINATION METHODS
|
||||||
### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
|
|
||||||
|
|
||||||
|
runHNSOAuthTest()
|
||||||
|
{
|
||||||
|
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||||
|
PROPERTIES=("fs.azure.account.auth.type")
|
||||||
|
VALUES=("OAuth")
|
||||||
|
triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||||
|
}
|
||||||
|
|
||||||
combination=HNS-OAuth
|
runHNSSharedKeyTest()
|
||||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
|
{
|
||||||
"fs.azure.account.auth.type")
|
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||||
values=("{account name}.dfs.core.windows.net" "true" "OAuth")
|
PROPERTIES=("fs.azure.account.auth.type")
|
||||||
generateconfigs
|
VALUES=("SharedKey")
|
||||||
|
triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||||
|
}
|
||||||
|
|
||||||
combination=AppendBlob-HNS-OAuth
|
runNonHNSSharedKeyTest()
|
||||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
|
{
|
||||||
"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
|
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||||
values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
|
PROPERTIES=("fs.azure.account.auth.type")
|
||||||
generateconfigs
|
VALUES=("SharedKey")
|
||||||
|
triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||||
|
}
|
||||||
|
|
||||||
combination=HNS-SharedKey
|
runAppendBlobHNSOAuthTest()
|
||||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
|
{
|
||||||
values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
|
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
|
||||||
generateconfigs
|
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
|
||||||
|
VALUES=("OAuth" "true")
|
||||||
|
triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
|
||||||
|
}
|
||||||
|
|
||||||
combination=NonHNS-SharedKey
|
runTest=false
|
||||||
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
|
cleanUpTestContainers=false
|
||||||
values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
|
echo 'Ensure below are complete before running script:'
|
||||||
generateconfigs
|
echo '1. Account specific settings file is present.'
|
||||||
|
echo ' Copy accountName_settings.xml.template to accountName_settings.xml'
|
||||||
|
echo ' where accountName in copied file name should be the test account name without domain'
|
||||||
|
echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
|
||||||
|
echo ' folder. New account settings file to be added to same folder.)'
|
||||||
|
echo ' Follow instructions in the template to populate settings correctly for the account'
|
||||||
|
echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
|
||||||
|
echo ' where accountNames should be the test account names without domain'
|
||||||
|
echo ' '
|
||||||
|
echo ' '
|
||||||
|
echo 'Choose action:'
|
||||||
|
echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
|
||||||
|
select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
|
||||||
|
do
|
||||||
|
case $scriptMode in
|
||||||
|
SET_ACTIVE_TEST_CONFIG)
|
||||||
|
runTest=false
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
RUN_TEST)
|
||||||
|
runTest=true
|
||||||
|
read -r -p "Enter parallel test run process count [default - 8]: " processCount
|
||||||
|
processCount=${processCount:-8}
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
CLEAN_UP_OLD_TEST_CONTAINERS)
|
||||||
|
runTest=false
|
||||||
|
cleanUpTestContainers=true
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
SET_OR_CHANGE_TEST_ACCOUNT)
|
||||||
|
runTest=false
|
||||||
|
cleanUpTestContainers=false
|
||||||
|
accountSettingsFile="src/test/resources/azure-auth-keys.xml"
|
||||||
|
if [[ ! -f "$accountSettingsFile" ]];
|
||||||
|
then
|
||||||
|
logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
|
||||||
|
cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
|
||||||
|
fi
|
||||||
|
|
||||||
|
vi $accountSettingsFile
|
||||||
|
exit 0
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
|
||||||
|
runTest=false
|
||||||
|
cleanUpTestContainers=false
|
||||||
|
logFilePaths=/tmp/logPaths
|
||||||
|
find target/ -name "*output.txt" > $logFilePaths
|
||||||
|
logOutput "$(cat $logFilePaths)"
|
||||||
|
rm $logFilePaths
|
||||||
|
exit 0
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
*) logOutput "ERROR: Invalid selection"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
### DO NOT EDIT THE LINES BELOW.
|
## SECTION: COMBINATION DEFINITIONS AND TRIGGER
|
||||||
|
|
||||||
runtests "$@"
|
echo ' '
|
||||||
|
echo 'Set the active test combination to run the action:'
|
||||||
|
select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
|
||||||
|
do
|
||||||
|
case $combo in
|
||||||
|
HNS-OAuth)
|
||||||
|
runHNSOAuthTest
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
HNS-SharedKey)
|
||||||
|
runHNSSharedKeyTest
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
nonHNS-SharedKey)
|
||||||
|
runNonHNSSharedKeyTest
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
AppendBlob-HNS-OAuth)
|
||||||
|
runAppendBlobHNSOAuthTest
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
AllCombinationsTestRun)
|
||||||
|
if [ $runTest == false ]
|
||||||
|
then
|
||||||
|
logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
runHNSOAuthTest
|
||||||
|
runHNSSharedKeyTest
|
||||||
|
runNonHNSSharedKeyTest
|
||||||
|
runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
Quit)
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*) logOutput "ERROR: Invalid selection"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ $runTest == true ]
|
||||||
|
then
|
||||||
|
printAggregate
|
||||||
|
fi
|
||||||
|
|
|
||||||
|
|
@ -15,117 +15,88 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
testresourcesdir=src/test/resources
|
resourceDir=src/test/resources/
|
||||||
combconfsdir=$testresourcesdir/combinationConfigFiles
|
accountSettingsFolderName=accountSettings
|
||||||
combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
|
combtestfile=$resourceDir
|
||||||
|
combtestfile+=abfs-combination-test-configs.xml
|
||||||
|
logdir=dev-support/testlogs/
|
||||||
|
|
||||||
logdir=dev-support/testlogs
|
|
||||||
testresultsregex="Results:(\n|.)*?Tests run:"
|
testresultsregex="Results:(\n|.)*?Tests run:"
|
||||||
testresultsfilename=
|
accountConfigFileSuffix="_settings.xml"
|
||||||
starttime=
|
testOutputLogFolder=$logdir
|
||||||
threadcount=
|
testlogfilename=combinationTestLogFile
|
||||||
defaultthreadcount=8
|
|
||||||
|
|
||||||
properties=
|
fullRunStartTime=$(date +%s)
|
||||||
values=
|
STARTTIME=$(date +%s)
|
||||||
|
ENDTIME=$(date +%s)
|
||||||
|
|
||||||
validate() {
|
outputFormatOn="\033[0;95m"
|
||||||
if [ -z "$threadcount" ] ; then
|
outputFormatOff="\033[0m"
|
||||||
threadcount=$defaultthreadcount
|
|
||||||
fi
|
triggerRun()
|
||||||
numberegex='^[0-9]+$'
|
{
|
||||||
if ! [[ $threadcount =~ $numberegex ]] ; then
|
echo ' '
|
||||||
echo "Exiting. The script param (threadcount) should be a number"
|
combination=$1
|
||||||
exit -1
|
accountName=$2
|
||||||
fi
|
runTest=$3
|
||||||
if [ -z "$combination" ]; then
|
processcount=$4
|
||||||
echo "Exiting. combination cannot be empty"
|
cleanUpTestContainers=$5
|
||||||
exit -1
|
|
||||||
fi
|
if [ -z "$accountName" ]; then
|
||||||
propertiessize=${#properties[@]}
|
logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account."
|
||||||
valuessize=${#values[@]}
|
exit 1;
|
||||||
if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
|
|
||||||
echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
for filename in "${combinations[@]}"; do
|
|
||||||
if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
|
|
||||||
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
checkdependencies() {
|
|
||||||
if ! [ "$(command -v pcregrep)" ]; then
|
|
||||||
echo "Exiting. pcregrep is required to run the script."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
if ! [ "$(command -v xmlstarlet)" ]; then
|
|
||||||
echo "Exiting. xmlstarlet is required to run the script."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
cleancombinationconfigs() {
|
|
||||||
rm -rf $combconfsdir
|
|
||||||
mkdir -p $combconfsdir
|
|
||||||
}
|
|
||||||
|
|
||||||
generateconfigs() {
|
|
||||||
combconffile="$combconfsdir/$combination.xml"
|
|
||||||
rm -rf "$combconffile"
|
|
||||||
cat > "$combconffile" << ENDOFFILE
|
|
||||||
<configuration>
|
|
||||||
|
|
||||||
</configuration>
|
|
||||||
ENDOFFILE
|
|
||||||
|
|
||||||
propertiessize=${#properties[@]}
|
|
||||||
valuessize=${#values[@]}
|
|
||||||
if [ "$propertiessize" -ne "$valuessize" ]; then
|
|
||||||
echo "Exiting. Number of properties and values differ for $combination"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
for ((i = 0; i < propertiessize; i++)); do
|
|
||||||
key=${properties[$i]}
|
|
||||||
val=${values[$i]}
|
|
||||||
changeconf "$key" "$val"
|
|
||||||
done
|
|
||||||
formatxml "$combconffile"
|
|
||||||
}
|
|
||||||
|
|
||||||
formatxml() {
|
|
||||||
xmlstarlet fo -s 2 "$1" > "$1.tmp"
|
|
||||||
mv "$1.tmp" "$1"
|
|
||||||
}
|
|
||||||
|
|
||||||
setactiveconf() {
|
|
||||||
if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
|
|
||||||
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
|
|
||||||
exit -1
|
|
||||||
fi
|
fi
|
||||||
|
accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix
|
||||||
rm -rf $combtestfile
|
rm -rf $combtestfile
|
||||||
cat > $combtestfile << ENDOFFILE
|
cat > $combtestfile << ENDOFFILE
|
||||||
<configuration>
|
<configuration>
|
||||||
|
|
||||||
</configuration>
|
</configuration>
|
||||||
ENDOFFILE
|
ENDOFFILE
|
||||||
|
propertiessize=${#PROPERTIES[@]}
|
||||||
|
valuessize=${#VALUES[@]}
|
||||||
|
if [ "$propertiessize" -ne "$valuessize" ]; then
|
||||||
|
logOutput "Exiting. Number of properties and values differ for $combination"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for ((i = 0; i < propertiessize; i++)); do
|
||||||
|
key=${PROPERTIES[$i]}
|
||||||
|
val=${VALUES[$i]}
|
||||||
|
echo "Combination specific property setting: [ key=$key , value=$val ]"
|
||||||
|
changeconf "$key" "$val"
|
||||||
|
done
|
||||||
|
formatxml "$combtestfile"
|
||||||
xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
|
xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
|
||||||
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
|
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile
|
||||||
xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
|
xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
|
||||||
formatxml $combtestfile
|
formatxml $combtestfile
|
||||||
}
|
echo ' '
|
||||||
|
echo "Activated [$combtestfile] - for account: $accountName for combination $combination"
|
||||||
|
testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt"
|
||||||
|
touch "$testlogfilename"
|
||||||
|
|
||||||
changeconf() {
|
if [ "$runTest" == true ]
|
||||||
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
|
|
||||||
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
|
|
||||||
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
|
|
||||||
then
|
then
|
||||||
echo "Exiting. Changing config property failed."
|
STARTTIME=$(date +%s)
|
||||||
exit -1
|
echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]"
|
||||||
|
logOutput "Test run report can be seen in $testlogfilename"
|
||||||
|
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true
|
||||||
|
ENDTIME=$(date +%s)
|
||||||
|
summary
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "$cleanUpTestContainers" == true ]
|
||||||
|
then
|
||||||
|
mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true
|
||||||
|
if grep -q "There are test failures" "$testlogfilename";
|
||||||
|
then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename"
|
||||||
|
pcregrep -M "$testresultsregex" "$testlogfilename"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
logOutput "Delete test containers - complete. Test run logs in - $testlogfilename"
|
||||||
|
fi
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
summary() {
|
summary() {
|
||||||
|
|
@ -134,17 +105,42 @@ summary() {
|
||||||
echo "$combination"
|
echo "$combination"
|
||||||
echo "========================"
|
echo "========================"
|
||||||
pcregrep -M "$testresultsregex" "$testlogfilename"
|
pcregrep -M "$testresultsregex" "$testlogfilename"
|
||||||
} >> "$testresultsfilename"
|
} >> "$aggregatedTestResult"
|
||||||
printf "\n----- Test results -----\n"
|
printf "\n----- Test results -----\n"
|
||||||
pcregrep -M "$testresultsregex" "$testlogfilename"
|
pcregrep -M "$testresultsregex" "$testlogfilename"
|
||||||
|
|
||||||
secondstaken=$((ENDTIME - STARTTIME))
|
secondstaken=$((ENDTIME - STARTTIME))
|
||||||
mins=$((secondstaken / 60))
|
mins=$((secondstaken / 60))
|
||||||
secs=$((secondstaken % 60))
|
secs=$((secondstaken % 60))
|
||||||
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
|
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
|
||||||
echo "Find test logs for the combination ($combination) in: $testlogfilename"
|
echo "Find test result for the combination ($combination) in: $testlogfilename"
|
||||||
echo "Find consolidated test results in: $testresultsfilename"
|
logOutput "Consolidated test result is saved in: $aggregatedTestResult"
|
||||||
echo "----------"
|
echo "------------------------"
|
||||||
|
}
|
||||||
|
|
||||||
|
checkdependencies() {
|
||||||
|
if ! [ "$(command -v pcregrep)" ]; then
|
||||||
|
logOutput "Exiting. pcregrep is required to run the script."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [ "$(command -v xmlstarlet)" ]; then
|
||||||
|
logOutput "Exiting. xmlstarlet is required to run the script."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
formatxml() {
|
||||||
|
xmlstarlet fo -s 2 "$1" > "$1.tmp"
|
||||||
|
mv "$1.tmp" "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
changeconf() {
|
||||||
|
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile"
|
||||||
|
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile"
|
||||||
|
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile"
|
||||||
|
then
|
||||||
|
logOutput "Exiting. Changing config property failed."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
init() {
|
init() {
|
||||||
|
|
@ -153,89 +149,24 @@ init() {
|
||||||
then
|
then
|
||||||
echo ""
|
echo ""
|
||||||
echo "Exiting. Build failed."
|
echo "Exiting. Build failed."
|
||||||
exit -1
|
exit 1
|
||||||
fi
|
fi
|
||||||
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
|
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||||
mkdir -p "$logdir"
|
testOutputLogFolder+=$starttime
|
||||||
testresultsfilename="$logdir/$starttime/Test-Results.txt"
|
mkdir -p "$testOutputLogFolder"
|
||||||
if [[ -z "$combinations" ]]; then
|
aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
|
||||||
combinations=( $( ls $combconfsdir/*.xml ))
|
}
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
runtests() {
|
printAggregate() {
|
||||||
parseoptions "$@"
|
echo :::: AGGREGATED TEST RESULT ::::
|
||||||
validate
|
cat "$aggregatedTestResult"
|
||||||
if [ -z "$starttime" ]; then
|
fullRunEndTime=$(date +%s)
|
||||||
init
|
fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
|
||||||
fi
|
mins=$((fullRunTimeInSecs / 60))
|
||||||
shopt -s nullglob
|
secs=$((fullRunTimeInSecs % 60))
|
||||||
for combconffile in "${combinations[@]}"; do
|
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
|
||||||
STARTTIME=$(date +%s)
|
}
|
||||||
combination=$(basename "$combconffile" .xml)
|
|
||||||
mkdir -p "$logdir/$starttime"
|
|
||||||
testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
|
|
||||||
printf "\nRunning the combination: %s..." "$combination"
|
|
||||||
setactiveconf
|
|
||||||
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
|
|
||||||
ENDTIME=$(date +%s)
|
|
||||||
summary
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
begin() {
|
logOutput() {
|
||||||
cleancombinationconfigs
|
echo -e "$outputFormatOn" "$1" "$outputFormatOff"
|
||||||
}
|
|
||||||
|
|
||||||
parseoptions() {
|
|
||||||
runactivate=0
|
|
||||||
runtests=0
|
|
||||||
while getopts ":c:a:t:" option; do
|
|
||||||
case "${option}" in
|
|
||||||
a)
|
|
||||||
if [[ "$runactivate" -eq "1" ]]; then
|
|
||||||
echo "-a Option is not multivalued"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
runactivate=1
|
|
||||||
combination=$(basename "$OPTARG" .xml)
|
|
||||||
;;
|
|
||||||
c)
|
|
||||||
runtests=1
|
|
||||||
combination=$(basename "$OPTARG" .xml)
|
|
||||||
combinations+=("$combination")
|
|
||||||
;;
|
|
||||||
t)
|
|
||||||
threadcount=$OPTARG
|
|
||||||
;;
|
|
||||||
*|?|h)
|
|
||||||
if [[ -z "$combinations" ]]; then
|
|
||||||
combinations=( $( ls $combconfsdir/*.xml ))
|
|
||||||
fi
|
|
||||||
combstr=""
|
|
||||||
for combconffile in "${combinations[@]}"; do
|
|
||||||
combname=$(basename "$combconffile" .xml)
|
|
||||||
combstr="${combname}, ${combstr}"
|
|
||||||
done
|
|
||||||
combstr=${combstr:0:-2}
|
|
||||||
|
|
||||||
echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
|
|
||||||
echo ""
|
|
||||||
echo "Where:"
|
|
||||||
echo " -a COMBINATION_NAME Specify the combination name which needs to be activated."
|
|
||||||
echo " Configured combinations: ${combstr}"
|
|
||||||
echo " -c COMBINATION_NAME Specify the combination name for test runs"
|
|
||||||
echo " -t THREAD_COUNT Specify the thread count"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
|
|
||||||
echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [[ "$runactivate" -eq "1" ]]; then
|
|
||||||
setactiveconf
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -602,26 +602,76 @@ various test combinations, it will:
|
||||||
2. Run tests for all combinations
|
2. Run tests for all combinations
|
||||||
3. Summarize results across all the test combination runs.
|
3. Summarize results across all the test combination runs.
|
||||||
|
|
||||||
As a pre-requisite step, fill config values for test accounts and credentials
|
Below are the pre-requiste steps to follow:
|
||||||
needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
|
1. Copy
|
||||||
and rename as `src/test/resources/azure-auth-keys.xml`.
|
|
||||||
|
|
||||||
**To add a new test combination:** Templates for mandatory test combinations
|
./src/test/resources/azure-auth-keys.xml.template
|
||||||
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
|
TO
|
||||||
If a new one needs to be added, add a combination set within
|
./src/test/resources/azure-auth-keys.xml
|
||||||
`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
|
Update account names that should be used in the test run for HNS and non-HNS
|
||||||
and
|
combinations in the 2 properties present in the xml (account name should be
|
||||||
1. Provide a new combination name
|
without domain part), namely
|
||||||
2. Update properties and values array which need to be effective for the test
|
|
||||||
combination
|
fs.azure.hnsTestAccountName
|
||||||
3. Call generateconfigs
|
fs.azure.nonHnsTestAccountName
|
||||||
|
azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
|
||||||
|
|
||||||
|
```
|
||||||
|
XInclude is supported, so for extra security secrets may be
|
||||||
|
kept out of the source tree then referenced through an an XInclude element:
|
||||||
|
|
||||||
|
<include xmlns="http://www.w3.org/2001/XInclude"
|
||||||
|
href="/users/self/.secrets/auth-keys.xml" />
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Create account config files (one config file per account) in folder:
|
||||||
|
|
||||||
|
./src/test/resources/accountSettings/
|
||||||
|
Follow the instruction in the start of the template file
|
||||||
|
|
||||||
|
accountName_settings.xml.template
|
||||||
|
within accountSettings folder while creating account config file.
|
||||||
|
New files created in folder accountSettings is listed in .gitignore to
|
||||||
|
prevent accidental cred leaks.
|
||||||
|
|
||||||
**To run PR validation:** Running command
|
**To run PR validation:** Running command
|
||||||
* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
|
* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
|
||||||
each of the combinations defined and run tests for all the combinations.
|
```bash
|
||||||
* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
|
Choose action:
|
||||||
combinations can be provided with -c option. If combinations are provided
|
[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
|
||||||
with -c option, tests for only those combinations will be run.
|
1) SET_ACTIVE_TEST_CONFIG 4) SET_OR_CHANGE_TEST_ACCOUNT
|
||||||
|
2) RUN_TEST 5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
|
||||||
|
3) CLEAN_UP_OLD_TEST_CONTAINERS
|
||||||
|
#? 2
|
||||||
|
```
|
||||||
|
Enter 1: for setting active combination for IDE test run/single mvn test class runs.
|
||||||
|
|
||||||
|
Enter 2: for choosing the combination to choose for mvn full test suite.
|
||||||
|
|
||||||
|
Enter 3: For clean-up of any abruptly ending test leaving auto generated test
|
||||||
|
container on the account.
|
||||||
|
|
||||||
|
Enter 4: To create/modify the config file that decides the account to use for specific test combination.
|
||||||
|
|
||||||
|
Enter 5: To print the log4j paths the last test runs.
|
||||||
|
|
||||||
|
On next prompt, current list of combinations to choose are provided.
|
||||||
|
Sample for Run_TEST action:
|
||||||
|
```bash
|
||||||
|
Enter parallel test run process count [default - 8]: 4
|
||||||
|
Set the active test combination to run the action:
|
||||||
|
1) HNS-OAuth 3) nonHNS-SharedKey 5) AllCombinationsTestRun
|
||||||
|
2) HNS-SharedKey 4) AppendBlob-HNS-OAuth 6) Quit
|
||||||
|
#? 1
|
||||||
|
|
||||||
|
Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
|
||||||
|
|
||||||
|
Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
|
||||||
|
Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
|
||||||
|
Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
|
||||||
|
````
|
||||||
|
|
||||||
|
Provide the option for the action chosen first.
|
||||||
|
|
||||||
**Test logs:** Test runs will create a folder within dev-support/testlogs to
|
**Test logs:** Test runs will create a folder within dev-support/testlogs to
|
||||||
save the test logs. Folder name will be the test start timestamp. The mvn verify
|
save the test logs. Folder name will be the test start timestamp. The mvn verify
|
||||||
|
|
@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
|
||||||
Test-Results.log in the same folder. When run for PR validation, the
|
Test-Results.log in the same folder. When run for PR validation, the
|
||||||
consolidated test results needs to be pasted into the PR comment section.
|
consolidated test results needs to be pasted into the PR comment section.
|
||||||
|
|
||||||
**To generate config for use in IDE:** Running command with -a (activate) option
|
**To add a new test combination:** Templates for mandatory test combinations
|
||||||
`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
|
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
|
||||||
the effective config relevant for the specific test combination. Hence the same
|
If a new one needs to be added, add a combination to
|
||||||
config files used by the mvn test runs can be used for IDE without any manual
|
`dev-support/testrun-scripts/runtests.sh`.
|
||||||
updates needed within config file.
|
(Refer to current active combinations within
|
||||||
|
`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
|
||||||
|
`SECTION: TEST COMBINATION METHODS` in the script).
|
||||||
|
|
||||||
**Other command line options:**
|
**Test Configuration Details:**
|
||||||
* -a <COMBINATION_NAME> Specify the combination name which needs to be
|
|
||||||
activated. This is to be used to generate config for use in IDE.
|
|
||||||
* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
|
|
||||||
config is specified, tests for only the specified combinations will run. All
|
|
||||||
combinations of tests will be running if this config is not specified.
|
|
||||||
* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
|
|
||||||
are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
|
|
||||||
|
|
||||||
In order to test ABFS, please add the following configuration to your
|
Note that the ABFS tests include compatibility tests which require WASB
|
||||||
`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
|
credentials, in addition to the ABFS credentials.
|
||||||
compatibility tests which require WASB credentials, in addition to the ABFS
|
|
||||||
credentials.
|
|
||||||
|
|
||||||
```xml
|
```xml
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.azurebfs.utils;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||||
|
import com.microsoft.azure.storage.StorageCredentials;
|
||||||
|
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
|
||||||
|
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This looks like a test, but it is really a command to invoke to
|
||||||
|
* clean up containers created in other test runs.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class CleanupTestContainers extends AbstractAbfsIntegrationTest {
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class);
|
||||||
|
private static final String CONTAINER_PREFIX = "abfs-testcontainer-";
|
||||||
|
|
||||||
|
public CleanupTestContainers() throws Exception {
|
||||||
|
}
|
||||||
|
|
||||||
|
@org.junit.Test
|
||||||
|
public void testDeleteContainers() throws Throwable {
|
||||||
|
int count = 0;
|
||||||
|
AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration();
|
||||||
|
String accountName = abfsConfig.getAccountName().split("\\.")[0];
|
||||||
|
LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName());
|
||||||
|
|
||||||
|
String accountKey = abfsConfig.getStorageAccountKey();
|
||||||
|
if ((accountKey == null) || (accountKey.isEmpty())) {
|
||||||
|
LOG.debug("Clean up not possible. Account ket not present in config");
|
||||||
|
}
|
||||||
|
final StorageCredentials credentials;
|
||||||
|
credentials = new StorageCredentialsAccountAndKey(
|
||||||
|
accountName, accountKey);
|
||||||
|
CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
|
||||||
|
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
|
||||||
|
Iterable<CloudBlobContainer> containers
|
||||||
|
= blobClient.listContainers(CONTAINER_PREFIX);
|
||||||
|
for (CloudBlobContainer container : containers) {
|
||||||
|
LOG.info("Container {} URI {}",
|
||||||
|
container.getName(),
|
||||||
|
container.getUri());
|
||||||
|
if (container.deleteIfExists()) {
|
||||||
|
count++;
|
||||||
|
LOG.info("Current deleted test containers count - #{}", count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LOG.info("Summary: Deleted {} test containers", count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,185 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
|
||||||
|
NOTE:
|
||||||
|
Each config file will hold configs for one storage account.
|
||||||
|
ABFS Driver tests run with different Auth settings, OAuth and SharedKey, on same test account.
|
||||||
|
Below template collects information needed for both Auth settings on the account.
|
||||||
|
|
||||||
|
OAuth test mode works with token provider type - ClientCredsTokenProvider
|
||||||
|
For specific details on test config or pre-requisite, refer to
|
||||||
|
https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
|
||||||
|
|
||||||
|
Pre-requisite before configuring:
|
||||||
|
1. An ABFS account
|
||||||
|
2. A container within account
|
||||||
|
3. A Service principal with RBAC - Storage Blob Data Owner (superuser) role on account
|
||||||
|
4. A Service principal with no RBAC roles assigned on account. This is needed for checkaccess tests (noRBACUser)
|
||||||
|
5. A Service principal with Storage Blob Data Contributor RBAC roles assigned on account. This is needed for RBAC tests (contributorRBACUser)
|
||||||
|
6. A Service principal with Storage Blob Data Reader RBAC roles assigned on account. This is needed for RBAC tests (readerRBACUser)
|
||||||
|
|
||||||
|
Replace below strings with actual value:
|
||||||
|
## ACCOUNT SETTINGS ##
|
||||||
|
1. ACCOUNTNAME -> to account name without domain
|
||||||
|
2. IS_NAMESPACE_ENABLED -> to true or false
|
||||||
|
3. CONTAINER_NAME -> name of an existing container
|
||||||
|
4. ACCOUNT_KEY -> Account access key
|
||||||
|
## IDENTITY SETTINGS ##
|
||||||
|
5. SUPERUSER_TENANT_ID -> AAD tenant ID of Superuser Service principal
|
||||||
|
6. SUPERUSER_CLIENT_ID -> SuperUser Service principal's client ID
|
||||||
|
7. SUPERUSER_CLIENT_SECRET -> SuperUser Service principal's client secret
|
||||||
|
|
||||||
|
8. NO_RBAC_USER_CLIENT_ID -> noRBACUser Service principal's client ID
|
||||||
|
9. NO_RBAC_USER_CLIENT_SECRET -> noRBACUser Service principal's client secret
|
||||||
|
10. NO_RBAC_USER_OID -> noRBACUser Service principal's OID
|
||||||
|
|
||||||
|
11. CONTRIBUTOR_RBAC_USER_CLIENT_ID -> contributorRBACUser Service principal's client ID
|
||||||
|
12. CONTRIBUTOR_RBAC_USER_CLIENT_SECRET -> contributorRBACUser Service principal's client secret
|
||||||
|
|
||||||
|
13. READER_RBAC_USER_CLIENT_ID -> readerRBACUser Service principal's client ID
|
||||||
|
14. READER_RBAC_USER_CLIENT_SECRET -> readerRBACUser Service principal's client secret
|
||||||
|
-->
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
|
||||||
|
<!-- ACCOUNT SETTINGS -->
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.abfs.account.name</name>
|
||||||
|
<value>ACCOUNTNAME.dfs.core.windows.net</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.test.fs.abfs</name>
|
||||||
|
<value>abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.test.fs.abfss</name>
|
||||||
|
<value>abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.test.fs.wasb</name>
|
||||||
|
<value>wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.wasb.account.name</name>
|
||||||
|
<value>ACCOUNTNAME.blob.core.windows.net</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.scale.test.enabled</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>IS_NAMESPACE_ENABLED</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.test.namespace.enabled</name>
|
||||||
|
<value>IS_NAMESPACE_ENABLED</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>IS_NAMESPACE_ENABLED</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.hns.enabled</name>
|
||||||
|
<value>IS_NAMESPACE_ENABLED</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>ACCOUNT_KEY</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
|
||||||
|
<value>ACCOUNT_KEY</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>ACCOUNT_KEY</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
|
||||||
|
<value>ACCOUNT_KEY</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<!-- SUPERUSER SETTINGS -->
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.client.endpoint</name>
|
||||||
|
<value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth.provider.type</name>
|
||||||
|
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>SUPERUSER_CLIENT_ID</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.client.id</name>
|
||||||
|
<value>SUPERUSER_CLIENT_ID</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net</name>
|
||||||
|
<value>SUPERUSER_CLIENT_SECRET</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.client.secret</name>
|
||||||
|
<value>SUPERUSER_CLIENT_SECRET</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<!-- NO RBAC USER SETTINGS -->
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.enable.check.access</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.test.oauth2.client.id</name>
|
||||||
|
<value>NO_RBAC_USER_CLIENT_ID</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.test.oauth2.client.secret</name>
|
||||||
|
<value>NO_RBAC_USER_CLIENT_SECRET</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.check.access.testuser.guid</name>
|
||||||
|
<value>NO_RBAC_USER_OID</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<!-- CONTRIBUTOR RBAC USER SETTINGS -->
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.contributor.client.id</name>
|
||||||
|
<value>CONTRIBUTOR_RBAC_USER_CLIENT_ID</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.contributor.client.secret</name>
|
||||||
|
<value>CONTRIBUTOR_RBAC_USER_CLIENT_SECRET</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<!-- READER RBAC USER SETTINGS -->
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.reader.client.id</name>
|
||||||
|
<value>READER_RBAC_USER_CLIENT_ID</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.azure.account.oauth2.reader.client.secret</name>
|
||||||
|
<value>READER_RBAC_USER_CLIENT_ID</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
|
|
@ -14,162 +14,16 @@
|
||||||
-->
|
-->
|
||||||
|
|
||||||
<configuration>
|
<configuration>
|
||||||
|
<!-- Account names without domain -->
|
||||||
<!--====================== IMPORTANT!! ========================-->
|
|
||||||
<!--
|
|
||||||
1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
|
|
||||||
2. UPDATE runtests.sh
|
|
||||||
A. "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
|
|
||||||
ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
|
|
||||||
SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
|
|
||||||
B. PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
|
|
||||||
DIFFERENT VARIANTS OF CONFIGS.
|
|
||||||
3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
|
|
||||||
4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
|
|
||||||
-->
|
|
||||||
|
|
||||||
<!--=============== Auth type ===============-->
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.auth.type</name>
|
|
||||||
<value>SharedKey</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<!--=============== Auth related accounts ===============-->
|
|
||||||
<!-- This set of configs needs to be provided for all the accounts with which
|
|
||||||
the tests needs to be ran. -->
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
|
|
||||||
<value>{ACCOUNT_ACCESS_KEY}</value>
|
|
||||||
<description>Account access key</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>
|
|
||||||
fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
|
||||||
</name>
|
|
||||||
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
|
|
||||||
<description>OAuth token provider implementation class</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>
|
|
||||||
fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
|
||||||
</name>
|
|
||||||
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
|
|
||||||
<description>Token end point, this can be found through Azure portal
|
|
||||||
</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>
|
|
||||||
fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
|
||||||
</name>
|
|
||||||
<value>{client id}</value>
|
|
||||||
<description>AAD client id.</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>
|
|
||||||
fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
|
|
||||||
</name>
|
|
||||||
<value>{client secret}</value>
|
|
||||||
<description>AAD client secret</description>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<!--=============== Configs for hadoop contract tests for ABFS ===============-->
|
|
||||||
<property>
|
|
||||||
<name>fs.contract.test.fs.abfs</name>
|
|
||||||
<value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.contract.test.fs.abfss</name>
|
|
||||||
<value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<!--=============== WASB Configs ===============-->
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.wasb.account.name</name>
|
|
||||||
<value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
|
|
||||||
<value>WASB account key</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.contract.test.fs.wasb</name>
|
|
||||||
<value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
|
|
||||||
</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.oauth2.contributor.client.id</name>
|
|
||||||
<value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.oauth2.contributor.client.secret</name>
|
|
||||||
<value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.oauth2.reader.client.id</name>
|
|
||||||
<value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.oauth2.reader.client.secret</name>
|
|
||||||
<value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<!--=========================== FOR CheckAccess =========================-->
|
|
||||||
<!-- To run ABFS CheckAccess tests, you must register an app, with no role
|
|
||||||
assignments, and set the configuration discussed below:
|
|
||||||
|
|
||||||
1) Register a new app with no RBAC
|
|
||||||
2) As part of the test configs you need to provide the guid for the above
|
|
||||||
created app. Please follow the below steps to fetch the guid.
|
|
||||||
a) Get an access token with the above created app. Please refer the
|
|
||||||
following documentation for the same. https://docs.microsoft
|
|
||||||
.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
|
|
||||||
b) Decode the token fetched with the above step. You may use https
|
|
||||||
://jwt.ms/ to decode the token
|
|
||||||
d) The oid field in the decoded string is the guid.
|
|
||||||
3) Set the following configurations:
|
|
||||||
-->
|
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>fs.azure.account.test.oauth2.client.id</name>
|
<name>fs.azure.hnsTestAccountName</name>
|
||||||
<value>{client id}</value>
|
<value></value>
|
||||||
<description>The client id(app id) for the app created on step 1
|
|
||||||
</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.test.oauth2.client.secret</name>
|
|
||||||
<value>{client secret}</value>
|
|
||||||
<description>
|
|
||||||
The client secret(application's secret) for the app created on step 1
|
|
||||||
</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.check.access.testuser.guid</name>
|
|
||||||
<value>{guid}</value>
|
|
||||||
<description>The guid fetched on step 2</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
|
|
||||||
.windows.net</name>
|
|
||||||
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
|
|
||||||
<description>
|
|
||||||
Token end point. This can be found through Azure portal. As part of CheckAccess
|
|
||||||
test cases. The access will be tested for an FS instance created with the
|
|
||||||
above mentioned client credentials. So this configuration is necessary to
|
|
||||||
create the test FS instance.
|
|
||||||
</description>
|
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
<!--========== Append blob related configs ===========-->
|
|
||||||
<property>
|
<property>
|
||||||
<name>fs.azure.test.appendblob.enabled</name>
|
<name>fs.azure.nonHnsTestAccountName</name>
|
||||||
<value>false</value>
|
<value></value>
|
||||||
<description>If made true, tests will be running under the assumption that
|
|
||||||
append blob is enabled and the root directory and contract test root
|
|
||||||
directory will be part of the append blob directories. Should be false for
|
|
||||||
non-HNS accounts.
|
|
||||||
</description>
|
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
|
||||||
|
|
@ -28,11 +28,8 @@
|
||||||
<value>false</value>
|
<value>false</value>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
<property>
|
<!--==================== ABFS CONFIGURATION ====================-->
|
||||||
<name>fs.azure.test.namespace.enabled</name>
|
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>fs.azure.abfs.latency.track</name>
|
<name>fs.azure.abfs.latency.track</name>
|
||||||
<value>false</value>
|
<value>false</value>
|
||||||
|
|
@ -43,9 +40,6 @@
|
||||||
<value>true</value>
|
<value>true</value>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
<!--==================== ABFS CONFIGURATION ====================-->
|
|
||||||
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
|
|
||||||
|
|
||||||
<!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
|
<!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
|
||||||
<property>
|
<property>
|
||||||
<name>fs.azure.user.agent.prefix</name>
|
<name>fs.azure.user.agent.prefix</name>
|
||||||
|
|
@ -59,7 +53,40 @@
|
||||||
STORE THE CONFIGURATION PROPERTIES WITHIN IT.
|
STORE THE CONFIGURATION PROPERTIES WITHIN IT.
|
||||||
TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
|
TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
|
||||||
THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
|
THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
|
||||||
IT IN PATCHES OR COMMITS. -->
|
IT IN PATCHES OR COMMITS.
|
||||||
|
|
||||||
|
TEST SCRIPT RUNS:
|
||||||
|
================
|
||||||
|
FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE
|
||||||
|
TRIGGERED OVER SCRIPT:
|
||||||
|
./dev-support/testrun-scripts/runtests.sh
|
||||||
|
(FROM hadoop-azure ROOT PROJECT PATH)
|
||||||
|
|
||||||
|
TO USE THE TEST SCRIPT,
|
||||||
|
1. COPY
|
||||||
|
./src/test/resources/azure-auth-keys.xml.template
|
||||||
|
TO
|
||||||
|
./src/test/resources/azure-auth-keys.xml
|
||||||
|
UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN
|
||||||
|
FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES
|
||||||
|
PRESENT IN THE XML, NAMELY
|
||||||
|
fs.azure.hnsTestAccountName and
|
||||||
|
fs.azure.nonHnsTestAccountName
|
||||||
|
(ACCOUNT NAME SHOULD BE WITHOUT DOMAIN)
|
||||||
|
|
||||||
|
azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY
|
||||||
|
ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED.
|
||||||
|
|
||||||
|
2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE
|
||||||
|
PER ACCOUNT) IN FOLDER:
|
||||||
|
./src/test/resources/accountSettings/
|
||||||
|
|
||||||
|
FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE
|
||||||
|
accountName_settings.xml.template
|
||||||
|
WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE.
|
||||||
|
|
||||||
|
NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore
|
||||||
|
TO PREVENT ACCIDENTAL CRED LEAKS. -->
|
||||||
<!--=============================================================-->
|
<!--=============================================================-->
|
||||||
|
|
||||||
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
|
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue