From ab6a5c9d07a50b49d696b983e1a1cd4f9ef2a44d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Thu, 8 Aug 2019 02:07:15 +0200 Subject: [PATCH] HDDS-1925. ozonesecure acceptance test broken by HTTP auth requirement (#1248) --- .../dist/src/main/compose/ozonesecure/test.sh | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 16 +++++++++++++++- .../dist/src/main/smoketest/basic/basic.robot | 5 ++--- .../src/main/smoketest/basic/ozone-shell.robot | 1 + .../dist/src/main/smoketest/commonlib.robot | 6 +++++- .../src/main/smoketest/s3/commonawslib.robot | 1 + .../dist/src/main/smoketest/s3/webui.robot | 7 ++++--- 7 files changed, 29 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index f13f010a1b2..01106b86154 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -33,7 +33,7 @@ execute_robot_test scm security execute_robot_test scm ozonefs/ozonefs.robot -execute_robot_test scm s3 +execute_robot_test s3g s3 stop_docker_env diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 065c53fef51..462b9fa8544 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -28,6 +28,20 @@ mkdir -p "$RESULT_DIR" #Should be writeable from the docker containers where user is different. chmod ogu+w "$RESULT_DIR" +## @description print the number of datanodes up +## @param the docker-compose file +count_datanodes() { + local compose_file=$1 + + local jmx_url='http://scm:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' + if [[ "${SECURITY_ENABLED}" == 'true' ]]; then + docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/scm@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab && curl --negotiate -u : -s '${jmx_url}'" + else + docker-compose -f "${compose_file}" exec -T scm curl -s "${jmx_url}" + fi \ + | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' +} + ## @description wait until datanodes are up (or 30 seconds) ## @param the docker-compose file ## @param number of datanodes to wait for (default: 3) @@ -43,7 +57,7 @@ wait_for_datanodes(){ #This line checks the number of HEALTHY datanodes registered in scm over the # jmx HTTP servlet - datanodes=$(docker-compose -f "${compose_file}" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value') + datanodes=$(count_datanodes "${compose_file}") if [[ "$datanodes" ]]; then if [[ ${datanodes} -ge ${datanode_count} ]]; then diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot index 88af0976270..c750521baef 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot @@ -25,9 +25,8 @@ ${DATANODE_HOST} datanode *** Test Cases *** Check webui static resources - ${result} = Execute curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js - Should contain ${result} 200 - ${result} = Execute curl -s -I http://om:9874/static/bootstrap-3.3.7/js/bootstrap.min.js + Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit HTTP user + ${result} = Execute curl --negotiate -u : -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js Should contain ${result} 200 Start freon testing diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot index 24e6a7442bb..dc10acce852 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot @@ -17,6 +17,7 @@ Documentation Test ozone shell CLI usage Library OperatingSystem Resource ../commonlib.robot +Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Test Timeout 2 minute *** Variables *** diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot index 1a94d53c18d..88f6c4a9e8e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot @@ -54,8 +54,12 @@ Install aws cli ${rc} ${output} = Run And Return Rc And Output yum --help Run Keyword if '${rc}' == '0' Install aws cli s3 centos +Kinit HTTP user + ${hostname} = Execute hostname + Wait Until Keyword Succeeds 2min 10sec Execute kinit -k HTTP/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab + Kinit test user [arguments] ${user} ${keytab} ${hostname} = Execute hostname Set Suite Variable ${TEST_USER} ${user}/${hostname}@EXAMPLE.COM - Wait Until Keyword Succeeds 2min 10sec Execute kinit -k ${user}/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/${keytab} \ No newline at end of file + Wait Until Keyword Succeeds 2min 10sec Execute kinit -k ${user}/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/${keytab} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index ea953ac9a19..13356354ea4 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -49,6 +49,7 @@ Setup v2 headers Set Environment Variable AWS_SECRET_ACCESS_KEY ANYKEY Setup v4 headers + Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab ${result} = Execute ozone s3 getsecret ${accessKey} = Get Regexp Matches ${result} (?<=awsAccessKey=).* ${accessKey} = Get Variable Value ${accessKey} sdsdasaasdasd diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/webui.robot b/hadoop-ozone/dist/src/main/smoketest/s3/webui.robot index 56f8a83c8c5..74ba4e7869a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/webui.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/webui.robot @@ -27,8 +27,9 @@ ${BUCKET} generated *** Test Cases *** -File upload and directory list - ${result} = Execute curl -v ${ENDPOINT_URL} +S3 Gateway Web UI + Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit HTTP user + ${result} = Execute curl --negotiate -u : -v ${ENDPOINT_URL} Should contain ${result} HTTP/1.1 307 Temporary Redirect - ${result} = Execute curl -v ${ENDPOINT_URL}/static/ + ${result} = Execute curl --negotiate -u : -v ${ENDPOINT_URL}/static/index.html Should contain ${result} Apache Hadoop Ozone S3