diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 0ce4e8fc2ec..6178dfe72aa 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -122,3 +122,6 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" . run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/blockade" . run cp -p -r "${ROOT}/hadoop-ozone/dist/target/k8s" kubernetes run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/Dockerfile" . + +#workaround for https://issues.apache.org/jira/browse/MRESOURCES-236 +find ./compose -name "*.sh" -exec chmod 755 {} \; diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index 2c88745282e..e90efcaf9a9 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -140,7 +140,7 @@ 3.1.0 - copy-compose + copy-compose-files compile copy-resources diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh new file mode 100755 index 00000000000..1c901751956 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +#Due to the limitation of the current auditparser test, it should be the +#first test in a clean cluster. + +execute_robot_test scm auditparser + +execute_robot_test scm basic/basic.robot + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/test.sh b/hadoop-ozone/dist/src/main/compose/ozonefs/test.sh new file mode 100755 index 00000000000..5ca756957c3 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/test.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +execute_robot_test scm ozonefs/ozonefs.robot + + +## TODO: As of now the o3fs tests are unstable. + +export OZONE_HOME=/opt/ozone + +#execute_robot_test hadoop32 ozonefs/hadoopo3fs.robot + +#execute_robot_test hadoop31 ozonefs/hadoopo3fs.robot + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh b/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh new file mode 100755 index 00000000000..f4bfcc3d57f --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +execute_robot_test scm basic/basic.robot + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh new file mode 100755 index 00000000000..0160da9e382 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +execute_robot_test scm basic/basic.robot + +execute_robot_test scm s3 + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh new file mode 100755 index 00000000000..f13f010a1b2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +export SECURITY_ENABLED=true + +start_docker_env + +execute_robot_test scm kinit.robot + +execute_robot_test scm basic + +execute_robot_test scm security + +execute_robot_test scm ozonefs/ozonefs.robot + +execute_robot_test scm s3 + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh new file mode 100755 index 00000000000..225acecd470 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/test-all.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Test executor to test all the compose/*/test.sh test scripts. +# + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd ) +ALL_RESULT_DIR="$SCRIPT_DIR/result" + +mkdir -p "$ALL_RESULT_DIR" +rm "$ALL_RESULT_DIR/*" + +RESULT=0 +IFS=$'\n' +# shellcheck disable=SC2044 +for test in $(find $SCRIPT_DIR -name test.sh); do + echo "Executing test in $(dirname "$test")" + + #required to read the .env file from the right location + cd "$(dirname "$test")" || continue + $test + ret=$? + if [[ $ret -ne 0 ]]; then + RESULT=-1 + fi + RESULT_DIR="$(dirname "$test")/result" + cp "$RESULT_DIR"/robot-*.xml "$ALL_RESULT_DIR" +done + +docker run --rm -v "$SCRIPT_DIR/result:/opt/result" apache/hadoop-runner rebot -N "smoketests" -d "/opt/result" "/opt/result/robot-*.xml" +exit $RESULT diff --git a/hadoop-ozone/dist/src/main/compose/test-single.sh b/hadoop-ozone/dist/src/main/compose/test-single.sh new file mode 100755 index 00000000000..f1203d30775 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/test-single.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Single test executor, can start a single robot test in any running container. +# + + +COMPOSE_DIR="$PWD" +export COMPOSE_DIR + +if [[ ! -f "$COMPOSE_DIR/docker-compose.yaml" ]]; then + echo "docker-compose.yaml is missing from the current dir. Please run this command from a docker-compose environment." + exit 1 +fi +if (( $# != 2 )); then +cat << EOF + Single test executor + + Usage: + + ../test-single.sh + + container: Name of the running docker-compose container (docker-compose.yaml is required in the current directory) + + robot_test: name of the robot test or directory relative to the smoketest dir. + + + +EOF + +fi + +# shellcheck source=testlib.sh +source "$COMPOSE_DIR/../testlib.sh" + +execute_robot_test "$1" "$2" + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh new file mode 100755 index 00000000000..0870c872829 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR") +COMPOSE_FILE=$COMPOSE_DIR/docker-compose.yaml +RESULT_DIR="$COMPOSE_DIR/result" +RESULT_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/compose/$(basename "$COMPOSE_ENV_NAME")/result" +SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest" + +#delete previous results +rm -rf "$RESULT_DIR" +mkdir -p "$RESULT_DIR" +#Should be writeable from the docker containers where user is different. +chmod ogu+w "$RESULT_DIR" + +## @description wait until 3 datanodes are up (or 30 seconds) +## @param the docker-compose file +wait_for_datanodes(){ + + #Reset the timer + SECONDS=0 + + #Don't give it up until 30 seconds + while [[ $SECONDS -lt 90 ]]; do + + #This line checks the number of HEALTHY datanodes registered in scm over the + # jmx HTTP servlet + datanodes=$(docker-compose -f "$1" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value') + if [[ "$datanodes" == "3" ]]; then + + #It's up and running. Let's return from the function. + echo "$datanodes datanodes are up and registered to the scm" + return + else + + #Print it only if a number. Could be not a number if scm is not yet started + if [[ "$datanodes" ]]; then + echo "$datanodes datanode is up and healthy (until now)" + fi + fi + + sleep 2 + done + echo "WARNING! Datanodes are not started successfully. Please check the docker-compose files" +} + +## @description Starts a docker-compose based test environment +start_docker_env(){ + docker-compose -f "$COMPOSE_FILE" down + docker-compose -f "$COMPOSE_FILE" up -d --scale datanode=3 + wait_for_datanodes "$COMPOSE_FILE" + sleep 10 +} + +## @description Execute robot tests in a specific container. +## @param Name of the container in the docker-compose file +## @param robot test file or directory relative to the smoketest dir +execute_robot_test(){ + CONTAINER="$1" + TEST="$2" + TEST_NAME=$(basename "$TEST") + TEST_NAME=${TEST_NAME%.*} + set +e + OUTPUT_NAME="$COMPOSE_ENV_NAME-$TEST_NAME-$CONTAINER" + docker-compose -f "$COMPOSE_FILE" exec -e SECURITY_ENABLED="${SECURITY_ENABLED}" -T "$CONTAINER" python -m robot --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$RESULT_DIR_INSIDE/robot-$OUTPUT_NAME.xml" "$SMOKETEST_DIR_INSIDE/$TEST" + set -e + +} + +## @description Stops a docker-compose based test environment (with saving the logs) +stop_docker_env(){ + docker-compose -f "$COMPOSE_FILE" logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log" + if [ "${KEEP_RUNNING:-false}" = false ]; then + docker-compose -f "$COMPOSE_FILE" down + fi +} + +## @description Generate robot framework reports based on the saved results. +generate_report(){ + #Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output) + docker run --rm -v "$DIR/..:${OZONE_DIR:-/opt/hadoop}" apache/hadoop-runner rebot -d "$RESULT_DIR_INSIDE" "$RESULT_DIR_INSIDE/robot-*.xml" +} diff --git a/hadoop-ozone/dist/src/main/smoketest/README.md b/hadoop-ozone/dist/src/main/smoketest/README.md index c521a54beb2..d181b8a3da6 100644 --- a/hadoop-ozone/dist/src/main/smoketest/README.md +++ b/hadoop-ozone/dist/src/main/smoketest/README.md @@ -27,4 +27,37 @@ The argument of the `robot` could be any robot file or directory. The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment. -The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters. +# Run tests in docker environment + +In the ./compose folder there are additional test scripts to make it easy to run all tests or run a specific test in a docker environment. + +## Test one environment + +Go to the compose directory and execute the test.sh directly from there: + +``` +cd compose/ozone +./test.sh +``` + +The results will be saved to the `compose/ozone/results` + +## Run all the tests + +``` +cd compose +./test-all.sh +``` + +The results will be combined to the `compose/results` folder. + +## Run one specific test case + +Start the compose environment and execute test: + +``` +cd compose/ozone +docker-compose up -d +#wait.... +../test-single.sh scm basic/basic.robot +``` \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/kinit.robot b/hadoop-ozone/dist/src/main/smoketest/kinit.robot new file mode 100644 index 00000000000..00693e1a50c --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/kinit.robot @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Kinit test user +Library OperatingSystem +Resource commonlib.robot +Test Timeout 2 minute + + +*** Test Cases *** +Kinit + Kinit test user diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot new file mode 100644 index 00000000000..1915e366512 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone fs with hadoopfs +Library OperatingSystem +Library String +Resource ../commonlib.robot + +*** Variables *** +${DATANODE_HOST} datanode +${PREFIX} ozone + +*** Test cases *** + +Test hadoop dfs + ${random} = Generate Random String 5 [NUMBERS] + ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.vol1/${PREFIX}-${random} + ${result} = Execute hdfs dfs -ls o3fs://bucket1.vol1/ + Should contain ${PREFIX}-${random} diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh index 0fa7a341ad7..b2cdfc31bc8 100755 --- a/hadoop-ozone/dist/src/main/smoketest/test.sh +++ b/hadoop-ozone/dist/src/main/smoketest/test.sh @@ -14,154 +14,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" RESULT_DIR=result #delete previous results rm -rf "${DIR:?}/$RESULT_DIR" -mkdir -p "$DIR/$RESULT_DIR" -#Should be writeable from the docker containers where user is different. -chmod ogu+w "$DIR/$RESULT_DIR" -## @description wait until 3 datanodes are up (or 30 seconds) -## @param the docker-compose file -wait_for_datanodes(){ +REPLACEMENT="$DIR/../compose/test-all.sh" +echo "THIS SCRIPT IS DEPRECATED. Please use $REPLACEMENT instead." - #Reset the timer - SECONDS=0 +${REPLACEMENT} - #Don't give it up until 30 seconds - while [[ $SECONDS -lt 30 ]]; do - - #This line checks the number of HEALTHY datanodes registered in scm over the - # jmx HTTP servlet - datanodes=$(docker-compose -f "$1" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value') - if [[ "$datanodes" == "3" ]]; then - - #It's up and running. Let's return from the function. - echo "$datanodes datanodes are up and registered to the scm" - return - else - - #Print it only if a number. Could be not a number if scm is not yet started - if [[ "$datanodes" ]]; then - echo "$datanodes datanode is up and healthy (until now)" - fi - fi - - sleep 2 - done - echo "WARNING! Datanodes are not started successfully. Please check the docker-compose files" -} - -## @description Execute selected test suites in a specified docker-compose engironment -## @param the name of the docker-compose env relative to ../compose -## @param the name of the tests (array of subdir names of the dir of this script) -execute_tests(){ - COMPOSE_DIR=$1 - COMPOSE_FILE=$DIR/../compose/$COMPOSE_DIR/docker-compose.yaml - TESTS=$2 - echo "-------------------------------------------------" - echo "Executing test(s): [${TESTS[*]}]" - echo "" - echo " Cluster type: $COMPOSE_DIR" - echo " Compose file: $COMPOSE_FILE" - echo " Output dir: $DIR/$RESULT_DIR" - echo " Command to rerun: ./test.sh --keep --env $COMPOSE_DIR $TESTS" - echo "-------------------------------------------------" - if [ ${COMPOSE_DIR} == "ozonesecure" ]; then - SECURITY_ENABLED="true" - else - SECURITY_ENABLED="false" - fi - - docker-compose -f "$COMPOSE_FILE" down - docker-compose -f "$COMPOSE_FILE" up -d --scale datanode=3 - wait_for_datanodes "$COMPOSE_FILE" - #TODO: we need to wait for the OM here - sleep 10 - for TEST in "${TESTS[@]}"; do - TITLE="Ozone $TEST tests with $COMPOSE_DIR cluster" - set +e - OUTPUT_NAME="$COMPOSE_DIR-${TEST//\//_}" - docker-compose -f "$COMPOSE_FILE" exec -e SECURITY_ENABLED="${SECURITY_ENABLED}" -T om python -m robot --log NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "smoketest/$RESULT_DIR/robot-$OUTPUT_NAME.xml" --logtitle "$TITLE" --reporttitle "$TITLE" "smoketest/$TEST" - set -e - docker-compose -f "$COMPOSE_FILE" logs > "$DIR/$RESULT_DIR/docker-$OUTPUT_NAME.log" - done - if [ "$KEEP_RUNNING" = false ]; then - docker-compose -f "$COMPOSE_FILE" down - fi -} -RUN_ALL=true -KEEP_RUNNING=false -POSITIONAL=() -while [[ $# -gt 0 ]] -do -key="$1" - -case $key in - --env) - DOCKERENV="$2" - RUN_ALL=false - shift # past argument - shift # past value - ;; - --keep) - KEEP_RUNNING=true - shift # past argument - ;; - --help|-h|-help) - cat << EOF - - Acceptance test executor for ozone. - - This is a lightweight test executor for ozone. - - You can run it with - - ./test.sh - - Which executes all the tests in all the available environments. - - Or you can run manually one test with - - ./test.sh --keep --env ozone-hdfs basic - - --keep means that docker cluster won't be stopped after the test (optional) - --env defines the subdirectory under the compose dir - The remaining parameters define the test suites under smoketest dir. - Could be any directory or robot file relative to the smoketest dir. -EOF - exit 0 - ;; - *) - POSITIONAL+=("$1") # save it in an array for later - shift # past argument - ;; -esac -done - -if [ "$RUN_ALL" = true ]; then -# -# This is the definition of the ozone acceptance test suite -# -# We select the test suites and execute them on multiple type of clusters -# - TESTS=("basic") - execute_tests ozone "${TESTS[@]}" - TESTS=("auditparser") - execute_tests ozone "${TESTS[@]}" - TESTS=("ozonefs") - execute_tests ozonefs "${TESTS[@]}" - TESTS=("basic") - execute_tests ozone-hdfs "${TESTS[@]}" - TESTS=("s3") - execute_tests ozones3 "${TESTS[@]}" - TESTS=("security") - execute_tests ozonesecure . -else - execute_tests "$DOCKERENV" "${POSITIONAL[@]}" -fi - -#Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output) -docker run --rm -v "$DIR/..:/opt/hadoop" apache/hadoop-runner rebot -d "smoketest/$RESULT_DIR" "smoketest/$RESULT_DIR/robot-*.xml" +cp -r "$DIR/../compose/result" "$DIR"