From 77946f92649c8deba307f99eaf5320d6e76fe390 Mon Sep 17 00:00:00 2001 From: zhangyue19921010 <69956021+zhangyue19921010@users.noreply.github.com> Date: Tue, 2 Feb 2021 07:48:42 +0800 Subject: [PATCH] K8s IT Test enhance (#10785) * do build and stop action in IT * change base dir from druidHome to druidHome/integration-tests * add env DRUID_HOME * bug fix * modify stop_sh * ready to test * bug fix * modify dir * tested on dev * modify dir * move DRUID_HOME env * done Co-authored-by: yuezhang --- .travis.yml | 9 +--- .../docker/DockerfileBuildTarAdvanced | 4 ++ integration-tests/pom.xml | 46 +++++++++++++++++++ .../script/build_run_k8s_cluster.sh | 40 ++++++++++++++++ .../script/setup_druid_on_k8s.sh | 3 +- .../script/setup_druid_operator_on_k8s.sh | 1 + integration-tests/script/stop_k8s_cluster.sh | 15 +++++- 7 files changed, 108 insertions(+), 10 deletions(-) create mode 100755 integration-tests/script/build_run_k8s_cluster.sh diff --git a/.travis.yml b/.travis.yml index d5c9391d556..ffb6c1dbd99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -601,14 +601,9 @@ jobs: jdk: openjdk8 services: &integration_test_services_k8s - docker - env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default - before_script: - - integration-tests/script/setup_k8s_cluster.sh - - integration-tests/script/setup_druid_operator_on_k8s.sh - - integration-tests/script/setup_druid_on_k8s.sh + env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default BUILD_DRUID_CLSUTER=true script: &run_integration_test_k8s - - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} - after_script: integration-tests/script/stop_k8s_cluster.sh + - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLSUTER} after_failure: &integration_test_diags_k8s - for v in broker middlemanager router coordinator historical ; do echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; diff --git a/distribution/docker/DockerfileBuildTarAdvanced b/distribution/docker/DockerfileBuildTarAdvanced index a58087f4e51..0c04cd2bf42 100644 --- a/distribution/docker/DockerfileBuildTarAdvanced +++ b/distribution/docker/DockerfileBuildTarAdvanced @@ -55,6 +55,10 @@ RUN addgroup -S -g 1000 druid \ COPY --chown=druid:druid --from=builder /opt /opt COPY distribution/docker/druid.sh /druid.sh +RUN mkdir -p /opt/druid/var \ + && chown -R druid:druid /opt \ + && chmod 775 /opt/druid/var + USER druid VOLUME /opt/druid/var WORKDIR /opt/druid diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index ddcdaaa3c04..b47bb9cc2ec 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -498,8 +498,54 @@ int-tests-config-file + + int-test + default + false + + + + + \ %Россия\ 한국\ 中国!? + + + org.codehaus.mojo + exec-maven-plugin + + + K8s-package + + exec + + pre-integration-test + + + ${pod.name} + ${pod.namespace} + ${build.druid.cluster} + + ${project.basedir}/script/build_run_k8s_cluster.sh + + + + stop-k8s-druid-cluster + + exec + + post-integration-test + + + ${pod.name} + ${pod.namespace} + ${build.druid.cluster} + + ${project.basedir}/script/stop_k8s_cluster.sh + + + + org.apache.maven.plugins maven-failsafe-plugin diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh new file mode 100755 index 00000000000..c2ac8e5fea6 --- /dev/null +++ b/integration-tests/script/build_run_k8s_cluster.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if ($BUILD_DRUID_CLSUTER); then + + DRUID_HOME=$(dirname `pwd`) + echo "SET DRUID_HOME: $DRUID_HOME" + minikubeFile="/usr/local/bin/minikube*" + minikubeFile2="/usr/local/bin/minikube" + + if [ -f "$minikubeFile" ] || [ -f "$minikubeFile2" ]; then + bash $DRUID_HOME/integration-tests/script/stop_k8s_cluster.sh + fi + + cd $DRUID_HOME + echo "Start to setup k8s cluster" + bash $DRUID_HOME/integration-tests/script/setup_k8s_cluster.sh + + echo "Start to setup druid operator on k8s" + bash $DRUID_HOME/integration-tests/script/setup_druid_operator_on_k8s.sh + + echo "Start to setup druid on k8s" + bash $DRUID_HOME/integration-tests/script/setup_druid_on_k8s.sh +fi + diff --git a/integration-tests/script/setup_druid_on_k8s.sh b/integration-tests/script/setup_druid_on_k8s.sh index aab018f238b..d5118dd9409 100755 --- a/integration-tests/script/setup_druid_on_k8s.sh +++ b/integration-tests/script/setup_druid_on_k8s.sh @@ -35,6 +35,7 @@ mvn -B -ff -q dependency:go-offline \ docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced . # This tmp dir is used for MiddleManager pod and Historical Pod to cache segments. +sudo rm -rf tmp mkdir tmp chmod 777 tmp @@ -43,7 +44,7 @@ sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml $KUBECTL apply -f integration-tests/k8s/tiny-cluster.yaml # Wait a bit -sleep 120 +sleep 60 ## Debug And FastFail diff --git a/integration-tests/script/setup_druid_operator_on_k8s.sh b/integration-tests/script/setup_druid_operator_on_k8s.sh index 39284197663..d99ec6a2bf6 100755 --- a/integration-tests/script/setup_druid_operator_on_k8s.sh +++ b/integration-tests/script/setup_druid_operator_on_k8s.sh @@ -21,6 +21,7 @@ export KUBECTL="sudo /usr/local/bin/kubectl" # Prepare For Druid-Operator +rm -rf druid-operator git clone https://github.com/druid-io/druid-operator.git cd druid-operator git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION diff --git a/integration-tests/script/stop_k8s_cluster.sh b/integration-tests/script/stop_k8s_cluster.sh index 6e8969bd12f..ff9e2e94a75 100755 --- a/integration-tests/script/stop_k8s_cluster.sh +++ b/integration-tests/script/stop_k8s_cluster.sh @@ -15,5 +15,16 @@ # limitations under the License. set -e -sudo /usr/local/bin/minikube delete -sudo rm -rf `pwd`/tmp + +if ($BUILD_DRUID_CLSUTER); then + + DRUID_HOME=$(dirname `pwd`) + echo "SET DRUID_HOME: $DRUID_HOME" + + sudo /usr/local/bin/minikube delete + sudo rm -rf $DRUID_HOME/tmp + sudo rm -rf $DRUID_HOME/druid-operator + docker ps + +fi +