K8s IT Test enhance (#10785)

* do build and stop action in IT

* change base dir from druidHome to druidHome/integration-tests

* add env DRUID_HOME

* bug fix

* modify stop_sh

* ready to test

* bug fix

* modify dir

* tested on dev

* modify dir

* move DRUID_HOME env

* done

Co-authored-by: yuezhang <yuezhang@freewheel.tv>

.travis.yml

@@ -601,14 +601,9 @@ jobs:
       jdk: openjdk8
       services: &integration_test_services_k8s
         - docker
-      env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default
+      env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default BUILD_DRUID_CLSUTER=true
-      before_script:
-        - integration-tests/script/setup_k8s_cluster.sh
-        - integration-tests/script/setup_druid_operator_on_k8s.sh
-        - integration-tests/script/setup_druid_on_k8s.sh
       script: &run_integration_test_k8s
-        - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP}
+        - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLSUTER}
-      after_script: integration-tests/script/stop_k8s_cluster.sh
       after_failure: &integration_test_diags_k8s
         - for v in broker middlemanager router coordinator historical ; do
           echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";

distribution/docker/DockerfileBuildTarAdvanced

@@ -55,6 +55,10 @@ RUN addgroup -S -g 1000 druid \
 COPY --chown=druid:druid --from=builder /opt /opt
 COPY distribution/docker/druid.sh /druid.sh
 
+RUN mkdir -p /opt/druid/var \
+ && chown -R druid:druid /opt \
+ && chmod 775 /opt/druid/var
+
 USER druid
 VOLUME /opt/druid/var
 WORKDIR /opt/druid

integration-tests/pom.xml

@@ -498,8 +498,54 @@
 
         <profile>
             <id>int-tests-config-file</id>
+            <properties>
+                <pod.name>int-test</pod.name>
+                <pod.namespace>default</pod.namespace>
+                <build.druid.cluster>false</build.druid.cluster>
+                <override.config.path />
+                <resource.file.dir.path />
+
+                <!-- Would like to put emojis in here too, but they throw "Input buffer too short" errors due to https://issues.apache.org/jira/browse/SUREFIRE-1865 -->
+                <extra.datasource.name.suffix>\ %Россия\ 한국\ 中国!?</extra.datasource.name.suffix>
+            </properties>
             <build>
                 <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>K8s-package</id>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <phase>pre-integration-test</phase>
+                                <configuration>
+                                    <environmentVariables>
+                                        <POD_NAME>${pod.name}</POD_NAME>
+                                        <POD_NAMESPACE>${pod.namespace}</POD_NAMESPACE>
+                                        <BUILD_DRUID_CLSUTER>${build.druid.cluster}</BUILD_DRUID_CLSUTER>
+                                    </environmentVariables>
+                                    <executable>${project.basedir}/script/build_run_k8s_cluster.sh</executable>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>stop-k8s-druid-cluster</id>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <phase>post-integration-test</phase>
+                                <configuration>
+                                    <environmentVariables>
+                                        <POD_NAME>${pod.name}</POD_NAME>
+                                        <POD_NAMESPACE>${pod.namespace}</POD_NAMESPACE>
+                                        <BUILD_DRUID_CLSUTER>${build.druid.cluster}</BUILD_DRUID_CLSUTER>
+                                    </environmentVariables>
+                                    <executable>${project.basedir}/script/stop_k8s_cluster.sh</executable>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-failsafe-plugin</artifactId>

integration-tests/script/build_run_k8s_cluster.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+if ($BUILD_DRUID_CLSUTER); then
+
+  DRUID_HOME=$(dirname `pwd`)
+  echo "SET DRUID_HOME: $DRUID_HOME"
+  minikubeFile="/usr/local/bin/minikube*"
+  minikubeFile2="/usr/local/bin/minikube"
+
+  if [ -f "$minikubeFile" ] || [ -f "$minikubeFile2" ]; then
+    bash $DRUID_HOME/integration-tests/script/stop_k8s_cluster.sh
+  fi
+
+  cd $DRUID_HOME
+  echo "Start to setup k8s cluster"
+  bash $DRUID_HOME/integration-tests/script/setup_k8s_cluster.sh
+
+  echo "Start to setup druid operator on k8s"
+  bash $DRUID_HOME/integration-tests/script/setup_druid_operator_on_k8s.sh
+
+  echo "Start to setup druid on k8s"
+  bash $DRUID_HOME/integration-tests/script/setup_druid_on_k8s.sh
+fi
+

integration-tests/script/setup_druid_on_k8s.sh

@@ -35,6 +35,7 @@ mvn -B -ff -q dependency:go-offline \
 docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
 
 # This tmp dir is used for MiddleManager pod and Historical Pod to cache segments.
+sudo rm -rf tmp
 mkdir tmp
 chmod 777 tmp
 
@@ -43,7 +44,7 @@ sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml
 $KUBECTL apply -f integration-tests/k8s/tiny-cluster.yaml
 
 # Wait a bit
-sleep 120
+sleep 60
 
 ## Debug And FastFail
 

integration-tests/script/setup_druid_operator_on_k8s.sh

@@ -21,6 +21,7 @@ export KUBECTL="sudo /usr/local/bin/kubectl"
 
 
 # Prepare For Druid-Operator
+rm -rf druid-operator
 git clone https://github.com/druid-io/druid-operator.git
 cd druid-operator
 git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION

integration-tests/script/stop_k8s_cluster.sh

@@ -15,5 +15,16 @@
 # limitations under the License.
 
 set -e
+
+if ($BUILD_DRUID_CLSUTER); then
+
+  DRUID_HOME=$(dirname `pwd`)
+  echo "SET DRUID_HOME: $DRUID_HOME"
+
   sudo /usr/local/bin/minikube delete
-sudo rm -rf `pwd`/tmp
+  sudo rm -rf $DRUID_HOME/tmp
+  sudo rm -rf $DRUID_HOME/druid-operator
+  docker ps
+
+fi
+