Use K3S instead of minikube for integration tests (#13782)

We are seeing failures on GHA while using minikube so switching to K3S instead.

.github/workflows/standard-its.yml

@@ -100,6 +100,54 @@ jobs:
       override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
       group: custom coordinator duties
 
+  integration-k8s-leadership-tests:
+    name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
+    runs-on: ubuntu-22.04
+    env:
+      MVN: mvn --no-snapshot-updates
+      MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
+      CONFIG_FILE: k8s_run_config_file.json
+      IT_TEST: -Dit.test=ITNestedQueryPushDownTest
+      POD_NAME: int-test
+      POD_NAMESPACE: default
+      BUILD_DRUID_CLUSTER: true
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v3
+
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: 8
+
+      - name: Restore Maven repository
+        id: maven-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: ~/.m2/repository
+          key: maven-${{ runner.os }}-8-${{ github.sha }}
+
+      - name: Maven build
+        if: steps.maven-restore.outputs.cache-hit != 'true'
+        run: |
+          ./it.sh ci
+
+      - name: Run IT
+        id: test
+        run: |
+          # Debug echo
+          echo "MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}"
+          MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER}
+
+      - name: Debug on failure
+        if: ${{ failure() && steps.test.conclusion == 'failure' }}
+        run: |
+          for v in broker middlemanager router coordinator historical ; do
+          echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";
+          sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0;
+          done
+
   integration-other-tests:
     strategy:
       fail-fast: false

integration-tests/script/build_run_k8s_cluster.sh

@@ -20,10 +20,9 @@ if ($BUILD_DRUID_CLSUTER); then
 
   DRUID_HOME=$(dirname `pwd`)
   echo "SET DRUID_HOME: $DRUID_HOME"
-  minikubeFile="/usr/local/bin/minikube*"
-  minikubeFile2="/usr/local/bin/minikube"
+  k3sFile="/usr/local/bin/k3s*"
 
-  if [ -f "$minikubeFile" ] || [ -f "$minikubeFile2" ]; then
+  if [ -f "$k3sFile" ]; then
     bash $DRUID_HOME/integration-tests/script/stop_k8s_cluster.sh
   fi
 

integration-tests/script/setup_druid_on_k8s.sh

@@ -16,7 +16,7 @@
 
 set -e
 
-export KUBECTL="sudo /usr/local/bin/kubectl"
+export KUBECTL="/usr/local/bin/kubectl"
 
 # setup client keystore
 cd integration-tests

integration-tests/script/setup_druid_operator_on_k8s.sh

@@ -17,8 +17,7 @@
 set -e
 
 export DRUID_OPERATOR_VERSION=0.0.3
-export KUBECTL="sudo /usr/local/bin/kubectl"
-
+export KUBECTL="/usr/local/bin/kubectl"
 
 # Prepare For Druid-Operator
 rm -rf druid-operator

integration-tests/script/setup_k8s_cluster.sh

@@ -16,19 +16,13 @@
 
 set -e
 
-# Set Necessary ENV
-export CHANGE_MINIKUBE_NONE_USER=true
-export MINIKUBE_WANTUPDATENOTIFICATION=false
-export MINIKUBE_WANTREPORTERRORPROMPT=false
-export MINIKUBE_HOME=$HOME
+export INSTALL_K3S_VERSION=v1.21.14+k3s1
 export KUBECONFIG=$HOME/.kube/config
 
-sudo apt install -y conntrack
-
-# Lacunch K8S cluster
+# Launch K8S cluster
 curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-sudo /usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1
-sudo /usr/local/bin/minikube update-context
-
+curl -sfL https://get.k3s.io | bash /dev/stdin  --docker
+mkdir -p $HOME/.kube
+sudo cp /etc/rancher/k3s/k3s.yaml $KUBECONFIG
+sudo chmod 777 $KUBECONFIG
 echo "Setup K8S Cluster Done!"

integration-tests/script/stop_k8s_cluster.sh

@@ -21,7 +21,7 @@ if ($BUILD_DRUID_CLSUTER); then
   DRUID_HOME=$(dirname `pwd`)
   echo "SET DRUID_HOME: $DRUID_HOME"
 
-  sudo /usr/local/bin/minikube delete
+  /usr/local/bin/k3s-uninstall.sh
   sudo rm -rf $DRUID_HOME/tmp
   sudo rm -rf $DRUID_HOME/druid-operator
   docker ps