Fix flaky Revised ITs failures on GHA runners (#14348)

* Fix read timed out failures and remove containers before test

* remove containers before loading images

* add labels to IT docker containers, download stable minio docker image release instead of latest

.github/workflows/reusable-revised-its.yml

@@ -66,6 +66,8 @@ env:
   AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
   SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+  DOCKER_CLIENT_TIMEOUT: 120
+  COMPOSE_HTTP_TIMEOUT: 120
 
 jobs:
   test: # GitHub job that runs a given revised/new IT against retrieved cached druid docker image
@@ -118,6 +120,12 @@ jobs:
           echo $DRUID_IT_IMAGE_NAME
           docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ inputs.build_jdk }}.tar.gz
 
+      - name: Stop and remove docker containers
+        run: |
+          echo "Force stopping all containers and pruning"
+          docker ps -aq --filter "label=druid-int-test=true" | xargs -r docker rm -f
+          docker system prune -af --volumes
+
       - name: Load docker image
         run: |
           docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz

integration-tests-ex/cases/cluster/Common/dependencies.yaml

@@ -27,6 +27,8 @@ services:
   zookeeper:
     image: zookeeper:${ZK_VERSION}
     container_name: zookeeper
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.4
@@ -42,6 +44,8 @@ services:
   kafka:
     image: bitnami/kafka:${KAFKA_VERSION}
     container_name: kafka
+    labels:
+      druid-int-test: "true"
     ports:
       - 9092:9092
       - 9093:9093
@@ -71,6 +75,8 @@ services:
     # platform: linux/x86_64
     image: mysql:$MYSQL_IMAGE_VERSION
     container_name: metadata
+    labels:
+      druid-int-test: "true"
     restart: always
     command:
       - --character-set-server=utf8mb4
@@ -89,11 +95,13 @@ services:
 
   minio:
     container_name: minio
+    labels:
+      druid-int-test: "true"
     command: server /data --console-address ":9001"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.5
-    image: minio/minio:latest
+    image: minio/minio:RELEASE.2023-05-27T05-56-19Z
     ports:
       - '9000:9000'
       - '9001:9001'
@@ -121,6 +129,8 @@ services:
   openldap:
     image: osixia/openldap:1.4.0
     container_name: openldap
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.102
@@ -135,6 +145,8 @@ services:
   schema-registry:
     image: confluentinc/cp-schema-registry:5.5.1
     container_name: schema-registry
+    labels:
+      druid-int-test: "true"
     ports:
       - 8085:8085
     networks:
@@ -157,6 +169,8 @@ services:
     ## Giving fake version
     image: druid-it/hadoop:9.9.9
     container_name: druid-it-hadoop
+    labels:
+      druid-int-test: "true"
     ports:
       - 2049:2049
       - 2122:2122

integration-tests-ex/cases/cluster/Common/druid.yaml

@@ -48,6 +48,8 @@ services:
   overlord:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: overlord
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.10
@@ -67,6 +69,8 @@ services:
   coordinator:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: coordinator
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.20
@@ -86,6 +90,8 @@ services:
   historical:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: historical
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.30
@@ -105,6 +111,8 @@ services:
   middlemanager:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: middlemanager
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.40
@@ -136,6 +144,8 @@ services:
   indexer:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: indexer
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.50
@@ -155,6 +165,8 @@ services:
   broker:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: broker
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.60
@@ -174,6 +186,8 @@ services:
   router:
     image: ${DRUID_IT_IMAGE_NAME}
     container_name: router
+    labels:
+      druid-int-test: "true"
     networks:
       druid-it-net:
         ipv4_address: 172.172.172.70