Using MinIO to run S3DeepStorage ITs (#13997)

* Using MinIO to S3DeepStorage ITs

* Adding S3DeepStorageTest to github actions revised ITs

.github/workflows/reusable-revised-its.yml

@@ -42,9 +42,29 @@ on:
         required: false
         type: string
         default: com.mysql.jdbc.Driver
+      DRUID_CLOUD_BUCKET:
+        required: false
+        type: string
+      DRUID_CLOUD_PATH:
+        required: false
+        type: string
+      AWS_REGION:
+        required: false
+        type: string
+      AWS_ACCESS_KEY_ID:
+        required: false
+        type: string
+      AWS_SECRET_ACCESS_KEY:
+        required: false
+        type: string
 
 env:
   MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly.
+  DRUID_CLOUD_BUCKET: ${{ inputs.DRUID_CLOUD_BUCKET }}
+  DRUID_CLOUD_PATH: ${{ inputs.DRUID_CLOUD_PATH }}
+  AWS_REGION: ${{ inputs.AWS_REGION }}
+  AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
   SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
 
 jobs:

.github/workflows/revised-its.yml

@@ -36,3 +36,18 @@ jobs:
       script: ./it.sh github ${{ matrix.it }}
       it: ${{ matrix.it }}
       mysql_driver: com.mysql.jdbc.Driver
+
+  s3-deep-storage-minio:
+    uses: ./.github/workflows/reusable-revised-its.yml
+    with:
+      build_jdk: 8
+      runtime_jdk: 11
+      use_indexer: middleManager
+      script: ./it.sh github S3DeepStorage
+      it: S3DeepStorage
+      mysql_driver: com.mysql.jdbc.Driver
+      DRUID_CLOUD_BUCKET: druid-qa
+      DRUID_CLOUD_PATH: aws-${{ github.run_id }}-${{ github.run_attempt }}
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: admin
+      AWS_SECRET_ACCESS_KEY: miniopassword

integration-tests-ex/cases/cluster/Common/dependencies.yaml

@@ -86,7 +86,35 @@ services:
       MYSQL_USER: druid
       MYSQL_PASSWORD: diurd
 
-## TODO: Not yet retested
+  minio:
+    container_name: minio
+    command: server /data --console-address ":9001"
+    networks:
+      druid-it-net:
+        ipv4_address: 172.172.172.5
+    image: minio/minio:latest
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    volumes:
+      - ${SHARED_DIR}/minio:/data
+    environment:
+      - MINIO_ROOT_USER=${AWS_ACCESS_KEY_ID}
+      - MINIO_ROOT_PASSWORD=${AWS_SECRET_ACCESS_KEY}
+
+  create_minio_buckets:
+    image: minio/mc
+    networks:
+      druid-it-net:
+        ipv4_address: 172.172.172.6
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set s3 http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY};
+      /usr/bin/mc mb s3/${DRUID_CLOUD_BUCKET};
+      /usr/bin/mc anonymous set public s3/${DRUID_CLOUD_BUCKET};
+      "
+
+  ## TODO: Not yet retested
 ### Optional supporting infra
 
   openldap:

integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java

@@ -19,9 +19,12 @@
 
 package org.apache.druid.testsEx.utils;
 
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -81,10 +84,16 @@ public class S3TestUtil
   private AmazonS3 s3Client()
   {
     AWSCredentials credentials = new BasicAWSCredentials(S3_ACCESS_KEY, S3_SECRET_KEY);
+    ClientConfiguration clientConfig = new ClientConfiguration();
+    clientConfig.setProtocol(Protocol.HTTP);
     return AmazonS3ClientBuilder
         .standard()
         .withCredentials(new AWSStaticCredentialsProvider(credentials))
-        .withRegion(S3_REGION)
+        // Setting endpoint to MinIO S3 API endpoint (e.g., "http://localhost:9000")
+        // configured in integration-tests-ex/cases/cluster/Common/dependencies.yaml
+        .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:9000", S3_REGION))
+        .withPathStyleAccessEnabled(true)
+        .withClientConfiguration(clientConfig)
         .build();
   }
 

integration-tests-ex/cases/templates/S3DeepStorage.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from template import BaseTemplate, generate
+from template import ZOO_KEEPER, METADATA, OVERLORD
 
 class Template(BaseTemplate):
 
@@ -39,8 +40,32 @@ class Template(BaseTemplate):
         self.add_property(service, 'druid.storage.baseKey', '${DRUID_CLOUD_PATH}')
         self.add_env(service, 'AWS_REGION', '${AWS_REGION}')
 
+        # Adding the following to make druid work with MinIO
+        # See https://blog.min.io/how-to-druid-superset-minio/ for more details
+        self.add_property(service, 'druid.s3.protocol', 'http')
+        self.add_property(service, 'druid.s3.enablePathStyleAccess', 'true')
+        self.add_property(service, 'druid.s3.endpoint.url', 'http://172.172.172.5:9000/')
+
+    def define_overlord(self):
+        service = self.define_druid_service(OVERLORD, OVERLORD)
+        self.add_depends(service, [ZOO_KEEPER, METADATA, "create_minio_buckets"])
+        return service
+
     # This test uses different data than the default.
     def define_data_dir(self, service):
         self.add_volume(service, '../data', '/resources')
 
+    def create_minio_container(self):
+        return self.define_external_service("minio")
+
+    def create_minio_bucket(self):
+        service = self.define_external_service("create_minio_buckets")
+        self.add_depends(service, ["minio"])
+        return service
+
+    def define_custom_services(self):
+        self.create_minio_container()
+        self.create_minio_bucket()
+
+
 generate(__file__, Template())

it.sh

@@ -148,7 +148,7 @@ function prepare_docker {
 }
 
 function require_env_var {
-	if [ -n "$1" ]; then
+	if [ -z "$1" ]; then
 	  echo "$1 must be set for test category $CATEGORY" 1>&2
 	  exit 1
     fi