From eb3120740227f10f6d198ee614e32ba7aacd810d Mon Sep 17 00:00:00 2001 From: abhagraw <99210446+abhagraw@users.noreply.github.com> Date: Fri, 31 Mar 2023 00:45:53 +0530 Subject: [PATCH] Using MinIO to run S3DeepStorage ITs (#13997) * Using MinIO to S3DeepStorage ITs * Adding S3DeepStorageTest to github actions revised ITs --- .github/workflows/reusable-revised-its.yml | 20 +++++++++++++ .github/workflows/revised-its.yml | 15 ++++++++++ .../cases/cluster/Common/dependencies.yaml | 30 ++++++++++++++++++- .../druid/testsEx/utils/S3TestUtil.java | 11 ++++++- .../cases/templates/S3DeepStorage.py | 25 ++++++++++++++++ it.sh | 2 +- 6 files changed, 100 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 5a220d5755a..4f75c9f5ba1 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -42,9 +42,29 @@ on: required: false type: string default: com.mysql.jdbc.Driver + DRUID_CLOUD_BUCKET: + required: false + type: string + DRUID_CLOUD_PATH: + required: false + type: string + AWS_REGION: + required: false + type: string + AWS_ACCESS_KEY_ID: + required: false + type: string + AWS_SECRET_ACCESS_KEY: + required: false + type: string env: MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly. + DRUID_CLOUD_BUCKET: ${{ inputs.DRUID_CLOUD_BUCKET }} + DRUID_CLOUD_PATH: ${{ inputs.DRUID_CLOUD_PATH }} + AWS_REGION: ${{ inputs.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 jobs: diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 83cad23f42f..576fbd3fb5a 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -36,3 +36,18 @@ jobs: script: ./it.sh github ${{ matrix.it }} it: ${{ matrix.it }} mysql_driver: com.mysql.jdbc.Driver + + s3-deep-storage-minio: + uses: ./.github/workflows/reusable-revised-its.yml + with: + build_jdk: 8 + runtime_jdk: 11 + use_indexer: middleManager + script: ./it.sh github S3DeepStorage + it: S3DeepStorage + mysql_driver: com.mysql.jdbc.Driver + DRUID_CLOUD_BUCKET: druid-qa + DRUID_CLOUD_PATH: aws-${{ github.run_id }}-${{ github.run_attempt }} + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: miniopassword diff --git a/integration-tests-ex/cases/cluster/Common/dependencies.yaml b/integration-tests-ex/cases/cluster/Common/dependencies.yaml index 4eeb85a49cd..1f43dc5a751 100644 --- a/integration-tests-ex/cases/cluster/Common/dependencies.yaml +++ b/integration-tests-ex/cases/cluster/Common/dependencies.yaml @@ -86,7 +86,35 @@ services: MYSQL_USER: druid MYSQL_PASSWORD: diurd -## TODO: Not yet retested + minio: + container_name: minio + command: server /data --console-address ":9001" + networks: + druid-it-net: + ipv4_address: 172.172.172.5 + image: minio/minio:latest + ports: + - '9000:9000' + - '9001:9001' + volumes: + - ${SHARED_DIR}/minio:/data + environment: + - MINIO_ROOT_USER=${AWS_ACCESS_KEY_ID} + - MINIO_ROOT_PASSWORD=${AWS_SECRET_ACCESS_KEY} + + create_minio_buckets: + image: minio/mc + networks: + druid-it-net: + ipv4_address: 172.172.172.6 + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set s3 http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}; + /usr/bin/mc mb s3/${DRUID_CLOUD_BUCKET}; + /usr/bin/mc anonymous set public s3/${DRUID_CLOUD_BUCKET}; + " + + ## TODO: Not yet retested ### Optional supporting infra openldap: diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java index 73c851e428e..26244f78e27 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java @@ -19,9 +19,12 @@ package org.apache.druid.testsEx.utils; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.DeleteObjectsRequest; @@ -81,10 +84,16 @@ public class S3TestUtil private AmazonS3 s3Client() { AWSCredentials credentials = new BasicAWSCredentials(S3_ACCESS_KEY, S3_SECRET_KEY); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProtocol(Protocol.HTTP); return AmazonS3ClientBuilder .standard() .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .withRegion(S3_REGION) + // Setting endpoint to MinIO S3 API endpoint (e.g., "http://localhost:9000") + // configured in integration-tests-ex/cases/cluster/Common/dependencies.yaml + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:9000", S3_REGION)) + .withPathStyleAccessEnabled(true) + .withClientConfiguration(clientConfig) .build(); } diff --git a/integration-tests-ex/cases/templates/S3DeepStorage.py b/integration-tests-ex/cases/templates/S3DeepStorage.py index 9fb85ca6a7f..772ce333d1b 100644 --- a/integration-tests-ex/cases/templates/S3DeepStorage.py +++ b/integration-tests-ex/cases/templates/S3DeepStorage.py @@ -14,6 +14,7 @@ # limitations under the License. from template import BaseTemplate, generate +from template import ZOO_KEEPER, METADATA, OVERLORD class Template(BaseTemplate): @@ -39,8 +40,32 @@ class Template(BaseTemplate): self.add_property(service, 'druid.storage.baseKey', '${DRUID_CLOUD_PATH}') self.add_env(service, 'AWS_REGION', '${AWS_REGION}') + # Adding the following to make druid work with MinIO + # See https://blog.min.io/how-to-druid-superset-minio/ for more details + self.add_property(service, 'druid.s3.protocol', 'http') + self.add_property(service, 'druid.s3.enablePathStyleAccess', 'true') + self.add_property(service, 'druid.s3.endpoint.url', 'http://172.172.172.5:9000/') + + def define_overlord(self): + service = self.define_druid_service(OVERLORD, OVERLORD) + self.add_depends(service, [ZOO_KEEPER, METADATA, "create_minio_buckets"]) + return service + # This test uses different data than the default. def define_data_dir(self, service): self.add_volume(service, '../data', '/resources') + def create_minio_container(self): + return self.define_external_service("minio") + + def create_minio_bucket(self): + service = self.define_external_service("create_minio_buckets") + self.add_depends(service, ["minio"]) + return service + + def define_custom_services(self): + self.create_minio_container() + self.create_minio_bucket() + + generate(__file__, Template()) diff --git a/it.sh b/it.sh index b03b7acb57e..2c6aa86062e 100755 --- a/it.sh +++ b/it.sh @@ -148,7 +148,7 @@ function prepare_docker { } function require_env_var { - if [ -n "$1" ]; then + if [ -z "$1" ]; then echo "$1 must be set for test category $CATEGORY" 1>&2 exit 1 fi