mirror of https://github.com/apache/druid.git
Using MinIO to run S3DeepStorage ITs (#13997)
* Using MinIO to S3DeepStorage ITs * Adding S3DeepStorageTest to github actions revised ITs
This commit is contained in:
abhagraw
GitHub
parent
47face9ca9
commit
eb31207402
|
|
@ -42,9 +42,29 @@ on:
|
|||
required: false
|
||||
type: string
|
||||
default: com.mysql.jdbc.Driver
|
||||
DRUID_CLOUD_BUCKET:
|
||||
required: false
|
||||
type: string
|
||||
DRUID_CLOUD_PATH:
|
||||
required: false
|
||||
type: string
|
||||
AWS_REGION:
|
||||
required: false
|
||||
type: string
|
||||
AWS_ACCESS_KEY_ID:
|
||||
required: false
|
||||
type: string
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly.
|
||||
DRUID_CLOUD_BUCKET: ${{ inputs.DRUID_CLOUD_BUCKET }}
|
||||
DRUID_CLOUD_PATH: ${{ inputs.DRUID_CLOUD_PATH }}
|
||||
AWS_REGION: ${{ inputs.AWS_REGION }}
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -36,3 +36,18 @@ jobs:
|
|||
script: ./it.sh github ${{ matrix.it }}
|
||||
it: ${{ matrix.it }}
|
||||
mysql_driver: com.mysql.jdbc.Driver
|
||||
|
||||
s3-deep-storage-minio:
|
||||
uses: ./.github/workflows/reusable-revised-its.yml
|
||||
with:
|
||||
build_jdk: 8
|
||||
runtime_jdk: 11
|
||||
use_indexer: middleManager
|
||||
script: ./it.sh github S3DeepStorage
|
||||
it: S3DeepStorage
|
||||
mysql_driver: com.mysql.jdbc.Driver
|
||||
DRUID_CLOUD_BUCKET: druid-qa
|
||||
DRUID_CLOUD_PATH: aws-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
AWS_REGION: us-east-1
|
||||
AWS_ACCESS_KEY_ID: admin
|
||||
AWS_SECRET_ACCESS_KEY: miniopassword
|
||||
|
|
|
|||
|
|
@ -86,7 +86,35 @@ services:
|
|||
MYSQL_USER: druid
|
||||
MYSQL_PASSWORD: diurd
|
||||
|
||||
## TODO: Not yet retested
|
||||
minio:
|
||||
container_name: minio
|
||||
command: server /data --console-address ":9001"
|
||||
networks:
|
||||
druid-it-net:
|
||||
ipv4_address: 172.172.172.5
|
||||
image: minio/minio:latest
|
||||
ports:
|
||||
- '9000:9000'
|
||||
- '9001:9001'
|
||||
volumes:
|
||||
- ${SHARED_DIR}/minio:/data
|
||||
environment:
|
||||
- MINIO_ROOT_USER=${AWS_ACCESS_KEY_ID}
|
||||
- MINIO_ROOT_PASSWORD=${AWS_SECRET_ACCESS_KEY}
|
||||
|
||||
create_minio_buckets:
|
||||
image: minio/mc
|
||||
networks:
|
||||
druid-it-net:
|
||||
ipv4_address: 172.172.172.6
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
/usr/bin/mc alias set s3 http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY};
|
||||
/usr/bin/mc mb s3/${DRUID_CLOUD_BUCKET};
|
||||
/usr/bin/mc anonymous set public s3/${DRUID_CLOUD_BUCKET};
|
||||
"
|
||||
|
||||
## TODO: Not yet retested
|
||||
### Optional supporting infra
|
||||
|
||||
openldap:
|
||||
|
|
|
|||
|
|
@ -19,9 +19,12 @@
|
|||
|
||||
package org.apache.druid.testsEx.utils;
|
||||
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.Protocol;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
|
||||
|
|
@ -81,10 +84,16 @@ public class S3TestUtil
|
|||
private AmazonS3 s3Client()
|
||||
{
|
||||
AWSCredentials credentials = new BasicAWSCredentials(S3_ACCESS_KEY, S3_SECRET_KEY);
|
||||
ClientConfiguration clientConfig = new ClientConfiguration();
|
||||
clientConfig.setProtocol(Protocol.HTTP);
|
||||
return AmazonS3ClientBuilder
|
||||
.standard()
|
||||
.withCredentials(new AWSStaticCredentialsProvider(credentials))
|
||||
.withRegion(S3_REGION)
|
||||
// Setting endpoint to MinIO S3 API endpoint (e.g., "http://localhost:9000")
|
||||
// configured in integration-tests-ex/cases/cluster/Common/dependencies.yaml
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:9000", S3_REGION))
|
||||
.withPathStyleAccessEnabled(true)
|
||||
.withClientConfiguration(clientConfig)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
from template import BaseTemplate, generate
|
||||
from template import ZOO_KEEPER, METADATA, OVERLORD
|
||||
|
||||
class Template(BaseTemplate):
|
||||
|
||||
|
|
@ -39,8 +40,32 @@ class Template(BaseTemplate):
|
|||
self.add_property(service, 'druid.storage.baseKey', '${DRUID_CLOUD_PATH}')
|
||||
self.add_env(service, 'AWS_REGION', '${AWS_REGION}')
|
||||
|
||||
# Adding the following to make druid work with MinIO
|
||||
# See https://blog.min.io/how-to-druid-superset-minio/ for more details
|
||||
self.add_property(service, 'druid.s3.protocol', 'http')
|
||||
self.add_property(service, 'druid.s3.enablePathStyleAccess', 'true')
|
||||
self.add_property(service, 'druid.s3.endpoint.url', 'http://172.172.172.5:9000/')
|
||||
|
||||
def define_overlord(self):
|
||||
service = self.define_druid_service(OVERLORD, OVERLORD)
|
||||
self.add_depends(service, [ZOO_KEEPER, METADATA, "create_minio_buckets"])
|
||||
return service
|
||||
|
||||
# This test uses different data than the default.
|
||||
def define_data_dir(self, service):
|
||||
self.add_volume(service, '../data', '/resources')
|
||||
|
||||
def create_minio_container(self):
|
||||
return self.define_external_service("minio")
|
||||
|
||||
def create_minio_bucket(self):
|
||||
service = self.define_external_service("create_minio_buckets")
|
||||
self.add_depends(service, ["minio"])
|
||||
return service
|
||||
|
||||
def define_custom_services(self):
|
||||
self.create_minio_container()
|
||||
self.create_minio_bucket()
|
||||
|
||||
|
||||
generate(__file__, Template())
|
||||
|
|
|
|||
Loading…
Reference in New Issue