druid/integration-tests/docker/docker-compose.base.yml

402 lines
10 KiB
YAML
Raw Normal View History

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: "2.2"
# IP address ranges:
# 172.172.172.2: zookeeper + kafka (dockerfile depends on this)
# 172.172.172.3: metadata store
# 172.172.172.10-19: overlord
# 172.172.172.20-29: coordinator
# 172.172.172.30-39: historicals
# 172.172.172.40-49: middle managers
# 172.172.172.50-59: indexers
# 172.172.172.60-69: brokers
# 172.172.172.70-79: routers
# 172.172.172.101+: hadoop, other supporting infra
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
### always there supporting infra:
druid-zookeeper-kafka:
image: druid/cluster
container_name: druid-zookeeper-kafka
ports:
- 2181:2181
- 9092:9092
- 9093:9093
networks:
druid-it-net:
## Dockerfile depends on this address for this container it seems...
ipv4_address: 172.172.172.2
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/zookeeper.conf:/usr/lib/druid/conf/zookeeper.conf
- ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
env_file:
- ./environment-configs/common
environment:
- ZK_VERSION
druid-metadata-storage:
image: druid/cluster
container_name: druid-metadata-storage
ports:
- 3306:3306
networks:
druid-it-net:
ipv4_address: 172.172.172.3
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/metadata-storage.conf:/usr/lib/druid/conf/metadata-storage.conf
env_file:
- ./environment-configs/common
### overlords
druid-overlord:
image: druid/cluster
container_name: druid-overlord
hostname: druid-overlord
networks:
druid-it-net:
ipv4_address: 172.172.172.10
ports:
- 8090:8090
- 8290:8290
- 5009:5009
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/overlord
druid-overlord-two:
image: druid/cluster
container_name: druid-overlord-two
hostname: druid-overlord-two
networks:
druid-it-net:
ipv4_address: 172.172.172.11
ports:
- 8590:8090
- 8790:8290
- 6009:5009
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/overlord
### coordinators
druid-coordinator:
image: druid/cluster
container_name: druid-coordinator
hostname: druid-coordinator
networks:
druid-it-net:
ipv4_address: 172.172.172.20
ports:
- 8081:8081
- 8281:8281
- 5006:5006
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/coordinator
druid-coordinator-two:
image: druid/cluster
container_name: druid-coordinator-two
hostname: druid-coordinator-two
networks:
druid-it-net:
ipv4_address: 172.172.172.21
ports:
- 8581:8081
- 8781:8281
- 6006:5006
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/coordinator
### historicals
druid-historical:
image: druid/cluster
container_name: druid-historical
hostname: druid-historical
networks:
druid-it-net:
ipv4_address: 172.172.172.30
ports:
- 8083:8083
- 8283:8283
- 5007:5007
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/historical
### middle managers
druid-middlemanager:
image: druid/cluster
container_name: druid-middlemanager
hostname: druid-middlemanager
networks:
druid-it-net:
ipv4_address: 172.172.172.40
ports:
- 5008:5008
- 8091:8091
- 8291:8291
- 8100:8100
- 8101:8101
- 8102:8102
- 8103:8103
- 8104:8104
- 8105:8105
- 8300:8300
- 8301:8301
- 8302:8302
- 8303:8303
- 8304:8304
- 8305:8305
privileged: true
volumes:
- ./../src/test/resources:/resources
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/middlemanager
### indexers
Fixes and tests related to the Indexer process. (#10631) * Fixes and tests related to the Indexer process. Three bugs fixed: 1) Indexers would not announce themselves as segment servers if they did not have storage locations defined. This used to work, but was broken in #9971. Fixed this by adding an "isSegmentServer" method to ServerType and updating SegmentLoadDropHandler to always announce if this method returns true. 2) Certain batch task types were written in a way that assumed "isReady" would be called before "run", which is not guaranteed. In particular, they relied on it in order to initialize "taskLockHelper". Fixed this by updating AbstractBatchIndexTask to ensure "isReady" is called before "run" for these tasks. 3) UnifiedIndexerAppenderatorsManager did not properly handle complex datasources. Introduced DataSourceAnalysis in order to fix this. Test changes: 1) Add a new "docker-compose.cli-indexer.yml" config that spins up an Indexer instead of a MiddleManager. 2) Introduce a "USE_INDEXER" environment variable that determines if docker-compose will start up an Indexer or a MiddleManager. 3) Duplicate all the jdk8 tests and run them in both MiddleManager and Indexer mode. 4) Various adjustments to encourage fail-fast errors in the Docker build scripts. 5) Various adjustments to speed up integration tests and reduce memory usage. 6) Add another Mac-specific approach to determining a machine's own IP. This was useful on my development machine. 7) Update segment-count check in ITCompactionTaskTest to eliminate a race condition (it was looking for 6 segments, which only exist together briefly, until the older 4 are marked unused). Javadoc updates: 1) AbstractBatchIndexTask: Added javadocs to determineLockGranularityXXX that make it clear when taskLockHelper will be initialized as a side effect. (Related to the second bug above.) 2) Task: Clarified that "isReady" is not guaranteed to be called before "run". It was already implied, but now it's explicit. 3) ZkCoordinator: Clarified deprecation message. 4) DataSegmentServerAnnouncer: Clarified deprecation message. * Fix stop_cluster script. * Fix sanity check in script. * Fix hashbang lines. * Test and doc adjustments. * Additional tests, and adjustments for tests. * Split ITs back out. * Revert change to druid_coordinator_period_indexingPeriod. * Set Indexer capacity to match MM. * Bump up Historical memory. * Bump down coordinator, overlord memory. * Bump up Broker memory.
2020-12-08 19:02:26 -05:00
druid-indexer:
image: druid/cluster
container_name: druid-indexer
hostname: druid-indexer
Fixes and tests related to the Indexer process. (#10631) * Fixes and tests related to the Indexer process. Three bugs fixed: 1) Indexers would not announce themselves as segment servers if they did not have storage locations defined. This used to work, but was broken in #9971. Fixed this by adding an "isSegmentServer" method to ServerType and updating SegmentLoadDropHandler to always announce if this method returns true. 2) Certain batch task types were written in a way that assumed "isReady" would be called before "run", which is not guaranteed. In particular, they relied on it in order to initialize "taskLockHelper". Fixed this by updating AbstractBatchIndexTask to ensure "isReady" is called before "run" for these tasks. 3) UnifiedIndexerAppenderatorsManager did not properly handle complex datasources. Introduced DataSourceAnalysis in order to fix this. Test changes: 1) Add a new "docker-compose.cli-indexer.yml" config that spins up an Indexer instead of a MiddleManager. 2) Introduce a "USE_INDEXER" environment variable that determines if docker-compose will start up an Indexer or a MiddleManager. 3) Duplicate all the jdk8 tests and run them in both MiddleManager and Indexer mode. 4) Various adjustments to encourage fail-fast errors in the Docker build scripts. 5) Various adjustments to speed up integration tests and reduce memory usage. 6) Add another Mac-specific approach to determining a machine's own IP. This was useful on my development machine. 7) Update segment-count check in ITCompactionTaskTest to eliminate a race condition (it was looking for 6 segments, which only exist together briefly, until the older 4 are marked unused). Javadoc updates: 1) AbstractBatchIndexTask: Added javadocs to determineLockGranularityXXX that make it clear when taskLockHelper will be initialized as a side effect. (Related to the second bug above.) 2) Task: Clarified that "isReady" is not guaranteed to be called before "run". It was already implied, but now it's explicit. 3) ZkCoordinator: Clarified deprecation message. 4) DataSegmentServerAnnouncer: Clarified deprecation message. * Fix stop_cluster script. * Fix sanity check in script. * Fix hashbang lines. * Test and doc adjustments. * Additional tests, and adjustments for tests. * Split ITs back out. * Revert change to druid_coordinator_period_indexingPeriod. * Set Indexer capacity to match MM. * Bump up Historical memory. * Bump down coordinator, overlord memory. * Bump up Broker memory.
2020-12-08 19:02:26 -05:00
networks:
druid-it-net:
ipv4_address: 172.172.172.50
Fixes and tests related to the Indexer process. (#10631) * Fixes and tests related to the Indexer process. Three bugs fixed: 1) Indexers would not announce themselves as segment servers if they did not have storage locations defined. This used to work, but was broken in #9971. Fixed this by adding an "isSegmentServer" method to ServerType and updating SegmentLoadDropHandler to always announce if this method returns true. 2) Certain batch task types were written in a way that assumed "isReady" would be called before "run", which is not guaranteed. In particular, they relied on it in order to initialize "taskLockHelper". Fixed this by updating AbstractBatchIndexTask to ensure "isReady" is called before "run" for these tasks. 3) UnifiedIndexerAppenderatorsManager did not properly handle complex datasources. Introduced DataSourceAnalysis in order to fix this. Test changes: 1) Add a new "docker-compose.cli-indexer.yml" config that spins up an Indexer instead of a MiddleManager. 2) Introduce a "USE_INDEXER" environment variable that determines if docker-compose will start up an Indexer or a MiddleManager. 3) Duplicate all the jdk8 tests and run them in both MiddleManager and Indexer mode. 4) Various adjustments to encourage fail-fast errors in the Docker build scripts. 5) Various adjustments to speed up integration tests and reduce memory usage. 6) Add another Mac-specific approach to determining a machine's own IP. This was useful on my development machine. 7) Update segment-count check in ITCompactionTaskTest to eliminate a race condition (it was looking for 6 segments, which only exist together briefly, until the older 4 are marked unused). Javadoc updates: 1) AbstractBatchIndexTask: Added javadocs to determineLockGranularityXXX that make it clear when taskLockHelper will be initialized as a side effect. (Related to the second bug above.) 2) Task: Clarified that "isReady" is not guaranteed to be called before "run". It was already implied, but now it's explicit. 3) ZkCoordinator: Clarified deprecation message. 4) DataSegmentServerAnnouncer: Clarified deprecation message. * Fix stop_cluster script. * Fix sanity check in script. * Fix hashbang lines. * Test and doc adjustments. * Additional tests, and adjustments for tests. * Split ITs back out. * Revert change to druid_coordinator_period_indexingPeriod. * Set Indexer capacity to match MM. * Bump up Historical memory. * Bump down coordinator, overlord memory. * Bump up Broker memory.
2020-12-08 19:02:26 -05:00
ports:
- 5008:5008
- 8091:8091
- 8291:8291
privileged: true
volumes:
- ./../src/test/resources:/resources
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/indexer
### brokers
druid-broker:
image: druid/cluster
hostname: druid-broker
container_name: druid-broker
networks:
druid-it-net:
ipv4_address: 172.172.172.60
ports:
- 5005:5005
- 8082:8082
- 8282:8282
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/broker
### routers
druid-router:
image: druid/cluster
container_name: druid-router
hostname: druid-router
networks:
druid-it-net:
ipv4_address: 172.172.172.70
ports:
- 5004:5004
- 8888:8888
- 9088:9088
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/router
druid-router-permissive-tls:
image: druid/cluster
container_name: druid-router-permissive-tls
networks:
druid-it-net:
ipv4_address: 172.172.172.71
ports:
- 5001:5001
- 8889:8889
- 9089:9089
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/router-permissive-tls
druid-router-no-client-auth-tls:
image: druid/cluster
container_name: druid-router-no-client-auth-tls
networks:
druid-it-net:
ipv4_address: 172.172.172.72
ports:
- 5002:5002
- 8890:8890
- 9090:9090
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/router-no-client-auth-tls
druid-router-custom-check-tls:
image: druid/cluster
container_name: druid-router-custom-check-tls
networks:
druid-it-net:
ipv4_address: 172.172.172.73
ports:
- 5003:5003
- 8891:8891
- 9091:9091
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf
env_file:
- ./environment-configs/common
- ./environment-configs/router-custom-check-tls
### optional supporting infra
druid-it-hadoop:
image: druid-it/hadoop:2.8.5
container_name: druid-it-hadoop
ports:
- 2049:2049
- 2122:2122
- 8020:8020
- 8021:8021
- 8030:8030
- 8031:8031
- 8032:8032
- 8033:8033
- 8040:8040
- 8042:8042
- 8088:8088
- 8443:8443
- 9000:9000
- 10020:10020
- 19888:19888
- 34455:34455
- 50010:50010
- 50020:50020
- 50030:50030
- 50060:50060
- 50070:50070
- 50075:50075
- 50090:50090
- 51111:51111
networks:
druid-it-net:
ipv4_address: 172.172.172.101
privileged: true
volumes:
- ${HOME}/shared:/shared
- ./../src/test/resources:/resources
hostname: "druid-it-hadoop"
command: "bash -c 'echo Start druid-it-hadoop container... && \
/etc/bootstrap.sh && \
tail -f /dev/null'"
druid-openldap:
image: osixia/openldap:1.4.0
container_name: druid-openldap
networks:
druid-it-net:
ipv4_address: 172.172.172.102
ports:
- 8389:389
- 8636:636
privileged: true
volumes:
- ./ldap-configs/bootstrap.ldif:/container/service/slapd/assets/config/bootstrap/ldif/bootstrap.ldif
- ${HOME}/shared:/shared
env_file:
- ./environment-configs/common
command: --copy-service
schema-registry:
image: confluentinc/cp-schema-registry:5.5.1
container_name: schema-registry
ports:
- 8085:8085
networks:
druid-it-net:
ipv4_address: 172.172.172.103
volumes:
- ${HOME}/shared:/shared
- ./schema-registry/jaas_config.file:/usr/lib/druid/conf/jaas_config.file
- ./schema-registry/password-file:/usr/lib/druid/conf/password-file
privileged: true
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8085"
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: druid-zookeeper-kafka:9092
SCHEMA_REGISTRY_AUTHENTICATION_METHOD: BASIC
SCHEMA_REGISTRY_AUTHENTICATION_REALM: druid
SCHEMA_REGISTRY_AUTHENTICATION_ROLES: users
SCHEMA_REGISTRY_OPTS: -Djava.security.auth.login.config=/usr/lib/druid/conf/jaas_config.file