BAEL-2321
This commit is contained in:
Markus Gulden
parent
8c9f493571
commit
c39fe1c85c
|
|
@ -1,88 +0,0 @@
|
||||||
##
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
##
|
|
||||||
|
|
||||||
# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended
|
|
||||||
# to be used with the examples, and some settings may differ from those used in a production system, especially
|
|
||||||
# the `bootstrap.servers` and those specifying replication factors.
|
|
||||||
|
|
||||||
# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster.
|
|
||||||
bootstrap.servers=localhost:9092
|
|
||||||
|
|
||||||
# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs
|
|
||||||
group.id=connect-cluster
|
|
||||||
|
|
||||||
# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
|
|
||||||
# need to configure these based on the format they want their data in when loaded from or stored into Kafka
|
|
||||||
key.converter=org.apache.kafka.connect.json.JsonConverter
|
|
||||||
value.converter=org.apache.kafka.connect.json.JsonConverter
|
|
||||||
# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
|
|
||||||
# it to
|
|
||||||
key.converter.schemas.enable=true
|
|
||||||
value.converter.schemas.enable=true
|
|
||||||
|
|
||||||
# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted.
|
|
||||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
|
||||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
|
||||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
|
||||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
|
||||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
|
||||||
offset.storage.topic=connect-offsets
|
|
||||||
offset.storage.replication.factor=1
|
|
||||||
#offset.storage.partitions=25
|
|
||||||
|
|
||||||
# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated,
|
|
||||||
# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
|
||||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
|
||||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
|
||||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
|
||||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
|
||||||
config.storage.topic=connect-configs
|
|
||||||
config.storage.replication.factor=1
|
|
||||||
|
|
||||||
# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted.
|
|
||||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
|
||||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
|
||||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
|
||||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
|
||||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
|
||||||
status.storage.topic=connect-status
|
|
||||||
status.storage.replication.factor=1
|
|
||||||
#status.storage.partitions=5
|
|
||||||
|
|
||||||
# Flush much faster than normal, which is useful for testing/debugging
|
|
||||||
offset.flush.interval.ms=10000
|
|
||||||
|
|
||||||
# These are provided to inform the user about the presence of the REST host and port configs
|
|
||||||
# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests.
|
|
||||||
#rest.host.name=
|
|
||||||
#rest.port=8083
|
|
||||||
|
|
||||||
# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers.
|
|
||||||
#rest.advertised.host.name=
|
|
||||||
#rest.advertised.port=
|
|
||||||
|
|
||||||
# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
|
|
||||||
# (connectors, converters, transformations). The list should consist of top level directories that include
|
|
||||||
# any combination of:
|
|
||||||
# a) directories immediately containing jars with plugins and their dependencies
|
|
||||||
# b) uber-jars with plugins and their dependencies
|
|
||||||
# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
|
|
||||||
# Examples:
|
|
||||||
# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
|
|
||||||
# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a
|
|
||||||
# directory other than the home directory of Confluent Platform.
|
|
||||||
plugin.path=./share/java
|
|
||||||
|
|
@ -1,22 +1,14 @@
|
||||||
{
|
{
|
||||||
"firstName": "John",
|
"name": "mongodb-sink",
|
||||||
"lastName": "Smith",
|
"config": {
|
||||||
"age": 25,
|
"connector.class": "at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector",
|
||||||
"address": {
|
"tasks.max": 1,
|
||||||
"streetAddress": "21 2nd Street",
|
"topics": "connect-custom",
|
||||||
"city": "New York",
|
"mongodb.connection.uri": "mongodb://mongo-db/test?retryWrites=true",
|
||||||
"state": "NY",
|
"mongodb.collection": "MyCollection",
|
||||||
"postalCode": "10021"
|
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||||
},
|
"key.converter.schemas.enable": false,
|
||||||
"phoneNumber": [{
|
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||||
"type": "home",
|
"value.converter.schemas.enable": false
|
||||||
"number": "212 555-1234"
|
|
||||||
}, {
|
|
||||||
"type": "fax",
|
|
||||||
"number": "646 555-4567"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"gender": {
|
|
||||||
"type": "male"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,11 @@
|
||||||
"config": {
|
"config": {
|
||||||
"connector.class": "io.confluent.connect.mqtt.MqttSourceConnector",
|
"connector.class": "io.confluent.connect.mqtt.MqttSourceConnector",
|
||||||
"tasks.max": 1,
|
"tasks.max": 1,
|
||||||
"mqtt.server.uri": "ws://broker.hivemq.com:8000/mqtt",
|
"mqtt.server.uri": "tcp://mosquitto:1883",
|
||||||
"mqtt.topics": "baeldung",
|
"mqtt.topics": "baeldung",
|
||||||
"kafka.topic": "connect-custom",
|
"kafka.topic": "connect-custom",
|
||||||
"value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter"
|
"value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
|
||||||
|
"confluent.topic.bootstrap.servers": "kafka:9092",
|
||||||
|
"confluent.topic.replication.factor": 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
version: '3.3'
|
||||||
|
|
||||||
|
services:
|
||||||
|
mosquitto:
|
||||||
|
image: eclipse-mosquitto:1.5.5
|
||||||
|
hostname: mosquitto
|
||||||
|
container_name: mosquitto
|
||||||
|
expose:
|
||||||
|
- "1883"
|
||||||
|
ports:
|
||||||
|
- "1883:1883"
|
||||||
|
zookeeper:
|
||||||
|
image: zookeeper:3.4.9
|
||||||
|
restart: unless-stopped
|
||||||
|
hostname: zookeeper
|
||||||
|
container_name: zookeeper
|
||||||
|
ports:
|
||||||
|
- "2181:2181"
|
||||||
|
environment:
|
||||||
|
ZOO_MY_ID: 1
|
||||||
|
ZOO_PORT: 2181
|
||||||
|
ZOO_SERVERS: server.1=zookeeper:2888:3888
|
||||||
|
volumes:
|
||||||
|
- ./zookeeper/data:/data
|
||||||
|
- ./zookeeper/datalog:/datalog
|
||||||
|
kafka:
|
||||||
|
image: confluentinc/cp-kafka:5.1.0
|
||||||
|
hostname: kafka
|
||||||
|
container_name: kafka
|
||||||
|
ports:
|
||||||
|
- "9092:9092"
|
||||||
|
environment:
|
||||||
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||||
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
||||||
|
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
|
||||||
|
KAFKA_BROKER_ID: 1
|
||||||
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||||
|
volumes:
|
||||||
|
- ./kafka/data:/var/lib/kafka/data
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
kafka-connect:
|
||||||
|
image: confluentinc/cp-kafka-connect:5.1.0
|
||||||
|
hostname: kafka-connect
|
||||||
|
container_name: kafka-connect
|
||||||
|
ports:
|
||||||
|
- "8083:8083"
|
||||||
|
environment:
|
||||||
|
CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
|
||||||
|
CONNECT_REST_ADVERTISED_HOST_NAME: connect
|
||||||
|
CONNECT_REST_PORT: 8083
|
||||||
|
CONNECT_GROUP_ID: compose-connect-group
|
||||||
|
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
|
||||||
|
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
|
||||||
|
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
|
||||||
|
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||||
|
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||||
|
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||||
|
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||||
|
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
|
||||||
|
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
|
||||||
|
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
|
||||||
|
CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars'
|
||||||
|
CONNECT_CONFLUENT_TOPIC_REPLICATION_FACTOR: 1
|
||||||
|
volumes:
|
||||||
|
- /tmp/custom/jars:/etc/kafka-connect/jars
|
||||||
|
depends_on:
|
||||||
|
- zookeeper
|
||||||
|
- kafka
|
||||||
|
- mosquitto
|
||||||
|
mongo-db:
|
||||||
|
image: mongo:4.0.5
|
||||||
|
hostname: mongo-db
|
||||||
|
container_name: mongo-db
|
||||||
|
expose:
|
||||||
|
- "27017"
|
||||||
|
ports:
|
||||||
|
- "27017:27017"
|
||||||
|
command: --bind_ip_all --smallfiles
|
||||||
|
volumes:
|
||||||
|
- ./mongo-db:/data
|
||||||
|
mongoclient:
|
||||||
|
image: mongoclient/mongoclient:2.2.0
|
||||||
|
container_name: mongoclient
|
||||||
|
hostname: mongoclient
|
||||||
|
depends_on:
|
||||||
|
- mongo-db
|
||||||
|
ports:
|
||||||
|
- 3000:3000
|
||||||
|
environment:
|
||||||
|
MONGO_URL: "mongodb://mongo-db:27017"
|
||||||
|
PORT: 3000
|
||||||
|
expose:
|
||||||
|
- "3000"
|
||||||
Loading…
Reference in New Issue