BAEL-2321
This commit is contained in:
Markus Gulden
parent
8c9f493571
commit
c39fe1c85c
|
|
@ -1,88 +0,0 @@
|
|||
##
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
##
|
||||
|
||||
# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended
|
||||
# to be used with the examples, and some settings may differ from those used in a production system, especially
|
||||
# the `bootstrap.servers` and those specifying replication factors.
|
||||
|
||||
# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster.
|
||||
bootstrap.servers=localhost:9092
|
||||
|
||||
# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs
|
||||
group.id=connect-cluster
|
||||
|
||||
# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
|
||||
# need to configure these based on the format they want their data in when loaded from or stored into Kafka
|
||||
key.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
value.converter=org.apache.kafka.connect.json.JsonConverter
|
||||
# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
|
||||
# it to
|
||||
key.converter.schemas.enable=true
|
||||
value.converter.schemas.enable=true
|
||||
|
||||
# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted.
|
||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
offset.storage.topic=connect-offsets
|
||||
offset.storage.replication.factor=1
|
||||
#offset.storage.partitions=25
|
||||
|
||||
# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated,
|
||||
# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
config.storage.topic=connect-configs
|
||||
config.storage.replication.factor=1
|
||||
|
||||
# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted.
|
||||
# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create
|
||||
# the topic before starting Kafka Connect if a specific topic configuration is needed.
|
||||
# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value.
|
||||
# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able
|
||||
# to run this example on a single-broker cluster and so here we instead set the replication factor to 1.
|
||||
status.storage.topic=connect-status
|
||||
status.storage.replication.factor=1
|
||||
#status.storage.partitions=5
|
||||
|
||||
# Flush much faster than normal, which is useful for testing/debugging
|
||||
offset.flush.interval.ms=10000
|
||||
|
||||
# These are provided to inform the user about the presence of the REST host and port configs
|
||||
# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests.
|
||||
#rest.host.name=
|
||||
#rest.port=8083
|
||||
|
||||
# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers.
|
||||
#rest.advertised.host.name=
|
||||
#rest.advertised.port=
|
||||
|
||||
# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
|
||||
# (connectors, converters, transformations). The list should consist of top level directories that include
|
||||
# any combination of:
|
||||
# a) directories immediately containing jars with plugins and their dependencies
|
||||
# b) uber-jars with plugins and their dependencies
|
||||
# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
|
||||
# Examples:
|
||||
# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
|
||||
# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a
|
||||
# directory other than the home directory of Confluent Platform.
|
||||
plugin.path=./share/java
|
||||
|
|
@ -1,22 +1,14 @@
|
|||
{
|
||||
"firstName": "John",
|
||||
"lastName": "Smith",
|
||||
"age": 25,
|
||||
"address": {
|
||||
"streetAddress": "21 2nd Street",
|
||||
"city": "New York",
|
||||
"state": "NY",
|
||||
"postalCode": "10021"
|
||||
},
|
||||
"phoneNumber": [{
|
||||
"type": "home",
|
||||
"number": "212 555-1234"
|
||||
}, {
|
||||
"type": "fax",
|
||||
"number": "646 555-4567"
|
||||
}
|
||||
],
|
||||
"gender": {
|
||||
"type": "male"
|
||||
"name": "mongodb-sink",
|
||||
"config": {
|
||||
"connector.class": "at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector",
|
||||
"tasks.max": 1,
|
||||
"topics": "connect-custom",
|
||||
"mongodb.connection.uri": "mongodb://mongo-db/test?retryWrites=true",
|
||||
"mongodb.collection": "MyCollection",
|
||||
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||
"key.converter.schemas.enable": false,
|
||||
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
|
||||
"value.converter.schemas.enable": false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@
|
|||
"config": {
|
||||
"connector.class": "io.confluent.connect.mqtt.MqttSourceConnector",
|
||||
"tasks.max": 1,
|
||||
"mqtt.server.uri": "ws://broker.hivemq.com:8000/mqtt",
|
||||
"mqtt.server.uri": "tcp://mosquitto:1883",
|
||||
"mqtt.topics": "baeldung",
|
||||
"kafka.topic": "connect-custom",
|
||||
"value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter"
|
||||
"value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
|
||||
"confluent.topic.bootstrap.servers": "kafka:9092",
|
||||
"confluent.topic.replication.factor": 1
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
version: '3.3'
|
||||
|
||||
services:
|
||||
mosquitto:
|
||||
image: eclipse-mosquitto:1.5.5
|
||||
hostname: mosquitto
|
||||
container_name: mosquitto
|
||||
expose:
|
||||
- "1883"
|
||||
ports:
|
||||
- "1883:1883"
|
||||
zookeeper:
|
||||
image: zookeeper:3.4.9
|
||||
restart: unless-stopped
|
||||
hostname: zookeeper
|
||||
container_name: zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
ZOO_MY_ID: 1
|
||||
ZOO_PORT: 2181
|
||||
ZOO_SERVERS: server.1=zookeeper:2888:3888
|
||||
volumes:
|
||||
- ./zookeeper/data:/data
|
||||
- ./zookeeper/datalog:/datalog
|
||||
kafka:
|
||||
image: confluentinc/cp-kafka:5.1.0
|
||||
hostname: kafka
|
||||
container_name: kafka
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
||||
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
volumes:
|
||||
- ./kafka/data:/var/lib/kafka/data
|
||||
depends_on:
|
||||
- zookeeper
|
||||
kafka-connect:
|
||||
image: confluentinc/cp-kafka-connect:5.1.0
|
||||
hostname: kafka-connect
|
||||
container_name: kafka-connect
|
||||
ports:
|
||||
- "8083:8083"
|
||||
environment:
|
||||
CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
|
||||
CONNECT_REST_ADVERTISED_HOST_NAME: connect
|
||||
CONNECT_REST_PORT: 8083
|
||||
CONNECT_GROUP_ID: compose-connect-group
|
||||
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
|
||||
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
|
||||
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
|
||||
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
|
||||
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
|
||||
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
|
||||
CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars'
|
||||
CONNECT_CONFLUENT_TOPIC_REPLICATION_FACTOR: 1
|
||||
volumes:
|
||||
- /tmp/custom/jars:/etc/kafka-connect/jars
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- kafka
|
||||
- mosquitto
|
||||
mongo-db:
|
||||
image: mongo:4.0.5
|
||||
hostname: mongo-db
|
||||
container_name: mongo-db
|
||||
expose:
|
||||
- "27017"
|
||||
ports:
|
||||
- "27017:27017"
|
||||
command: --bind_ip_all --smallfiles
|
||||
volumes:
|
||||
- ./mongo-db:/data
|
||||
mongoclient:
|
||||
image: mongoclient/mongoclient:2.2.0
|
||||
container_name: mongoclient
|
||||
hostname: mongoclient
|
||||
depends_on:
|
||||
- mongo-db
|
||||
ports:
|
||||
- 3000:3000
|
||||
environment:
|
||||
MONGO_URL: "mongodb://mongo-db:27017"
|
||||
PORT: 3000
|
||||
expose:
|
||||
- "3000"
|
||||
Loading…
Reference in New Issue