iSharkFly-Docs
/
java-tutorials
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5190 from dev-chirag/master 

BAEL-1426 Kafka Transactional API - achieving Exactly Once
This commit is contained in:
José Carlos Valero Sánchez 2018-09-12 09:04:37 +02:00 committed by GitHub
parent 7b256aa060 6f2d31b26a
commit e1689b6f31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 111 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libraries/pom.xml
View File

@ -933,7 +933,7 @@
<caffeine.version>2.5.5</caffeine.version>
<google-api.version>1.23.0</google-api.version>
<google-sheets.version>v4-rev493-1.21.0</google-sheets.version>
<kafka.version>1.0.0</kafka.version>
<kafka.version>2.0.0</kafka.version>
<smooks.version>1.7.0</smooks.version>
<docker.version>3.0.14</docker.version>
<async.http.client.version>2.2.0</async.http.client.version>

102
libraries/src/main/java/com/baeldung/kafka/TransactionalApp.java Normal file
View File

@ -0,0 +1,102 @@
package com.baeldung.kafka;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.TopicPartition;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singleton;
import static org.apache.kafka.clients.consumer.ConsumerConfig.*;
import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG;
import static org.apache.kafka.clients.producer.ProducerConfig.*;
public class TransactionalApp {
private static final String CONSUMER_GROUP_ID = "test";
private static final String OUTPUT_TOPIC = "output";
private static final String INPUT_TOPIC = "input";
public static void main(String[] args) {
KafkaConsumer<String, String> consumer = initConsumer();
KafkaProducer<String, String> producer = initProducer();
producer.initTransactions();
try {
while (true) {
ConsumerRecords<String, String> records = consumer.poll(ofSeconds(20));
producer.beginTransaction();
for (ConsumerRecord record : records)
producer.send(new ProducerRecord(OUTPUT_TOPIC, record));
Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();
for (TopicPartition partition : records.partitions()) {
List<ConsumerRecord<String, String>> partitionedRecords = records.records(partition);
long offset = partitionedRecords.get(partitionedRecords.size() - 1).offset();
offsetsToCommit.put(partition, new OffsetAndMetadata(offset));
}
producer.sendOffsetsToTransaction(offsetsToCommit, CONSUMER_GROUP_ID);
producer.commitTransaction();
}
} catch (KafkaException e) {
producer.abortTransaction();
}
}
private static KafkaConsumer<String, String> initConsumer() {
Properties props = new Properties();
props.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(GROUP_ID_CONFIG, CONSUMER_GROUP_ID);
props.put(ENABLE_AUTO_COMMIT_CONFIG, "false");
props.put(KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
consumer.subscribe(singleton(INPUT_TOPIC));
return consumer;
}
private static KafkaProducer<String, String> initProducer() {
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ACKS_CONFIG, "all");
props.put(RETRIES_CONFIG, 3);
props.put(BATCH_SIZE_CONFIG, 16384);
props.put(LINGER_MS_CONFIG, 1);
props.put(BUFFER_MEMORY_CONFIG, 33554432);
props.put(ENABLE_IDEMPOTENCE_CONFIG, "true");
props.put(TRANSACTIONAL_ID_CONFIG, "prod-1");
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
return new KafkaProducer(props);
}
}

14
libraries/src/test/java/com/baeldung/kafkastreams/KafkaStreamsLiveTest.java
View File

@ -4,10 +4,12 @@ import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KStreamBuilder;
import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.test.TestUtils;
import org.junit.Ignore;
import org.junit.Test;
@ -36,20 +38,20 @@ public class KafkaStreamsLiveTest {
streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
// when
KStreamBuilder builder = new KStreamBuilder();
StreamsBuilder builder = new StreamsBuilder();
KStream<String, String> textLines = builder.stream(inputTopic);
Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS);
KTable<String, Long> wordCounts = textLines.flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))).groupBy((key, word) -> word).count();
wordCounts.foreach((word, count) -> System.out.println("word: " + word + " -> " + count));
textLines.foreach((word, count) -> System.out.println("word: " + word + " -> " + count));
String outputTopic = "outputTopic";
final Serde<String> stringSerde = Serdes.String();
final Serde<Long> longSerde = Serdes.Long();
wordCounts.to(stringSerde, longSerde, outputTopic);
final Serde<String> longSerde = Serdes.String();
textLines.to(outputTopic, Produced.with(stringSerde,longSerde));
KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
KafkaStreams streams = new KafkaStreams(new Topology(), streamsConfiguration);
streams.start();
// then