From e183ccd1fcb8b2cf89bd3db5db7ee5b2d6d9f3a9 Mon Sep 17 00:00:00 2001 From: Lazzexus Date: Tue, 13 Oct 2026 07:00:06 +0200 Subject: [PATCH] Raise Kafka Streams transaction timeout above the commit interval Under exactly_once_v2, Streams defaults the producer transaction timeout to 10s. With commit.interval.ms raised to 10s (665cd1a), transactions on the heavy map-aggregation topology timed out, got fenced (ProducerFencedException / TaskMigratedException ~130 times during one large chunk import), and their work was redone. Default the transaction timeout to 60s, overridable via KAFKA_STREAMS_TRANSACTION_TIMEOUT_MS. Co-Authored-By: Claude Opus 5.5 --- modules/events-processor-core/src/main/resources/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/events-processor-core/src/main/resources/config.yml b/modules/events-processor-core/src/main/resources/config.yml index ad6604b..bca828e 100644 --- a/modules/events-processor-core/src/main/resources/config.yml +++ b/modules/events-processor-core/src/main/resources/config.yml @@ -23,6 +23,9 @@ kafka-streams-config: # repeated updates to the same (large) map chunk be merged before being forwarded. commit.interval.ms: ${KAFKA_STREAMS_COMMIT_INTERVAL_MS:-100} cache.max.bytes.buffering: ${KAFKA_STREAMS_CACHE_MAX_BYTES:-10485760} + # Under exactly_once_v2 Streams defaults the transaction timeout to 10s. It must be well above + # commit.interval.ms, or transactions time out, get fenced, and the work is redone. + producer.transaction.timeout.ms: ${KAFKA_STREAMS_TRANSACTION_TIMEOUT_MS:-60000} processing.guarantee: exactly_once_v2 auto.offset.reset: earliest default.key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde