From 665cd1a7d6ad24dfd6b6447c1b103e7cae6e880a Mon Sep 17 00:00:00 2001 From: Lazzexus Date: Mon, 12 Oct 2026 19:43:17 +0200 Subject: [PATCH] Speed up map processing 14x by merging aggregate updates before forwarding The zoom-out map layers aggregate 32x32 chunks into 256x256 and 512x512 chunks, holding up to 65k/262k tiles. Every incoming chunk rebuilt the whole aggregate, and with exactly_once_v2's default 100ms commit interval nearly every intermediate result was forwarded downstream, cascading into the next level. A small map also has few 512-level keys, so this work landed on one partition and one thread. Make commit.interval.ms and cache.max.bytes.buffering configurable (defaults unchanged), and in the Unraid example use a 10s commit interval with a 128MB cache, so repeated updates to the same chunk are merged in the record cache first. Measured on a live backlog: 41 -> 575 chunks/minute. Map updates now appear within ~10s instead of instantly. Co-Authored-By: Claude Opus 5.5 --- docs/kafkatorio-platform/unraid/docker-compose.yml | 3 +++ modules/events-processor-core/src/main/resources/config.yml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/docs/kafkatorio-platform/unraid/docker-compose.yml b/docs/kafkatorio-platform/unraid/docker-compose.yml index 6ea6a48..04c44c2 100644 --- a/docs/kafkatorio-platform/unraid/docker-compose.yml +++ b/docs/kafkatorio-platform/unraid/docker-compose.yml @@ -26,6 +26,9 @@ services: KAFKA_BOOTSTRAP_SERVERS: "kafka:29092" # one stream thread per partition (topics have 3) - parallelises map processing KAFKA_STREAMS_NUM_THREADS: "3" + # merge repeated map-chunk updates for 10s before forwarding them (128MB cache per topology) + KAFKA_STREAMS_COMMIT_INTERVAL_MS: "10000" + KAFKA_STREAMS_CACHE_MAX_BYTES: "134217728" JAVA_OPTS: "-Xmx4g" networks: [ kafkatorio-nw ] depends_on: [ infra-kafka ] diff --git a/modules/events-processor-core/src/main/resources/config.yml b/modules/events-processor-core/src/main/resources/config.yml index 476294f..ad6604b 100644 --- a/modules/events-processor-core/src/main/resources/config.yml +++ b/modules/events-processor-core/src/main/resources/config.yml @@ -18,6 +18,11 @@ kafka-streams-config: bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-localhost:9092} state.dir: ${KAFKA_STREAMS_STATE_DIR:-.state/kafka} num.stream.threads: ${KAFKA_STREAMS_NUM_THREADS:-1} + # With exactly_once_v2 the default commit interval is 100ms, so almost every intermediate + # aggregate update is forwarded downstream. A longer interval and a larger record cache let + # repeated updates to the same (large) map chunk be merged before being forwarded. + commit.interval.ms: ${KAFKA_STREAMS_COMMIT_INTERVAL_MS:-100} + cache.max.bytes.buffering: ${KAFKA_STREAMS_CACHE_MAX_BYTES:-10485760} processing.guarantee: exactly_once_v2 auto.offset.reset: earliest default.key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde