This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new fd83f4b5b5cc CAMEL-26822: camel-kafka - the Kafka idempotent and
key-value repositories use the acks of the Kafka client (#29587)
fd83f4b5b5cc is described below
commit fd83f4b5b5cc49ce900355f22b9fdedd2123609c
Author: allthingssecurity <[email protected]>
AuthorDate: Wed Nov 11 12:27:17 2026 +0530
CAMEL-26822: camel-kafka - the Kafka idempotent and key-value repositories
use the acks of the Kafka client (#29587)
* CAMEL-26822: camel-kafka - the Kafka idempotent and key-value
repositories use the acks of the Kafka client
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../processor/idempotent/kafka/KafkaChangelog.java | 20 ++++--
.../kafka/KafkaChangelogProducerConfigTest.java | 74 ++++++++++++++++++++++
.../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 12 ++++
3 files changed, 100 insertions(+), 6 deletions(-)
diff --git
a/components/camel-kafka/src/main/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelog.java
b/components/camel-kafka/src/main/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelog.java
index 18940390ca95..4873ee9df10e 100644
---
a/components/camel-kafka/src/main/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelog.java
+++
b/components/camel-kafka/src/main/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelog.java
@@ -156,12 +156,7 @@ public final class KafkaChangelog<V> {
consumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
valueDeserializer.getName());
consumer = new KafkaConsumer<>(consumerConfig);
- producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
StringSerializer.class.getName());
- producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
valueSerializer.getName());
- // set up the producer to remove all batching on send, we want all
sends to be fully synchronous
- producerConfig.putIfAbsent(ProducerConfig.ACKS_CONFIG, "1");
- producerConfig.putIfAbsent(ProducerConfig.BATCH_SIZE_CONFIG, "0");
- producer = new KafkaProducer<>(producerConfig);
+ producer = new KafkaProducer<>(producerProperties());
poller = new TopicPoller();
ServiceHelper.startService(poller);
@@ -179,6 +174,19 @@ public final class KafkaChangelog<V> {
}
}
+ /**
+ * The properties of the producer: the configured properties, with the
serializers, and without batching, as every
+ * change is written synchronously. The acks and the idempotence are the
ones of the Kafka client (acks=all with
+ * idempotence), unless the configured properties set them.
Package-private for tests.
+ */
+ Properties producerProperties() {
+ producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
StringSerializer.class.getName());
+ producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
valueSerializer.getName());
+ // set up the producer to remove all batching on send, we want all
sends to be fully synchronous
+ producerConfig.putIfAbsent(ProducerConfig.BATCH_SIZE_CONFIG, "0");
+ return producerConfig;
+ }
+
/**
* Stops the background thread, and closes the consumer and the producer.
*/
diff --git
a/components/camel-kafka/src/test/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelogProducerConfigTest.java
b/components/camel-kafka/src/test/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelogProducerConfigTest.java
new file mode 100644
index 000000000000..e23a894c823d
--- /dev/null
+++
b/components/camel-kafka/src/test/java/org/apache/camel/processor/idempotent/kafka/KafkaChangelogProducerConfigTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor.idempotent.kafka;
+
+import java.util.Properties;
+
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The producer that the Kafka idempotent and key-value repositories write
their changes with uses the acks and
+ * idempotence of the Kafka client (acks=all with idempotence), unless the
producer config of the repository sets them.
+ * No broker is needed: the client resolves the config without connecting.
+ */
+class KafkaChangelogProducerConfigTest {
+
+ @Test
+ void producerUsesTheAcksOfTheKafkaClient() {
+ ProducerConfig config = new ProducerConfig(producerProperties(new
Properties()));
+
+ // acks=all is -1 for the client
+ assertEquals("-1", config.getString(ProducerConfig.ACKS_CONFIG));
+
assertTrue(config.getBoolean(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG));
+ }
+
+ @Test
+ void producerConfigThatEnablesIdempotenceIsValid() {
+ Properties user = new Properties();
+ user.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true");
+
+ // with acks=1 forced, the client refuses idempotence (ConfigException)
+ assertDoesNotThrow(() -> new ProducerConfig(producerProperties(user)));
+ }
+
+ @Test
+ void acksOfTheProducerConfigWins() {
+ Properties user = new Properties();
+ user.put(ProducerConfig.ACKS_CONFIG, "1");
+
+ ProducerConfig config = new ProducerConfig(producerProperties(user));
+
+ assertEquals("1", config.getString(ProducerConfig.ACKS_CONFIG));
+ }
+
+ private static Properties producerProperties(Properties user) {
+ Properties producerConfig = KafkaChangelog.producerConfig(null,
"localhost:9092");
+ producerConfig.putAll(user);
+ KafkaChangelog<String> changelog = new KafkaChangelog<>(
+ "test", "test-topic", new Properties(), producerConfig,
StringDeserializer.class, StringSerializer.class,
+ 100, true, record -> {
+ });
+ return changelog.producerProperties();
+ }
+}
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
index 42c70d790bfb..fbb57a05ed5f 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
@@ -6915,6 +6915,18 @@ resumed after the failed record. The new consumer leaves
and rejoins the consume
rebalance of the group, also when the partition has a committed offset. There
is no option to keep the previous
behaviour.
+=== camel-kafka - the Kafka idempotent and key-value repositories use the acks
of the Kafka client
+
+`KafkaIdempotentRepository` and `KafkaKeyValueRepository` no longer set
`acks=1` on their producer, so the default of
+the Kafka client applies: `acks=all` with idempotence. Every change of a
repository is now acknowledged once the
+in-sync replicas have it, so it is not lost when the leader of the partition
fails over. Each change waits for the
+replicas; with fewer in-sync replicas than `min.insync.replicas` it waits up
to `delivery.timeout.ms` (2 minutes by
+default) and then fails, where it succeeded with `acks=1`. A broker before
Kafka 2.8 that uses ACLs needs the
+`IdempotentWrite` permission for the repository's producer. A `producerConfig`
that sets only
+`enable.idempotence=true` no longer fails at start. To keep the previous
behaviour, set `acks=1` in the `producerConfig`
+of the repository (with `bootstrap.servers`, as a `producerConfig` replaces
the `bootstrapServers` of the repository
+for the producer).
+
=== camel-dns - dig without a type, and the wikipedia type
`dns:dig` without the `CamelDnsType` header now asks for type A, and without
the `CamelDnsName` header fails with an
