gnodet-bot commented on code in PR #29992: URL: https://github.com/apache/camel/pull/29992#discussion_r4576372762
########## components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/KafkaWorld.java: ########## @@ -0,0 +1,557 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kafka.dst; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.Set; + +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.ServiceStatus; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.kafka.KafkaClientFactory; +import org.apache.camel.component.kafka.KafkaComponent; +import org.apache.camel.component.kafka.KafkaConfiguration; +import org.apache.camel.component.kafka.KafkaConstants; +import org.apache.camel.component.kafka.consumer.DefaultKafkaManualAsyncCommitFactory; +import org.apache.camel.component.kafka.consumer.DefaultKafkaManualCommitFactory; +import org.apache.camel.component.kafka.consumer.KafkaManualCommit; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.test.dst.Dst; +import org.apache.camel.test.dst.DstCamel; +import org.apache.camel.test.dst.DstExecutor; +import org.apache.camel.test.dst.DstLedger; +import org.apache.camel.test.dst.DstWorld; +import org.apache.camel.test.dst.Invariants; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.producer.Producer; + +/** + * The world around a camel-kafka consumer: a fake Kafka cluster and consumer group ({@link FakeKafka}), a producer, a + * route whose processing is sometimes slow and sometimes fails, and an operator that restarts the route, suspends and + * resumes it (with a second suspend and resume while the consumer still handles the first), and lets a phantom member + * join and leave the group, so that partitions are revoked and assigned again while records are processed. Faults: the + * consumer cannot be created, polls fail, commits fail or time out, commits are slow, processing fails. + * <p> + * The seed picks the commit mode: Kafka's auto-commit (autoCommitEnable=true), manual synchronous or asynchronous + * commits in the route (allowManualCommit=true with DefaultKafkaManualCommitFactory or + * DefaultKafkaManualAsyncCommitFactory, after every record or only for the last record of a poll), or batching + * (batching=true with autoCommitEnable=false: Camel commits each batch once it was processed), and breakOnFirstError + * and pollOnError (ERROR_HANDLER, DISCARD, RETRY, RECONNECT; not STOP, which ends the consumer by design). + * <p> + * Not explored: batching with Kafka's auto-commit (the auto-commit commits the position of records that Camel still + * holds in an incomplete batch, which the KafkaConsumer Javadoc warns about), and batching with breakOnFirstError (the + * failed batch is committed, see "Break on First Error in Batching Mode" in the component documentation); manual + * commits from another thread; consumersCount above 1; seekTo, offset repositories and resume strategies. + * <p> + * Left out of the default set: breakOnFirstError with the synchronous manual commits (SyncCommitManager), where a + * commitSync that times out in the break path is being looked at separately; + * {@code -Ddst.kafka.syncBreakOnFirstError=true} puts the combination back. + * <p> + * consumersCount is 1: a consumer thread loops for ever, and with the one-stack model a second looping thread nested + * above the first would never let it resume. For the same reason a route stop that runs while the consumer thread waits + * in poll cannot see it finish: the stop wakes up the Kafka consumer, waits for its timeouts, and the thread ends after + * the stop returned, so the operator starts the route again only once the previous Kafka consumer is closed. The + * operator does not stop or suspend the route while an exchange is in the route (the shutdown strategy would wait for + * it for ever, as it cannot see it finish) or while the consumer thread waits for the background task that creates its + * Kafka consumer (a stop interrupts that wait with real threads, which the one-stack model does not do). The lock of + * KafkaFetchRecords does not exclude on the one stack, so a stop never waits for the processing of the current records. + * <p> + * Properties: a committed offset never passes a record the route has not finished with (at-least-once: completed, or + * failed with breakOnFirstError=false, which skips it by design); a suspended route stops fetching once its consumer + * handled the suspend; once the faults cleared, every record is finished, the committed offsets reach the end of each + * partition, and the route holds one open Kafka consumer; the consumer uses the client as the Javadoc requires; no + * Kafka consumer is left open after the route stopped. Records may be consumed more than once. + */ +@DstWorld(component = "kafka", + fake = "FakeKafka: a topic with one or two partitions and a consumer group (classic protocol, eager" + + " rebalance), behind the Consumer interface of KafkaClientFactory", + covers = { + "commits by Kafka's auto-commit, by manual synchronous or asynchronous commits in the route, and per" + + " batch with batching=true", + "breakOnFirstError", + "pollOnError ERROR_HANDLER, DISCARD, RETRY and RECONNECT", + "a poll that fails while a batch is incomplete", + "route restart, also while the consumer polls", + "route suspend and resume, with a second request while the consumer handles the first", + "partitions revoked and assigned again by a rebalance while records are processed" }, + faults = { + "the Kafka consumer cannot be created", "poll fails", "commitSync times out", + "commitAsync fails", "auto-commit fails", "commits are slow", "committed answers slowly", + "processing fails", "processing is slow", "a member joins and leaves the group" }, + invariants = { + "the consumer uses the Kafka consumer as documented", + "a committed offset never passes a record the route has not finished with", + "a suspended route stops fetching", + "every record is finished once the faults cleared", + "the committed offsets reach the end once the faults cleared", + "the route holds one Kafka consumer once the faults cleared", + "no Kafka consumer is left open after the route stopped" }, + issues = { "CAMEL-25313", "CAMEL-25756" }) +final class KafkaWorld { + + static final String ROUTE = "kafka"; + + private final Dst dst; + private final DstLedger ledger; + private final FakeKafka kafka; + private final int partitions; + private final String mode; + private final boolean breakOnFirstError; + private final boolean commitEveryRecord; + private final String pollOnError; + private final long pollTimeoutMs; + private final long shutdownTimeout; + private final long commitTimeoutMs; + private final long backoffInterval; + private final double processFailRate; + private final List<String> violations = new ArrayList<>(); + private boolean faults = true; + private int sendsLeft; + private int opsLeft; + // exchanges in the route: a route stop or suspend nested above one on the one stack would wait for it for ever + // (the shutdown strategy cannot see it finish), so the operator stops and suspends only when none is in the route + private int inRoute; + private boolean finished; + private String finalStatus; + private String finalOpen; + private int finalOpenCount; + private boolean finalDone; + private boolean finalCommitted; + + KafkaWorld(Dst dst) { + this.dst = dst; + this.ledger = new DstLedger(dst); + this.partitions = dst.pick(List.of(1, 2)); + this.kafka = new FakeKafka(dst, ledger, partitions); + this.mode = dst.pick(Arrays.asList(System.getProperty("dst.kafka.modes", "AUTO,SYNC,ASYNC,BATCH").split(","))); + // see the class Javadoc: batching with breakOnFirstError is not explored, and the synchronous manual commits + // with breakOnFirstError are left out unless -Ddst.kafka.syncBreakOnFirstError=true + this.breakOnFirstError = !"BATCH".equals(mode) + && (!"SYNC".equals(mode) || Boolean.getBoolean("dst.kafka.syncBreakOnFirstError")) + && dst.chance(0.5); + this.commitEveryRecord = dst.chance(0.5); + this.pollOnError = dst.pick(Arrays.asList( + System.getProperty("dst.kafka.pollOnError", "ERROR_HANDLER,DISCARD,RETRY,RECONNECT").split(","))); + this.pollTimeoutMs = dst.pick(List.of(100L, 1000L, 5000L)); + this.shutdownTimeout = dst.pick(List.of(1000L, 30000L)); + this.commitTimeoutMs = dst.pick(List.of(1000L, 5000L)); + this.backoffInterval = dst.pick(List.of(500L, 5000L)); + kafka.autoCommit = "AUTO".equals(mode); + kafka.autoCommitIntervalMs = dst.pick(List.of(1000L, 5000L)); + kafka.breakOnFirstError = breakOnFirstError; + kafka.maxPollRecords = "BATCH".equals(mode) ? dst.pick(List.of(3, 10)) : dst.pick(List.of(1, 3, 10)); + kafka.createFailRate = dst.pick(List.of(0.0, 0.1, 0.3)); + kafka.pollFailRate = dst.pick(List.of(0.0, 0.02, 0.1)); + kafka.commitFailRate = dst.pick(List.of(0.0, 0.05, 0.2)); + kafka.slowCommits = dst.chance(0.5); + this.processFailRate = dst.pick(List.of(0.0, 0.05, 0.2)); + dst.trace("world: " + config()); + } + + private String config() { + return "partitions=" + partitions + " mode=" + mode + " breakOnFirstError=" + breakOnFirstError + + " commitEveryRecord=" + commitEveryRecord + " pollOnError=" + pollOnError + " pollTimeoutMs=" + + pollTimeoutMs + " maxPollRecords=" + kafka.maxPollRecords + " shutdownTimeout=" + shutdownTimeout + + " commitTimeoutMs=" + commitTimeoutMs + " autoCommitIntervalMs=" + kafka.autoCommitIntervalMs + + " backoff=" + backoffInterval + " createFail=" + kafka.createFailRate + " pollFail=" + + kafka.pollFailRate + " commitFail=" + kafka.commitFailRate + " slowCommits=" + kafka.slowCommits + + " processFail=" + processFailRate; + } + + void run() throws Exception { + DefaultCamelContext context = DstCamel.newContext(dst); + KafkaComponent component = new KafkaComponent(); + component.setCreateConsumerBackoffInterval(backoffInterval); + if ("SYNC".equals(mode)) { + component.setKafkaManualCommitFactory(new DefaultKafkaManualCommitFactory()); + } else if ("ASYNC".equals(mode)) { + component.setKafkaManualCommitFactory(new DefaultKafkaManualAsyncCommitFactory()); + } + component.setKafkaClientFactory(new KafkaClientFactory() { + @Override + public Producer getProducer(Properties kafkaProps) { + throw new UnsupportedOperationException("the world has no producer endpoint"); + } + + @Override + public Consumer getConsumer(Properties kafkaProps) { + return kafka.newConsumer(); + } + + @Override + public String getBrokers(KafkaConfiguration configuration) { + return configuration.getBrokers(); + } + }); + context.addComponent("kafka", component); + boolean manual = "SYNC".equals(mode) || "ASYNC".equals(mode); + String uri = "kafka:" + FakeKafka.TOPIC + "?brokers=127.0.0.1:9092&groupId=" + FakeKafka.GROUP + + "&consumersCount=1&autoOffsetReset=earliest&pollTimeoutMs=" + pollTimeoutMs + "&maxPollRecords=" + + kafka.maxPollRecords + "&shutdownTimeout=" + shutdownTimeout + "&commitTimeoutMs=" + + commitTimeoutMs + "&pollOnError=" + pollOnError + "&breakOnFirstError=" + breakOnFirstError + + "&autoCommitEnable=" + "AUTO".equals(mode) + "&autoCommitIntervalMs=" + + kafka.autoCommitIntervalMs + "&allowManualCommit=" + manual + "&batching=" + "BATCH".equals(mode); + context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from(uri).routeId(ROUTE).autoStartup(false) + .process(exchange -> process(exchange, manual)); + } + }); + context.start(); + DstExecutor operator = dst.executor("operator", 1); + try { + operator.schedule("start route", 0, () -> startRoute(context, operator)); + planSends(); + planOperations(context, operator); + operator.schedule("finisher", 1000, () -> finisher(context, operator, -1)); + dst.runUntil(() -> finished, "the finisher"); + // the consumer thread ends after the stop; let it close its Kafka consumer + dst.runFor(2 * shutdownTimeout + 10_000); + dst.check( + Invariants.of("the consumer uses the Kafka consumer as documented", + () -> kafka.violations.isEmpty() && violations.isEmpty() + ? null : kafka.violations + " " + violations + " (" + config() + ")"), + Invariants.of("a committed offset never passes a record the route has not finished with", + () -> kafka.lossViolations.isEmpty() ? null : kafka.lossViolations + " (" + config() + ")"), + Invariants.of("a suspended route stops fetching", + () -> kafka.suspendViolations.isEmpty() + ? null : kafka.suspendViolations + " (" + config() + ")"), + Invariants.of("every record is finished once the faults cleared", + () -> finalDone ? null : "not finished: " + notDone() + " (" + config() + ")"), + Invariants.of("the committed offsets reach the end once the faults cleared", + () -> finalCommitted ? null : kafka.notCommitted() + " (" + config() + ")"), + Invariants.of("the route holds one Kafka consumer once the faults cleared", + () -> "Started".equals(finalStatus) && finalOpenCount == 1 + ? null + : "route is " + finalStatus + ", open Kafka consumers " + finalOpen + " (" + + config() + ")"), + Invariants.of("no Kafka consumer is left open after the route stopped", + () -> kafka.open.isEmpty() + ? null : "open after stop: " + kafka.openNames() + " (" + config() + ")")); + } finally { + dst.dropPending(); + context.stop(); + } + } + + /** The route: a record (or each record of a batch) is routed, maybe slowly, then fails or completes. */ + private void process(Exchange exchange, boolean manual) throws Exception { + List<FakeKafka.Rec> recs = new ArrayList<>(); + Object body = exchange.getMessage().getBody(); + if (!(body instanceof List) && exchange.getMessage().getHeader(KafkaConstants.OFFSET) == null) { + // not a record: an exception of the consumer bridged to the error handler + return; + } + if (body instanceof List<?> batch) { + for (Object o : batch) { + recs.add(recOf(((Exchange) o).getMessage())); + } + } else { + recs.add(recOf(exchange.getMessage())); + } + inRoute++; + try { + processRecords(exchange, recs, manual); + } finally { + inRoute--; + } + } + + private void processRecords(Exchange exchange, List<FakeKafka.Rec> recs, boolean manual) { + String ids = recs.stream().map(r -> r.id).toList().toString(); + for (FakeKafka.Rec r : recs) { + r.routed++; + ledger.record("routed", "p" + r.partition, r.id); + } + if (faults && dst.chance(0.1)) { + dst.block(dst.between(1000, 10_000), "slow processing of " + ids); + } else { + dst.block(dst.between(0, 100), "processing of " + ids); + } + if (faults && dst.chance(processFailRate)) { + for (FakeKafka.Rec r : recs) { + r.failed++; + ledger.record("failed", "p" + r.partition, r.id); + } + throw Dst.injected(new IllegalStateException("dst: processing failed for " + ids)); + } + for (FakeKafka.Rec r : recs) { + r.completed++; + ledger.record("completed", "p" + r.partition, r.id); + } + if (manual) { + Message m = exchange.getMessage(); + boolean last = Boolean.TRUE.equals(m.getHeader(KafkaConstants.LAST_RECORD_BEFORE_COMMIT, Boolean.class)); + if (commitEveryRecord || last) { + // a commit that fails fails the exchange, after the record was processed + m.getHeader(KafkaConstants.MANUAL_COMMIT, KafkaManualCommit.class).commit(); + } + } + } + + private FakeKafka.Rec recOf(Message message) { + int p = message.getHeader(KafkaConstants.PARTITION, Integer.class); + long offset = message.getHeader(KafkaConstants.OFFSET, Long.class); + FakeKafka.Rec r = kafka.rec(p, offset); + if (!r.id.equals(message.getBody(String.class))) { + violations.add("record p" + p + "@" + offset + " has the body " + message.getBody() + ", not " + r.id); + } + return r; + } + + private String notDone() { + List<String> missing = new ArrayList<>(); + for (List<FakeKafka.Rec> partitionLog : kafka.log) { + for (FakeKafka.Rec r : partitionLog) { + if (!kafka.done(r)) { + missing.add(r.id + "@p" + r.partition + "/" + r.offset + " routed=" + r.routed + " failed=" + r.failed); + } + } + } + return missing.toString(); + } + + private boolean allDone() { + for (List<FakeKafka.Rec> partitionLog : kafka.log) { + for (FakeKafka.Rec r : partitionLog) { + if (!kafka.done(r)) { + return false; + } + } + } + return true; + } + + /** + * Waits for the sends and operations, switches the faults off, publishes one more record to each partition (its + * commit covers the records before it), waits for everything to be finished and committed (or a deadline), takes + * the final state and stops the route. + */ + private void finisher(DefaultCamelContext context, DstExecutor operator, long deadline) { + if (deadline == -1 && (sendsLeft > 0 || opsLeft > 0)) { + operator.schedule("finisher", 500, () -> finisher(context, operator, -1)); + return; + } + if (deadline == -1) { + long slow = dst.between(0, 10_000); + dst.trace("world: operations done, faults for another " + slow + " ms"); + operator.schedule("finisher", slow, () -> finisher(context, operator, -2)); + return; + } + if (deadline == -2) { + faults = false; + kafka.faults = false; + kafka.phantomLeaves(); + for (int p = 0; p < partitions; p++) { + kafka.publish(p, "end" + p); + } + dst.trace("world: faults cleared"); + long d = dst.now() + 10 * 60_000L; + operator.schedule("finisher", 500, () -> finisher(context, operator, d)); + return; + } + ServiceStatus status = context.getRouteController().getRouteStatus(ROUTE); + boolean done = allDone(); + boolean committed = kafka.allCommitted(); + if ((inRoute > 0 || !done || !committed || status != ServiceStatus.Started || kafka.open.size() != 1) + && dst.now() < deadline) { + operator.schedule("finisher", 500, () -> finisher(context, operator, deadline)); + return; + } + finalStatus = String.valueOf(status); + finalDone = done; + finalCommitted = committed; + finalOpenCount = kafka.open.size(); + finalOpen = kafka.openNames(); + if (inRoute > 0) { + // a stop would wait for it for ever (one-stack model) + finalDone = false; + finished = true; + return; + } + try { + context.getRouteController().stopRoute(ROUTE); + } catch (Exception e) { + violations.add("final stop failed: " + e); + } + finished = true; + } + + /** The operator starts the route; a failed start is retried, as the supervising route controller does. */ + private void startRoute(DefaultCamelContext context, DstExecutor operator) { + try { + context.getRouteController().startRoute(ROUTE); + ledger.record("routeStarted", "r", ROUTE); + } catch (Exception e) { + ledger.record("routeStartFailed", "r", ROUTE); + dst.trace("world: start failed (" + e.getMessage() + "), retry in 2 s"); + operator.schedule("retry start route", 2000, () -> startRoute(context, operator)); + } + } + + /** + * Starts the route again once the Kafka consumer of the stopped route is closed: its thread ends only after the + * stop returned (one-stack model), and a new consumer thread started above it would never let it resume. + */ + private void startWhenClosed(DefaultCamelContext context, DstExecutor operator, long since) { + if (!kafka.open.isEmpty() && dst.now() - since < 10 * 60_000L) { + operator.schedule("start route when the consumer closed", 200, + () -> startWhenClosed(context, operator, since)); + return; + } + if (!kafka.open.isEmpty()) { + violations.add("Kafka consumers " + kafka.openNames() + " still open 10 minutes after the route stopped"); + } + startRoute(context, operator); + opsLeft--; + } + + private void planSends() { + DstExecutor producer = dst.executor("producer", 1); + long t = dst.between(0, 500); + int n = 0; + int bursts = (int) dst.between(1, 4); + for (int b = 0; b < bursts; b++) { + int size = (int) dst.between(1, 15); + for (int i = 0; i < size; i++) { + String id = "m" + n++; + int p = partitions == 1 ? 0 : dst.nextInt(partitions); + sendsLeft++; + producer.schedule("publish " + id, t, () -> { + kafka.publish(p, id); + sendsLeft--; + }); + t += dst.between(0, 100); + } + t += dst.between(0, 8000); + } + } + + private void planOperations(DefaultCamelContext context, DstExecutor operator) { + int ops = (int) dst.between(0, 4); + long t = dst.between(500, 3000); + for (int i = 0; i < ops; i++) { + int kind = dst.nextInt(3); + opsLeft++; + if (kind == 0) { + operator.schedule("stop route", t, () -> whenConnected(operator, "stop route", () -> { + if (context.getRouteController().getRouteStatus(ROUTE) != ServiceStatus.Started) { + opsLeft--; + return; + } + try { + context.getRouteController().stopRoute(ROUTE); + ledger.record("routeStopped", "r", ROUTE); + } catch (Exception e) { + violations.add("stop failed: " + e); + } + long since = dst.now(); + operator.schedule("start route when the consumer closed", dst.between(0, 3000), + () -> startWhenClosed(context, operator, since)); + })); + } else if (kind == 1) { + // a second suspend and resume while the consumer may still handle the first resume (it reads the + // committed offsets before it resumes its partitions) + int times = dst.chance(0.5) ? 2 : 1; + operator.schedule("suspend route", t, () -> suspendAndResume(context, operator, times)); + } else { + Set<Integer> taken = partitions == 1 ? Set.of(0) : dst.pick(List.of(Set.of(0), Set.of(1), Set.of(0, 1))); + long held = dst.between(500, 15_000); + operator.schedule("a member joins the group", t, () -> { + kafka.phantomJoins(taken); + operator.schedule("the member leaves the group", held, () -> { + kafka.phantomLeaves(); + opsLeft--; + }); + }); + } + t += dst.between(500, 10_000); + } + } + + /** + * Runs an operation of the operator once no exchange is in the route. The operator task can run nested above an + * exchange that blocks (on the one stack), so it waits in a new task, not in this one. + */ + private void whenIdle(DstExecutor operator, String what, Runnable operation) { + if (inRoute > 0) { + operator.schedule(what + " (an exchange is in the route)", 50, () -> whenIdle(operator, what, operation)); + return; + } + operation.run(); + } + + /** + * Stops the route only while the consumer thread holds a Kafka consumer, not while it waits for the background task + * that creates one: a stop interrupts that wait with real threads (shutdownNow), which the one-stack model does not + * do, and the consumer thread would go on after the route was started again. + */ + private void whenConnected(DstExecutor operator, String what, Runnable operation) { + if (kafka.open.size() != 1 || inRoute > 0) { + operator.schedule(what + " (no Kafka consumer, or an exchange is in the route)", 100, + () -> whenConnected(operator, what, operation)); + return; + } + operation.run(); + } + + private void suspendAndResume(DefaultCamelContext context, DstExecutor operator, int times) { + whenIdle(operator, "suspend route", () -> { + if (context.getRouteController().getRouteStatus(ROUTE) != ServiceStatus.Started) { + opsLeft--; + return; + } + try { + suspend(context); + } catch (Exception e) { + violations.add("suspend failed: " + e); + opsLeft--; + return; + } + operator.schedule("resume route", dst.between(100, 5000), () -> { + try { + resume(context); + } catch (Exception e) { + violations.add("resume failed: " + e); + } + if (times > 1) { + operator.schedule("suspend route again", dst.between(0, pollTimeoutMs + 2000), + () -> suspendAndResume(context, operator, times - 1)); + } else { + opsLeft--; + } + }); + }); + } + + private void suspend(DefaultCamelContext context) throws Exception { + context.getRouteController().suspendRoute(ROUTE); + kafka.suspendedSince = dst.now(); + ledger.record("routeSuspended", "r", ROUTE); + } + + private void resume(DefaultCamelContext context) throws Exception { + kafka.suspendedSince = -1; + context.getRouteController().resumeRoute(ROUTE); Review Comment: ⚠️ **Exception safety in `resume()`**: `suspendedSince` is cleared on line 553 **before** `resumeRoute()` is called on line 554. If `resumeRoute()` throws, `suspendedSince` will be `-1` even though the route never actually resumed — the `checkSuspended()` invariant check will then report the route as not suspended when it still is. Consider swapping the two lines: ```suggestion context.getRouteController().resumeRoute(ROUTE); kafka.suspendedSince = -1; ledger.record("routeResumed", "r", ROUTE); ``` ########## components/camel-kafka/src/test/java/org/apache/camel/component/kafka/dst/FakeKafka.java: ########## @@ -0,0 +1,824 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kafka.dst; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.camel.test.dst.Dst; +import org.apache.camel.test.dst.DstLedger; +import org.apache.kafka.clients.consumer.CommitFailedException; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.consumer.RetriableCommitFailedException; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.record.TimestampType; + +/** + * A fake Kafka cluster for one topic with one or two partitions and one consumer group, and fake consumers behind the + * {@link Consumer} interface (dynamic proxies), as camel-kafka gets them from its {@code KafkaClientFactory}. + * <p> + * Written from the Javadoc of {@code org.apache.kafka.clients.consumer.KafkaConsumer} and {@code Consumer} + * (kafka-clients 4.3) for the classic group protocol with an eager assignor (the default {@code RangeAssignor}). + * kafka-clients' {@code MockConsumer} is not used: its poll returns at once, its commits never fail or take time, and + * it has no group (no rebalance of its own), so it could not drive the timing this world is about. When kafka-clients + * is upgraded, check the rules below against the new Javadoc. + * <p> + * Rules (quotes from the KafkaConsumer Javadoc): + * <ul> + * <li>[P] poll: "fetch data for the topics or partitions specified using one of the subscribe/assign APIs"; it returns + * when records are available or the timeout expired, at most max.poll.records records ("fewer records might be + * returned"), from the position of each assigned partition that is not paused; the position then moves past them. "It + * is an error to not have subscribed to any topics or partitions before polling for data" (IllegalStateException).</li> + * <li>[G] group management: "the consumer will ... rebalance partitions among the members of the group" when a member + * joins or leaves; the rebalance happens inside poll, where the listener is called: "onPartitionsRevoked ... will be + * called ... during a rebalance operation when the consumer has to give up some partitions" (eager: every partition is + * revoked, then the new assignment is given to onPartitionsAssigned). "Rebalance will not preserve the pause/resume + * state" (pause). After an assignment the position of a partition is its committed offset, else auto.offset.reset + * (earliest in this world).</li> + * <li>[A] enable.auto.commit: "the committed offset will be the position", committed "with a frequency controlled by + * the config auto.commit.interval.ms" while the consumer polls, before a rebalance revokes the partitions, and when it + * closes. ("you must consume all data returned from each call to poll(Duration) before any subsequent calls, or before + * closing the consumer. If you fail to do either of these, it is possible for the committed offset to get ahead of the + * consumed position, which results in missing records.")</li> + * <li>[S] commitSync: "This is a synchronous commit and will block until either the commit succeeds, an unrecoverable + * error is encountered ... or the timeout ... expires". TimeoutException "if the timeout specified by + * default.api.timeout.ms expires before successful completion of the offset commit": whether the broker applied it is + * not known, so the seed chooses (assumed: the Javadoc does not say). A commit that takes longer than its timeout ends + * the same way. CommitFailedException "when you are trying to commit to partitions that are no longer assigned to this + * consumer". Until the member rejoins the group in its next poll, the group is preparing the rebalance and takes the + * commits of the partitions the member still owns (assumed from the classic protocol: the coordinator accepts commits + * of the current generation then). "This is a synchronous commit and will also invoke any pending offset commit + * callbacks of async commits".</li> + * <li>[C] commitAsync: "Offsets committed through multiple calls to this API are guaranteed to be sent in the same + * order as the invocations. Corresponding commit callbacks are also invoked in the same order"; the callback is invoked + * from a later poll, commitSync, commitAsync or close of the same consumer; a failed commit is not applied + * (RetriableCommitFailedException or CommitFailedException in the callback).</li> + * <li>[R] committed: "This call will block to do a remote call to get the latest committed offsets from the server"; + * null for a partition without a committed offset.</li> + * <li>[W] wakeup: "The thread which is blocking in an operation will throw WakeupException. If no thread is blocking in + * a method which can throw WakeupException, the next call to such method will raise it instead." Here: poll, commitSync + * and committed.</li> + * <li>[X] seek, pause and resume of a partition that is not assigned: IllegalStateException ("No current assignment for + * partition"). unsubscribe leaves the group (revocation callback first); close commits with auto-commit, completes the + * pending async commits, and leaves the group.</li> + * <li>[L] an exception of the rebalance listener: the client gives up the partitions (or takes the new ones) first, + * then poll, unsubscribe or close throws it, wrapped in a KafkaException ("User rebalance callback throws an error"); a + * WakeupException or InterruptException is thrown at once, and the member keeps its partitions (assumed from the + * classic client; the Javadoc says only that the exception is propagated). close turns wakeups off first, as the + * classic client does.</li> + * </ul> + * A phantom member can join the group (and leave again): it takes partitions in the rebalance, but does not consume or + * commit, so its partitions wait for the Camel consumer to get them back. + * <p> + * Timing (one-stack rule): poll waits first and fetches at the end of its wait; the rebalance in poll blocks for the + * join before the listener is called; commits take effect at the start of their latency; committed blocks, then reads. + * The other calls do not block. + * <p> + * Not modelled: several topics, topic patterns and assign(), max.poll.interval.ms and session timeouts (a member is + * never removed for being slow), the cooperative rebalance protocol and group.protocol=consumer, transactions and + * isolation levels, leader epochs, log retention and offsets out of range. + * <p> + * The fake checks the at-least-once contract of the consumer when the group takes a commit: a committed offset must not + * pass a record that the route has not finished with ({@link #done(TopicPartition, long)}). + */ +final class FakeKafka { + + static final String TOPIC = "dst"; + static final String GROUP = "g"; + + /** A record of the topic, and what the route did with it. */ + static final class Rec { + final int partition; + final long offset; + final String id; + int routed; + int completed; + int failed; + + Rec(int partition, long offset, String id) { + this.partition = partition; + this.offset = offset; + this.id = id; + } + } + + /** An async commit waiting for its callback. */ + private record PendingCallback(Map<TopicPartition, OffsetAndMetadata> offsets, Exception error, + OffsetCommitCallback callback) { + } + + final class Member implements InvocationHandler { + final String name; + final Set<TopicPartition> owned = new TreeSet<>(FakeKafka::compare); + final Set<TopicPartition> paused = new TreeSet<>(FakeKafka::compare); + final Map<TopicPartition, Long> position = new TreeMap<>(FakeKafka::compare); + final List<PendingCallback> callbacks = new ArrayList<>(); + ConsumerRebalanceListener listener; + boolean subscribed; + boolean closed; + boolean wakeup; + long generation = -1; + long lastAutoCommit; + // for the check of a suspended route: when the last poll started, and whether it returned normally + long lastPollStart = -1; + boolean lastPollReturned; + + Member(String name) { + this.name = name; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) { + String m = method.getName(); + switch (m) { + case "toString": + return "FakeKafkaConsumer[" + name + "]"; + case "hashCode": + return System.identityHashCode(this); + case "equals": + return proxy == args[0]; + case "wakeup": + dst.trace("fake: " + name + " wakeup"); + wakeup = true; + return null; + case "close": + close(); + return null; + case "metrics": + return Map.of(); + default: + break; + } + checkOpen(m); + switch (m) { + case "subscribe": + subscribe(args); + return null; + case "unsubscribe": + unsubscribe(); + return null; + case "subscription": + return subscribed ? Set.of(TOPIC) : Set.of(); + case "assignment": + return copyOf(owned); + case "paused": + return copyOf(paused); + case "poll": + return poll((Duration) args[0]); + case "commitSync": + commitSync(args); + return null; + case "commitAsync": + commitAsync(args); + return null; + case "committed": + return committed(args); + case "seek": + seek((TopicPartition) args[0], + args[1] instanceof OffsetAndMetadata o ? o.offset() : (Long) args[1]); + return null; + case "pause": + pause(args, true); + return null; + case "resume": + pause(args, false); + return null; + default: + violations.add(name + " called " + m + " (not modelled)"); + throw new UnsupportedOperationException(m); + } + } + + private void checkOpen(String m) { + if (closed) { + violations.add(name + " used after close: " + m); + throw new IllegalStateException("This consumer has already been closed."); + } + } + + private void checkWakeup() { + if (wakeup) { + wakeup = false; + dst.trace("fake: " + name + " throws WakeupException"); + throw new WakeupException(); + } + } + + private void subscribe(Object[] args) { + if (!(args[0] instanceof Collection<?> topics)) { + violations.add(name + " subscribed with " + args[0].getClass().getSimpleName() + " (not modelled)"); + throw new UnsupportedOperationException("subscribe(" + args[0] + ")"); + } + if (!topics.equals(List.of(TOPIC)) && !topics.equals(Set.of(TOPIC))) { + violations.add(name + " subscribed to " + topics); + } + listener = args.length > 1 ? (ConsumerRebalanceListener) args[1] : null; + if (!subscribed) { + subscribed = true; + members.add(this); + groupChanged(name + " joins"); + } + } + + private void unsubscribe() { + if (!subscribed) { + return; + } + dst.trace("fake: " + name + " unsubscribe"); + leave(); + } + + /** [X] leaves the group: the revocation callback first, then the partitions are given up; [L]. */ + private void leave() { + RuntimeException failure = null; + if (!owned.isEmpty() && listener != null) { + failure = revoke(); + if (interruption(failure)) { + // [L] thrown at once: the member keeps its partitions and stays in the group + throw failure; + } + } + owned.clear(); + paused.clear(); + position.clear(); + subscribed = false; + members.remove(this); + groupChanged(name + " leaves"); + if (failure != null) { + throw listenerFailure(failure); + } + } + + /** The revocation callback; its exception is returned, to be thrown once the partitions are given up [L]. */ + private RuntimeException revoke() { + try { + listener.onPartitionsRevoked(copyOf(owned)); + return null; + } catch (RuntimeException e) { + dst.trace("fake: " + name + " the revocation callback throws " + e); + return e; + } + } + + private RuntimeException listenerFailure(RuntimeException e) { + if (interruption(e)) { + return e; + } + return new KafkaException("User rebalance callback throws an error", e); + } + + private boolean interruption(RuntimeException e) { + return e instanceof WakeupException || e instanceof InterruptException; + } + + private ConsumerRecords<Object, Object> poll(Duration timeout) { + invokeCallbacks(); + checkWakeup(); + if (!subscribed) { + violations.add(name + " polled without a subscription"); + throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); + } + checkSuspended(); + lastPollStart = dst.now(); + lastPollReturned = false; + rebalanceIfNeeded(); + // [A] the auto-commit of the positions, after the group was joined and before the fetch + if (autoCommit && dst.now() - lastAutoCommit >= autoCommitIntervalMs) { + autoCommit("interval"); + } + if (faults && dst.chance(pollFailRate)) { + dst.block(dst.between(1, 50), name + " poll fails"); + ledger.record("pollFault", "c", name); + throw Dst.injected(new KafkaException("dst: poll failed")); + } + long waitMillis = Math.max(1, timeout.toMillis()); + boolean ready = dst.awaitUntil( + () -> wakeup || closed || fetchable() || generation != FakeKafka.this.generation, waitMillis, + name + " poll"); + if (ready && !wakeup && !closed) { + // reads take effect at the end of their latency + dst.block(dst.between(1, 20), name + " fetch"); + } + if (closed) { + violations.add(name + " closed while it polls"); + throw new IllegalStateException("This consumer has already been closed."); + } + checkWakeup(); + rebalanceIfNeeded(); + Map<TopicPartition, List<ConsumerRecord<Object, Object>>> map = new LinkedHashMap<>(); + int budget = maxPollRecords; + if (budget > 1 && dst.chance(0.3)) { + // [P] "fewer records might be returned" + budget = (int) dst.between(1, budget - 1); + } + for (TopicPartition tp : owned) { + if (paused.contains(tp) || budget == 0) { + continue; + } + List<Rec> partitionLog = log.get(tp.partition()); + long pos = position.get(tp); + List<ConsumerRecord<Object, Object>> list = new ArrayList<>(); + while (budget > 0 && pos < partitionLog.size()) { + Rec r = partitionLog.get((int) pos); + list.add(new ConsumerRecord<>( + TOPIC, tp.partition(), r.offset, Dst.EPOCH_MILLIS + dst.now(), TimestampType.CREATE_TIME, -1, + -1, null, r.id, new RecordHeaders(), Optional.empty(), Optional.empty())); + ledger.record("fetched", "p" + tp.partition(), r.id); + pos++; + budget--; + } + position.put(tp, pos); + if (!list.isEmpty()) { + map.put(tp, list); + } + } + if (!map.isEmpty()) { + dst.trace("fake: " + name + " poll returns " + map.values().stream().mapToInt(List::size).sum() + + " records " + positions()); + } + lastPollReturned = true; + return new ConsumerRecords<>(map, Map.of()); + } + + /** + * The check of a suspended route: once the consumer completed a poll that started after the route was suspended + * (and then handled its records and its pause request), it must have paused its partitions. + */ + private void checkSuspended() { + if (suspendedSince >= 0 && lastPollStart >= suspendedSince && lastPollReturned) { + Set<TopicPartition> fetching = new TreeSet<>(FakeKafka::compare); + fetching.addAll(owned); + fetching.removeAll(paused); + if (!fetching.isEmpty()) { + suspendViolations.add(name + " still fetches " + fetching + " at t=" + dst.now() + + " after the route was suspended at t=" + suspendedSince); + } + } + } + + private String positions() { + return "positions=" + position; + } + + private boolean fetchable() { + for (TopicPartition tp : owned) { + if (!paused.contains(tp) && position.get(tp) < log.get(tp.partition()).size()) { + return true; + } + } + return false; + } + + /** [G] an eager rebalance inside poll, when the group changed since this member last joined. */ + private void rebalanceIfNeeded() { + if (generation == FakeKafka.this.generation) { + return; + } + // the join and sync round trips of the group + dst.block(dst.between(1, 50), name + " rebalance"); + if (closed || !subscribed) { + return; + } + if (autoCommit) { + autoCommit("before revocation"); + } + if (!owned.isEmpty() && listener != null) { + RuntimeException failure = revoke(); + if (interruption(failure)) { + // [L] thrown at once: the member keeps its partitions and joins again in its next poll + throw failure; + } + if (failure != null) { + // [L] the partitions are given up, the member joins again in its next poll + owned.clear(); + paused.clear(); + position.clear(); + throw listenerFailure(failure); + } + } + if (closed || !subscribed) { + return; + } + long gen = FakeKafka.this.generation; + Set<TopicPartition> assigned = assignmentOf(this); + owned.clear(); + paused.clear(); + position.clear(); + owned.addAll(assigned); + for (TopicPartition tp : owned) { + Long c = committed.get(tp.partition()); + position.put(tp, c != null ? c : 0L); + } + generation = gen; + dst.trace("fake: " + name + " assigned " + owned + " (generation " + gen + ") " + positions()); + if (listener != null) { + try { + listener.onPartitionsAssigned(copyOf(owned)); + } catch (RuntimeException e) { + // [L] the partitions are taken, then poll throws + throw listenerFailure(e); + } + } + } + + /** [A] commits the positions (before the fetch of a poll, before a revocation, at close). */ + private void autoCommit(String why) { + lastAutoCommit = dst.now(); + if (owned.isEmpty()) { + return; + } + if (faults && dst.chance(commitFailRate)) { + dst.trace("fake: " + name + " auto-commit (" + why + ") fails"); + ledger.record("commitFault", "c", name); + return; + } + Map<TopicPartition, OffsetAndMetadata> offsets = new TreeMap<>(FakeKafka::compare); + for (TopicPartition tp : owned) { + offsets.put(tp, new OffsetAndMetadata(position.get(tp))); + } + apply(this, offsets, "auto-commit (" + why + ")"); + } + + @SuppressWarnings("unchecked") + private void commitSync(Object[] args) { + invokeCallbacks(); + checkWakeup(); + Map<TopicPartition, OffsetAndMetadata> offsets = args != null && args.length > 0 && args[0] instanceof Map + ? (Map<TopicPartition, OffsetAndMetadata>) args[0] : consumedPositions(); + long timeout = defaultApiTimeoutMs; + if (args != null && args.length > 0 && args[args.length - 1] instanceof Duration d) { + timeout = d.toMillis(); + } + checkCommittable(offsets); + long latency = commitLatency(); + if (faults && dst.chance(commitFailRate) || latency > timeout) { Review Comment: 💡 **Parentheses for clarity**: `&&` has higher precedence than `||`, so this reads as `(faults && dst.chance(commitFailRate)) || (latency > timeout)`, which is likely the intended logic. Adding explicit parentheses would make that intent unambiguous and avoid any future misread. ```suggestion if ((faults && dst.chance(commitFailRate)) || latency > timeout) { ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
