allthingssecurity commented on code in PR #30130: URL: https://github.com/apache/camel/pull/30130#discussion_r4593960237
########## components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/dst/ServiceBusWorld.java: ########## @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.azure.servicebus.dst; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import com.azure.messaging.servicebus.ServiceBusErrorContext; +import com.azure.messaging.servicebus.ServiceBusReceivedMessageContext; +import org.apache.camel.AsyncCallback; +import org.apache.camel.Exchange; +import org.apache.camel.Route; +import org.apache.camel.ServiceStatus; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.azure.servicebus.ServiceBusComponent; +import org.apache.camel.component.azure.servicebus.ServiceBusConfiguration; +import org.apache.camel.component.azure.servicebus.ServiceBusConstants; +import org.apache.camel.component.azure.servicebus.ServiceBusEndpoint; +import org.apache.camel.component.azure.servicebus.client.ServiceBusClientFactory; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.spi.ShutdownAware; +import org.apache.camel.support.AsyncProcessorSupport; +import org.apache.camel.support.service.ServiceHelper; +import org.apache.camel.test.dst.Dst; +import org.apache.camel.test.dst.DstCamel; +import org.apache.camel.test.dst.DstExecutor; +import org.apache.camel.test.dst.DstLedger; +import org.apache.camel.test.dst.DstWorld; +import org.apache.camel.test.dst.Invariants; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The world around a camel-azure-servicebus consumer in peek-lock mode: a fake queue ({@link FakeServiceBus}), a + * sender, a route with an asynchronous step (a downstream call) that is sometimes longer than the lock duration and + * sometimes fails, and an operator that stops and starts the route while messages are processed. Faults: lock renewals + * fail or take long, settlements fail. + * <p> + * The route is asynchronous, so processMessage returns at once and the lock of a routed message is kept only by the + * lock renewer of the consumer (maxAutoLockRenewDuration), which renews every message whose lock expires before the run + * after the next one of its period task (every 10 seconds, or a second later). A message the consumer holds in + * processMessage without routing it keeps its lock through the renewal of the processor client while the call runs (see + * FakeServiceBus [A]); the invariants count only the routed deliveries as held by the consumer. The seed picks the lock + * duration of the queue, the max delivery count, maxConcurrentCalls (with the asynchronous route it bounds only the + * concurrent processMessage calls), maxAutoLockRenewDuration (60 or 300 seconds), enableDeadLettering, the latency of + * the renewals and the fault rates. + * <p> + * Properties: no lock is lost, and no message is delivered again, while the consumer holds the message, unless an + * injected renewal of its lock failed, the route was stopped while it was processed (once a stop timed out the consumer + * stops renewing, and the exchange goes on), or maxAutoLockRenewDuration passed. The consumer renews a lock again after + * a transient renewal error, but only at its next run: the lock renewer runs about every 11 seconds, so one transient + * renewal error can still lose a 30 second lock. {@code -Ddst.servicebus.strictRenew=true} does not excuse a failed + * renewal, and still fails on some seeds. The message of a completed exchange is completed, unless its settlement + * failed, found its lock lost under one of those excuses, or came after the close() of a stop that timed out while the + * exchange was in flight (by design); a graceful stop keeps the processor client open for the exchanges in flight (the + * V2 processor of the SDK closes its receiver on stop(), see FakeServiceBus; CAMEL-26970), so a settlement inside the + * graceful wait must find it open; a message is completed only after its exchange completed, and abandoned or + * dead-lettered only after it failed (or was in flight when a stop timed out, see below); once the faults cleared every + * message is completed or dead-lettered; the consumer settles each delivery at most once and renews only what it holds, + * through an open client. + * <p> + * The operator stops the route as DefaultShutdownStrategy does (its ShutdownTask), with the wait of the strategy as + * tasks of the operator (see {@link #gracefulStop}): a processMessage call that waits until the consumer stops would + * otherwise sit above the stop on the one stack, and could not see the stop. When the stop times out, the consumer is + * forced to stop as the strategy does, and the exchanges still in flight fail (their route rejects them). + * <p> + * Not explored: sessions, receiveAndDelete mode, a processorClient given by the user (the consumer does not renew its + * locks), a synchronous route (a processMessage call that lasts as long as the exchange), maxAutoLockRenewDuration=0 + * (no renewal by the consumer or by the processor client), topics. + */ +@DstWorld(component = "azure-servicebus", + fake = "FakeServiceBus: a queue in peek-lock mode (lock duration, delivery count, dead-letter queue) behind" + + " the processor and receiver clients of ServiceBusClientFactory", + covers = { + "the lock renewer of the consumer (PEEK_LOCK, maxAutoLockRenewDuration) with an asynchronous route", + "processing that takes longer than the lock duration", + "complete, abandon and dead-letter, also after the lock was lost", + "maxConcurrentCalls of one to three (with the asynchronous route it bounds the processMessage calls)", + "route stop and start while messages are processed" }, + faults = { + "a lock renewal fails", "a lock renewal takes long", "a settlement fails", "processing fails", + "processing is slower than the lock duration" }, + invariants = { + "no lock is lost and no message is delivered again while the consumer holds it, unless a renewal" + + " failed, the route was stopped or maxAutoLockRenewDuration passed", + "the message of a completed exchange is completed, unless its settlement failed, its lock was" + + " lost as excused above, or a forced stop closed the client first", + "a message is completed only after its exchange completed", + "a message is abandoned or dead-lettered only after its exchange failed", + "every message is completed or dead-lettered once the faults cleared", + "the consumer settles a delivery at most once and renews only what it holds, through an open client" }, + issues = { "CAMEL-25701", "CAMEL-26956", "CAMEL-26969", "CAMEL-26970" }) +final class ServiceBusWorld { + + static final String ROUTE = "servicebus"; + /** Excuses a lost lock only when the renewal found it lost (not after an injected transient renewal failure). */ + static final boolean STRICT_RENEW = Boolean.getBoolean("dst.servicebus.strictRenew"); Review Comment: Named CAMEL-27066 on STRICT_RENEW instead (CAMEL-26969 is fixed on main), in 5fa184ef1e1c. ########## components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/dst/ServiceBusWorld.java: ########## @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.azure.servicebus.dst; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import com.azure.messaging.servicebus.ServiceBusErrorContext; +import com.azure.messaging.servicebus.ServiceBusReceivedMessageContext; +import org.apache.camel.AsyncCallback; +import org.apache.camel.Exchange; +import org.apache.camel.Route; +import org.apache.camel.ServiceStatus; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.azure.servicebus.ServiceBusComponent; +import org.apache.camel.component.azure.servicebus.ServiceBusConfiguration; +import org.apache.camel.component.azure.servicebus.ServiceBusConstants; +import org.apache.camel.component.azure.servicebus.ServiceBusEndpoint; +import org.apache.camel.component.azure.servicebus.client.ServiceBusClientFactory; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.spi.ShutdownAware; +import org.apache.camel.support.AsyncProcessorSupport; +import org.apache.camel.support.service.ServiceHelper; +import org.apache.camel.test.dst.Dst; +import org.apache.camel.test.dst.DstCamel; +import org.apache.camel.test.dst.DstExecutor; +import org.apache.camel.test.dst.DstLedger; +import org.apache.camel.test.dst.DstWorld; +import org.apache.camel.test.dst.Invariants; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The world around a camel-azure-servicebus consumer in peek-lock mode: a fake queue ({@link FakeServiceBus}), a + * sender, a route with an asynchronous step (a downstream call) that is sometimes longer than the lock duration and + * sometimes fails, and an operator that stops and starts the route while messages are processed. Faults: lock renewals + * fail or take long, settlements fail. + * <p> + * The route is asynchronous, so processMessage returns at once and the lock of a routed message is kept only by the + * lock renewer of the consumer (maxAutoLockRenewDuration), which renews every message whose lock expires before the run + * after the next one of its period task (every 10 seconds, or a second later). A message the consumer holds in + * processMessage without routing it keeps its lock through the renewal of the processor client while the call runs (see + * FakeServiceBus [A]); the invariants count only the routed deliveries as held by the consumer. The seed picks the lock + * duration of the queue, the max delivery count, maxConcurrentCalls (with the asynchronous route it bounds only the + * concurrent processMessage calls), maxAutoLockRenewDuration (60 or 300 seconds), enableDeadLettering, the latency of + * the renewals and the fault rates. + * <p> + * Properties: no lock is lost, and no message is delivered again, while the consumer holds the message, unless an + * injected renewal of its lock failed, the route was stopped while it was processed (once a stop timed out the consumer + * stops renewing, and the exchange goes on), or maxAutoLockRenewDuration passed. The consumer renews a lock again after + * a transient renewal error, but only at its next run: the lock renewer runs about every 11 seconds, so one transient + * renewal error can still lose a 30 second lock. {@code -Ddst.servicebus.strictRenew=true} does not excuse a failed + * renewal, and still fails on some seeds. The message of a completed exchange is completed, unless its settlement + * failed, found its lock lost under one of those excuses, or came after the close() of a stop that timed out while the + * exchange was in flight (by design); a graceful stop keeps the processor client open for the exchanges in flight (the + * V2 processor of the SDK closes its receiver on stop(), see FakeServiceBus; CAMEL-26970), so a settlement inside the + * graceful wait must find it open; a message is completed only after its exchange completed, and abandoned or + * dead-lettered only after it failed (or was in flight when a stop timed out, see below); once the faults cleared every + * message is completed or dead-lettered; the consumer settles each delivery at most once and renews only what it holds, + * through an open client. + * <p> + * The operator stops the route as DefaultShutdownStrategy does (its ShutdownTask), with the wait of the strategy as + * tasks of the operator (see {@link #gracefulStop}): a processMessage call that waits until the consumer stops would + * otherwise sit above the stop on the one stack, and could not see the stop. When the stop times out, the consumer is + * forced to stop as the strategy does, and the exchanges still in flight fail (their route rejects them). + * <p> + * Not explored: sessions, receiveAndDelete mode, a processorClient given by the user (the consumer does not renew its + * locks), a synchronous route (a processMessage call that lasts as long as the exchange), maxAutoLockRenewDuration=0 + * (no renewal by the consumer or by the processor client), topics. + */ +@DstWorld(component = "azure-servicebus", + fake = "FakeServiceBus: a queue in peek-lock mode (lock duration, delivery count, dead-letter queue) behind" + + " the processor and receiver clients of ServiceBusClientFactory", + covers = { + "the lock renewer of the consumer (PEEK_LOCK, maxAutoLockRenewDuration) with an asynchronous route", + "processing that takes longer than the lock duration", + "complete, abandon and dead-letter, also after the lock was lost", + "maxConcurrentCalls of one to three (with the asynchronous route it bounds the processMessage calls)", + "route stop and start while messages are processed" }, + faults = { + "a lock renewal fails", "a lock renewal takes long", "a settlement fails", "processing fails", + "processing is slower than the lock duration" }, + invariants = { + "no lock is lost and no message is delivered again while the consumer holds it, unless a renewal" + + " failed, the route was stopped or maxAutoLockRenewDuration passed", + "the message of a completed exchange is completed, unless its settlement failed, its lock was" + + " lost as excused above, or a forced stop closed the client first", + "a message is completed only after its exchange completed", + "a message is abandoned or dead-lettered only after its exchange failed", + "every message is completed or dead-lettered once the faults cleared", + "the consumer settles a delivery at most once and renews only what it holds, through an open client" }, + issues = { "CAMEL-25701", "CAMEL-26956", "CAMEL-26969", "CAMEL-26970" }) +final class ServiceBusWorld { + Review Comment: Added CAMEL-27066 to @DstWorld(issues) and regenerated the JSON, catalog copy and table, in 5fa184ef1e1c. ########## components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/dst/ServiceBusWorld.java: ########## @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.azure.servicebus.dst; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import com.azure.messaging.servicebus.ServiceBusErrorContext; +import com.azure.messaging.servicebus.ServiceBusReceivedMessageContext; +import org.apache.camel.AsyncCallback; +import org.apache.camel.Exchange; +import org.apache.camel.Route; +import org.apache.camel.ServiceStatus; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.azure.servicebus.ServiceBusComponent; +import org.apache.camel.component.azure.servicebus.ServiceBusConfiguration; +import org.apache.camel.component.azure.servicebus.ServiceBusConstants; +import org.apache.camel.component.azure.servicebus.ServiceBusEndpoint; +import org.apache.camel.component.azure.servicebus.client.ServiceBusClientFactory; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.spi.ShutdownAware; +import org.apache.camel.support.AsyncProcessorSupport; +import org.apache.camel.support.service.ServiceHelper; +import org.apache.camel.test.dst.Dst; +import org.apache.camel.test.dst.DstCamel; +import org.apache.camel.test.dst.DstExecutor; +import org.apache.camel.test.dst.DstLedger; +import org.apache.camel.test.dst.DstWorld; +import org.apache.camel.test.dst.Invariants; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The world around a camel-azure-servicebus consumer in peek-lock mode: a fake queue ({@link FakeServiceBus}), a + * sender, a route with an asynchronous step (a downstream call) that is sometimes longer than the lock duration and + * sometimes fails, and an operator that stops and starts the route while messages are processed. Faults: lock renewals + * fail or take long, settlements fail. + * <p> + * The route is asynchronous, so processMessage returns at once and the lock of a routed message is kept only by the + * lock renewer of the consumer (maxAutoLockRenewDuration), which renews every message whose lock expires before the run + * after the next one of its period task (every 10 seconds, or a second later). A message the consumer holds in + * processMessage without routing it keeps its lock through the renewal of the processor client while the call runs (see + * FakeServiceBus [A]); the invariants count only the routed deliveries as held by the consumer. The seed picks the lock + * duration of the queue, the max delivery count, maxConcurrentCalls (with the asynchronous route it bounds only the + * concurrent processMessage calls), maxAutoLockRenewDuration (60 or 300 seconds), enableDeadLettering, the latency of + * the renewals and the fault rates. + * <p> + * Properties: no lock is lost, and no message is delivered again, while the consumer holds the message, unless an + * injected renewal of its lock failed, the route was stopped while it was processed (once a stop timed out the consumer + * stops renewing, and the exchange goes on), or maxAutoLockRenewDuration passed. The consumer renews a lock again after + * a transient renewal error, but only at its next run: the lock renewer runs about every 11 seconds, so one transient + * renewal error can still lose a 30 second lock. {@code -Ddst.servicebus.strictRenew=true} does not excuse a failed + * renewal, and still fails on some seeds. The message of a completed exchange is completed, unless its settlement + * failed, found its lock lost under one of those excuses, or came after the close() of a stop that timed out while the + * exchange was in flight (by design); a graceful stop keeps the processor client open for the exchanges in flight (the + * V2 processor of the SDK closes its receiver on stop(), see FakeServiceBus; CAMEL-26970), so a settlement inside the + * graceful wait must find it open; a message is completed only after its exchange completed, and abandoned or + * dead-lettered only after it failed (or was in flight when a stop timed out, see below); once the faults cleared every + * message is completed or dead-lettered; the consumer settles each delivery at most once and renews only what it holds, + * through an open client. + * <p> + * The operator stops the route as DefaultShutdownStrategy does (its ShutdownTask), with the wait of the strategy as + * tasks of the operator (see {@link #gracefulStop}): a processMessage call that waits until the consumer stops would + * otherwise sit above the stop on the one stack, and could not see the stop. When the stop times out, the consumer is + * forced to stop as the strategy does, and the exchanges still in flight fail (their route rejects them). + * <p> + * Not explored: sessions, receiveAndDelete mode, a processorClient given by the user (the consumer does not renew its + * locks), a synchronous route (a processMessage call that lasts as long as the exchange), maxAutoLockRenewDuration=0 + * (no renewal by the consumer or by the processor client), topics. + */ +@DstWorld(component = "azure-servicebus", + fake = "FakeServiceBus: a queue in peek-lock mode (lock duration, delivery count, dead-letter queue) behind" + + " the processor and receiver clients of ServiceBusClientFactory", + covers = { + "the lock renewer of the consumer (PEEK_LOCK, maxAutoLockRenewDuration) with an asynchronous route", + "processing that takes longer than the lock duration", + "complete, abandon and dead-letter, also after the lock was lost", + "maxConcurrentCalls of one to three (with the asynchronous route it bounds the processMessage calls)", + "route stop and start while messages are processed" }, + faults = { + "a lock renewal fails", "a lock renewal takes long", "a settlement fails", "processing fails", + "processing is slower than the lock duration" }, + invariants = { + "no lock is lost and no message is delivered again while the consumer holds it, unless a renewal" + + " failed, the route was stopped or maxAutoLockRenewDuration passed", + "the message of a completed exchange is completed, unless its settlement failed, its lock was" + + " lost as excused above, or a forced stop closed the client first", + "a message is completed only after its exchange completed", + "a message is abandoned or dead-lettered only after its exchange failed", + "every message is completed or dead-lettered once the faults cleared", + "the consumer settles a delivery at most once and renews only what it holds, through an open client" }, + issues = { "CAMEL-25701", "CAMEL-26956", "CAMEL-26969", "CAMEL-26970" }) +final class ServiceBusWorld { + + static final String ROUTE = "servicebus"; + /** Excuses a lost lock only when the renewal found it lost (not after an injected transient renewal failure). */ + static final boolean STRICT_RENEW = Boolean.getBoolean("dst.servicebus.strictRenew"); + + private final Dst dst; Review Comment: STRICT_RENEW now names CAMEL-27066, in 5fa184ef1e1c. ########## components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/dst/ServiceBusWorld.java: ########## @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.azure.servicebus.dst; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import com.azure.messaging.servicebus.ServiceBusErrorContext; +import com.azure.messaging.servicebus.ServiceBusReceivedMessageContext; +import org.apache.camel.AsyncCallback; +import org.apache.camel.Exchange; +import org.apache.camel.Route; +import org.apache.camel.ServiceStatus; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.azure.servicebus.ServiceBusComponent; +import org.apache.camel.component.azure.servicebus.ServiceBusConfiguration; +import org.apache.camel.component.azure.servicebus.ServiceBusConstants; +import org.apache.camel.component.azure.servicebus.ServiceBusEndpoint; +import org.apache.camel.component.azure.servicebus.client.ServiceBusClientFactory; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.spi.ShutdownAware; +import org.apache.camel.support.AsyncProcessorSupport; +import org.apache.camel.support.service.ServiceHelper; +import org.apache.camel.test.dst.Dst; +import org.apache.camel.test.dst.DstCamel; +import org.apache.camel.test.dst.DstExecutor; +import org.apache.camel.test.dst.DstLedger; +import org.apache.camel.test.dst.DstWorld; +import org.apache.camel.test.dst.Invariants; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The world around a camel-azure-servicebus consumer in peek-lock mode: a fake queue ({@link FakeServiceBus}), a + * sender, a route with an asynchronous step (a downstream call) that is sometimes longer than the lock duration and + * sometimes fails, and an operator that stops and starts the route while messages are processed. Faults: lock renewals + * fail or take long, settlements fail. + * <p> + * The route is asynchronous, so processMessage returns at once and the lock of a routed message is kept only by the + * lock renewer of the consumer (maxAutoLockRenewDuration), which renews every message whose lock expires before the run + * after the next one of its period task (every 10 seconds, or a second later). A message the consumer holds in + * processMessage without routing it keeps its lock through the renewal of the processor client while the call runs (see + * FakeServiceBus [A]); the invariants count only the routed deliveries as held by the consumer. The seed picks the lock + * duration of the queue, the max delivery count, maxConcurrentCalls (with the asynchronous route it bounds only the + * concurrent processMessage calls), maxAutoLockRenewDuration (60 or 300 seconds), enableDeadLettering, the latency of + * the renewals and the fault rates. + * <p> + * Properties: no lock is lost, and no message is delivered again, while the consumer holds the message, unless an + * injected renewal of its lock failed, the route was stopped while it was processed (once a stop timed out the consumer + * stops renewing, and the exchange goes on), or maxAutoLockRenewDuration passed. The consumer renews a lock again after + * a transient renewal error, but only at its next run: the lock renewer runs about every 11 seconds, so one transient + * renewal error can still lose a 30 second lock. {@code -Ddst.servicebus.strictRenew=true} does not excuse a failed + * renewal, and still fails on some seeds. The message of a completed exchange is completed, unless its settlement + * failed, found its lock lost under one of those excuses, or came after the close() of a stop that timed out while the + * exchange was in flight (by design); a graceful stop keeps the processor client open for the exchanges in flight (the + * V2 processor of the SDK closes its receiver on stop(), see FakeServiceBus; CAMEL-26970), so a settlement inside the + * graceful wait must find it open; a message is completed only after its exchange completed, and abandoned or + * dead-lettered only after it failed (or was in flight when a stop timed out, see below); once the faults cleared every + * message is completed or dead-lettered; the consumer settles each delivery at most once and renews only what it holds, + * through an open client. + * <p> + * The operator stops the route as DefaultShutdownStrategy does (its ShutdownTask), with the wait of the strategy as + * tasks of the operator (see {@link #gracefulStop}): a processMessage call that waits until the consumer stops would + * otherwise sit above the stop on the one stack, and could not see the stop. When the stop times out, the consumer is + * forced to stop as the strategy does, and the exchanges still in flight fail (their route rejects them). + * <p> + * Not explored: sessions, receiveAndDelete mode, a processorClient given by the user (the consumer does not renew its + * locks), a synchronous route (a processMessage call that lasts as long as the exchange), maxAutoLockRenewDuration=0 + * (no renewal by the consumer or by the processor client), topics. + */ +@DstWorld(component = "azure-servicebus", + fake = "FakeServiceBus: a queue in peek-lock mode (lock duration, delivery count, dead-letter queue) behind" + + " the processor and receiver clients of ServiceBusClientFactory", + covers = { + "the lock renewer of the consumer (PEEK_LOCK, maxAutoLockRenewDuration) with an asynchronous route", + "processing that takes longer than the lock duration", + "complete, abandon and dead-letter, also after the lock was lost", + "maxConcurrentCalls of one to three (with the asynchronous route it bounds the processMessage calls)", + "route stop and start while messages are processed" }, + faults = { + "a lock renewal fails", "a lock renewal takes long", "a settlement fails", "processing fails", + "processing is slower than the lock duration" }, + invariants = { + "no lock is lost and no message is delivered again while the consumer holds it, unless a renewal" + + " failed, the route was stopped or maxAutoLockRenewDuration passed", + "the message of a completed exchange is completed, unless its settlement failed, its lock was" + + " lost as excused above, or a forced stop closed the client first", + "a message is completed only after its exchange completed", + "a message is abandoned or dead-lettered only after its exchange failed", + "every message is completed or dead-lettered once the faults cleared", + "the consumer settles a delivery at most once and renews only what it holds, through an open client" }, + issues = { "CAMEL-25701", "CAMEL-26956", "CAMEL-26969", "CAMEL-26970" }) +final class ServiceBusWorld { + + static final String ROUTE = "servicebus"; + /** Excuses a lost lock only when the renewal found it lost (not after an injected transient renewal failure). */ + static final boolean STRICT_RENEW = Boolean.getBoolean("dst.servicebus.strictRenew"); + + private final Dst dst; + private final DstLedger ledger; + private final FakeServiceBus bus; + private final int maxConcurrentCalls; + private final boolean deadLettering; + private final double processFailRate; + private final long maxAutoLockRenewDuration; + private final List<String> violations = new ArrayList<>(); + private final List<String> lostWhileHeld = new ArrayList<>(); + /** The time each delivery was routed (the lock renewer of the consumer counts maxAutoLockRenewDuration from it). */ + private final Map<String, Long> routedAt = new HashMap<>(); + /** + * The outcome of the settlement of each delivery (COMPLETED, ABANDONED, DEAD_LETTERED, fault, lockLost, + * closedByStop, closed). + */ + private final Map<String, String> outcomes = new HashMap<>(); + private final List<String> notCompleted = new ArrayList<>(); + private final List<String> redeliveredWhileHeld = new ArrayList<>(); + private final List<String> wrongSettlements = new ArrayList<>(); + /** + * The deliveries ("id#n") whose exchange is in progress, by message id: routed and not done yet. A delivery the + * consumer holds in processMessage without routing it is not counted (the processor client renews its lock). + */ + private final Map<String, String> held = new HashMap<>(); + /** + * Every delivery whose exchange is in progress (a message delivered again while an earlier delivery of it is still + * in progress has two). + */ + private final Set<String> inProgress = new HashSet<>(); + private final Set<String> renewFaulted = new HashSet<>(); + /** The deliveries in progress when the route was stopped: the consumer stops renewing their locks. */ + private final Set<String> heldAtStop = new HashSet<>(); + /** + * The deliveries in progress when a graceful stop timed out and the consumer was forced to stop: stopping the + * consumer stops the processor of its route, which then rejects the exchange (it fails, and its message is + * abandoned or dead-lettered, through the closed client), even when the downstream call answered. + */ + private final Set<String> forcedAtStop = new HashSet<>(); + /** The deliveries of forcedAtStop abandoned or dead-lettered: that settlement must find the client closed. */ + private final Set<String> forcedSettlements = new HashSet<>(); + /** The deliveries routed, or settled (an abandon when routing failed), by the consumer. */ + private final Set<String> routedOrSettled = new HashSet<>(); + /** + * Whether the operator stopped the route and has not started it again (from deferShutdown until the restart): with + * the CAMEL-26970 fix a call that comes after the stop closed the client returns without routing, as the consumer + * stays in its stopping state until it is started again. + */ + private boolean stopping; + private final Set<String> completed = new HashSet<>(); + private final Set<String> failed = new HashSet<>(); + private boolean faults = true; + /** Exchanges whose asynchronous step has not answered yet. */ + private int inFlight; + private int sendsLeft; + private int opsLeft; + private boolean finished; + private String finalStatus; + private boolean finalSettled; + + ServiceBusWorld(Dst dst) { + this.dst = dst; + this.ledger = new DstLedger(dst); + this.bus = new FakeServiceBus(dst, ledger); + bus.lockDurationMillis = dst.pick(List.of(30_000L, 60_000L)); + bus.maxDeliveryCount = dst.pick(List.of(3, 10)); + bus.renewFailRate = dst.pick(List.of(0.0, 0.05, 0.2)); + bus.settleFailRate = dst.pick(List.of(0.0, 0.05)); + bus.renewLatencyMaxMillis = dst.pick(List.of(0L, 2000L)); + this.maxAutoLockRenewDuration = dst.pick(List.of(60_000L, 300_000L)); + this.maxConcurrentCalls = dst.pick(List.of(1, 2, 3)); + this.deadLettering = dst.chance(0.3); + this.processFailRate = dst.pick(List.of(0.0, 0.05, 0.2)); + bus.onDelivery = this::delivered; + bus.onRenewFault = renewFaulted::add; + bus.onSettle = this::settling; + bus.onSettled = this::settled; + bus.onLockExpired = this::lockExpired; + // what is in flight when the processor client stops: the stop happens inside the route stop, after the + // operator took its own snapshot, and a message can be delivered in between + bus.onProcessorClosed = () -> heldAtStop.addAll(inProgress); + dst.trace("world: " + config()); + } + + private String config() { + return "lockDuration=" + bus.lockDurationMillis + " maxDeliveryCount=" + bus.maxDeliveryCount + + " maxConcurrentCalls=" + maxConcurrentCalls + " maxAutoLockRenewDuration=" + maxAutoLockRenewDuration + + " enableDeadLettering=" + deadLettering + " renewFail=" + bus.renewFailRate + " renewLatencyMax=" Review Comment: The long line was :108 in the invariants annotation, split in 5fa184ef1e1c; config() was already under 128. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
