This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new cfa9fe6c3fde CAMEL-27037: camel-zookeeper-master - a group closed
while disconnected or during a create no longer leaves its znode (#30249)
cfa9fe6c3fde is described below
commit cfa9fe6c3fde0e89121536511046bebdc799dce8
Author: allthingssecurity <[email protected]>
AuthorDate: Tue Nov 17 14:30:11 2026 +0530
CAMEL-27037: camel-zookeeper-master - a group closed while disconnected or
during a create no longer leaves its znode (#30249)
…ed or during a create no longer leaves its znode
A ZooKeeperGroup that was closed while the client was disconnected skipped
the delete of its ephemeral znode and forgot its path, and a group closed
while its thread waited for the answer of a create never knew the path of
the znode the server created. With a shared CuratorFramework (the client of
the component or the curator bean) the session lives on, and so did the
znode: as the lowest member it kept the leadership of the group, and no
node ran the route.
When the client is disconnected the znode is now deleted with a guaranteed
delete in the background, which Curator does once the client is connected
again. When a create did not answer and the client is connected (asked of
the client too, as the close removes the listener of the group first and
the group may not know yet that the client is connected again), the group
deletes the znodes of the path that carry its uuid, as it already does for
the duplicate of a create whose answer was lost.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../group/internal/ZooKeeperGroup.java | 62 ++++-
.../group/internal/ZooKeeperGroupCloseTest.java | 254 +++++++++++++++++++++
2 files changed, 311 insertions(+), 5 deletions(-)
diff --git
a/components/camel-zookeeper-master/src/main/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroup.java
b/components/camel-zookeeper-master/src/main/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroup.java
index ce092dcfb568..c683edee9734 100644
---
a/components/camel-zookeeper-master/src/main/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroup.java
+++
b/components/camel-zookeeper-master/src/main/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroup.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -259,18 +260,36 @@ public class ZooKeeperGroup<T extends NodeState>
implements Group<T> {
if (state == null) {
if (id != null) {
try {
- if (isConnected()) {
+ // the state of the client, not the flag of the group: the
flag follows the connection state events,
+ // so it can still say connected after the client lost the
connection, and a foreground delete would
+ // then hold the close for the connection timeout times
the retries
+ if (client.getZookeeperClient().isConnected()) {
client.delete().guaranteed().forPath(id);
- unstable.set(false);
+ } else {
+ // the client is disconnected, but its session may
still be alive (it reconnects within the session
+ // timeout), and the znode would then stay in the
group as long as the session lives. A guaranteed
+ // delete in the background does not wait for the
connection here: Curator does it once the client
+ // is connected again, and tries again until it
succeeds (if the session expired first, ZooKeeper
+ // removed the znode already)
+
client.delete().guaranteed().inBackground().forPath(id);
}
+ unstable.set(false);
} catch (KeeperException.NoNodeException e) {
// Ignore
} finally {
id = null;
}
- } else if (creating.get()) {
- LOG.warn("Ephemeral node could be created in the registry, but
ZooKeeper group didn't record its id");
- unstable.set(true);
+ }
+ if (creating.get()) {
+ // the state of the client here too: the group may not have
been told yet that it is connected again
+ // (a close removes the listener of the group first), and the
create that did not answer may have been
+ // made just after the reconnect
+ if (client.getZookeeperClient().isConnected()) {
+ deleteUnrecordedNodes();
+ } else {
+ LOG.warn("Ephemeral node could be created in the registry,
but ZooKeeper group didn't record its id");
+ unstable.set(true);
+ }
}
} else if (isConnected()) {
// We could have created the sequence, but then have crashed and
our entry is already registered.
@@ -306,6 +325,39 @@ public class ZooKeeperGroup<T extends NodeState>
implements Group<T> {
return pathId;
}
+ // a create of this group did not answer (the connection was lost, or the
close of the group interrupted the thread
+ // that waited for the answer), so the group does not know the path of the
znode it may have made. The znodes of this
+ // group carry its uuid: they are deleted, so that none is left in the
group as long as the session lives. The reads
+ // come after the create in the order of the session, so they see the
znode if the create was applied
+ private void deleteUnrecordedNodes() throws Exception {
+ List<String> children;
+ try {
+ children = client.getChildren().forPath(path);
+ } catch (KeeperException.NoNodeException e) {
+ children = Collections.emptyList();
+ }
+ for (String child : children) {
+ String fullPath = ZKPaths.makePath(path, child);
+ T node;
+ try {
+ node = decode(client.getData().forPath(fullPath));
+ } catch (KeeperException.NoNodeException | IllegalStateException
e) {
+ // deleted meanwhile, or not the data of a member
+ continue;
+ }
+ if (uuid.equals(node.uuid)) {
+ LOG.debug("Deleting znode whose create did not answer: {}",
fullPath);
+ try {
+ client.delete().guaranteed().forPath(fullPath);
+ } catch (KeeperException.NoNodeException e) {
+ // Ignore
+ }
+ }
+ }
+ creating.set(false);
+ unstable.set(false);
+ }
+
private void updateEphemeralNode(T state) throws Exception {
state.uuid = uuid;
client.setData().forPath(id, encode(state));
diff --git
a/components/camel-zookeeper-master/src/test/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroupCloseTest.java
b/components/camel-zookeeper-master/src/test/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroupCloseTest.java
new file mode 100644
index 000000000000..bb91435f55d4
--- /dev/null
+++
b/components/camel-zookeeper-master/src/test/java/org/apache/camel/component/zookeepermaster/group/internal/ZooKeeperGroupCloseTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.zookeepermaster.group.internal;
+
+import java.io.File;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Type;
+import java.lang.reflect.TypeVariable;
+import java.net.InetSocketAddress;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.camel.component.zookeepermaster.group.NodeState;
+import org.apache.camel.test.AvailablePortFinder;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.listen.ListenerManager;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.curator.retry.RetryOneTime;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * A {@link ZooKeeperGroup} that is closed must not leave its ephemeral znode
in the group while the session of the
+ * client lives on (the client is shared, such as the CuratorFramework of the
component or the curator bean), as the
+ * znode would keep the leadership of the group or a place ahead of this
node's next znode.
+ */
+class ZooKeeperGroupCloseTest {
+
+ private static final String PATH =
"/camel/zookeepermaster/clusters/master/close";
+
+ @TempDir
+ Path dataDir;
+
+ private final int port = AvailablePortFinder.getNextAvailable();
+
+ private ZooKeeperServer server;
+ private NIOServerCnxnFactory factory;
+ private CuratorFramework curator;
+
+ @BeforeEach
+ void start() throws Exception {
+ startServer();
+ curator = CuratorFrameworkFactory.builder()
+ .connectString("127.0.0.1:" + port)
+ .sessionTimeoutMs(30000)
+ .connectionTimeoutMs(5000)
+ .retryPolicy(new RetryOneTime(100))
+ .build();
+ curator.start();
+ assertTrue(curator.blockUntilConnected(30, TimeUnit.SECONDS));
+ }
+
+ @AfterEach
+ void stop() {
+ if (curator != null) {
+ curator.close();
+ }
+ stopServer();
+ }
+
+ private void startServer() throws Exception {
+ // the same data directory: a restarted server keeps the sessions and
their ephemeral znodes
+ File dir = dataDir.toFile();
+ server = new ZooKeeperServer(dir, dir, 2000);
+ factory = new NIOServerCnxnFactory();
+ factory.configure(new InetSocketAddress("127.0.0.1", port), 60);
+ factory.startup(server);
+ }
+
+ private void stopServer() {
+ if (factory != null) {
+ factory.shutdown();
+ factory = null;
+ }
+ if (server != null) {
+ server.shutdown();
+ server = null;
+ }
+ }
+
+ private List<String> members() throws Exception {
+ return curator.getChildren().forPath(PATH);
+ }
+
+ @Test
+ void groupClosedWhileDisconnectedDeletesItsZnodeOnceReconnected() throws
Exception {
+ ZooKeeperGroup<NodeState> group = new ZooKeeperGroup<>(
+ curator, PATH, NodeState.class,
+ Executors.newSingleThreadExecutor());
+ group.start();
+ group.update(new NodeState("test", "container1"));
+ await().atMost(20, TimeUnit.SECONDS).until(() -> group.getId() !=
null);
+ assertEquals(1, members().size());
+ long session =
curator.getZookeeperClient().getZooKeeper().getSessionId();
+
+ // the connection is lost (shorter than the session timeout), and the
group is closed meanwhile
+ stopServer();
+ await().atMost(20, TimeUnit.SECONDS).until(() -> !group.isConnected());
+ group.close();
+
+ // the client reconnects to the same session: its ephemeral znodes are
still there
+ startServer();
+ assertTrue(curator.blockUntilConnected(30, TimeUnit.SECONDS));
+ assertEquals(session,
curator.getZookeeperClient().getZooKeeper().getSessionId());
+
+ await().atMost(20, TimeUnit.SECONDS).untilAsserted(
+ () -> assertEquals(List.of(), members(), "The closed group
must not leave its znode in the group"));
+ }
+
+ @Test
+ void groupClosedDuringACreateDeletesTheZnode() throws Exception {
+ CountDownLatch inCreate = new CountDownLatch(1);
+ ZooKeeperGroup<NodeState> group = new ZooKeeperGroup<>(
+ interruptedCreates(curator, inCreate), PATH,
+ NodeState.class, Executors.newSingleThreadExecutor());
+ group.start();
+ group.update(new NodeState("test", "container1"));
+
+ // the create is applied, and the thread of the group waits for its
answer when the group is closed
+ assertTrue(inCreate.await(20, TimeUnit.SECONDS));
+ assertEquals(1, members().size());
+ group.close();
+
+ assertEquals(List.of(), members(), "The closed group must not leave
its znode in the group");
+ }
+
+ @Test
+ void
groupClosedDuringACreateBeforeItKnowsItIsConnectedAgainDeletesTheZnode() throws
Exception {
+ CountDownLatch inCreate = new CountDownLatch(1);
+ ZooKeeperGroup<NodeState> group = new ZooKeeperGroup<>(
+ interruptedCreates(curator, inCreate), PATH,
+ NodeState.class, Executors.newSingleThreadExecutor());
+ group.start();
+ group.update(new NodeState("test", "container1"));
+ assertTrue(inCreate.await(20, TimeUnit.SECONDS));
+
+ // the group was told the connection was lost, and not yet that it is
back (the client is connected): the close
+ // removes the listener of the group, so it is not told any more
+ ((ListenerManager<?, ?>) curator.getConnectionStateListenable())
+ .forEach(listener -> ((ConnectionStateListener)
listener).stateChanged(curator, ConnectionState.SUSPENDED));
+ assertFalse(group.isConnected());
+ group.close();
+
+ assertEquals(List.of(), members(), "The closed group must not leave
its znode in the group");
+ }
+
+ /**
+ * The client, whose creates are applied and then wait for their answer
until the thread is interrupted, as when a
+ * close interrupts the thread that waits for the answer of a create the
server applied: the ZooKeeper client throws
+ * an InterruptedException, and the caller does not know the path of the
znode.
+ */
+ private static CuratorFramework interruptedCreates(CuratorFramework
client, CountDownLatch inCreate) {
+ return (CuratorFramework)
Proxy.newProxyInstance(ZooKeeperGroupCloseTest.class.getClassLoader(),
+ new Class<?>[] { CuratorFramework.class }, (proxy, method,
args) -> {
+ Object result = invoke(client, method, args);
+ return "create".equals(method.getName())
+ ? createBuilder(result, method.getReturnType(),
inCreate) : result;
+ });
+ }
+
+ // a step of the fluent create builder
(create().creatingParentsIfNeeded().withMode(mode).forPath(path, data))
+ private static Object createBuilder(Object target, Class<?> type,
CountDownLatch inCreate) {
+ return
Proxy.newProxyInstance(ZooKeeperGroupCloseTest.class.getClassLoader(), new
Class<?>[] { type },
+ (proxy, method, args) -> {
+ Object result = invoke(target, method, args);
+ if ("forPath".equals(method.getName())) {
+ inCreate.countDown();
+ // waits for the answer until the close of the group
interrupts this thread
+ new CountDownLatch(1).await();
+ return result;
+ }
+ Class<?> next = returnType(method, type);
+ return next != null && result != null ?
createBuilder(result, next, inCreate) : result;
+ });
+ }
+
+ private static Object invoke(Object target, Method method, Object[] args)
throws Throwable {
+ try {
+ return method.invoke(target, args);
+ } catch (InvocationTargetException e) {
+ throw e.getCause();
+ }
+ }
+
+ // the interface a builder method returns: Curator's builders are generic
(CreateModable<T>.withMode returns T), so a
+ // type variable is bound through the generic interfaces of the builder it
is called on
+ private static Class<?> returnType(Method method, Class<?> on) {
+ if (method.getReturnType().isInterface()) {
+ return method.getReturnType();
+ }
+ if (method.getGenericReturnType() instanceof TypeVariable<?> variable)
{
+ Type bound = bind(on, variable);
+ if (bound instanceof ParameterizedType parameterized) {
+ bound = parameterized.getRawType();
+ }
+ if (bound instanceof Class<?> c && c.isInterface()) {
+ return c;
+ }
+ }
+ return null;
+ }
+
+ private static Type bind(Class<?> type, TypeVariable<?> variable) {
+ for (Type generic : type.getGenericInterfaces()) {
+ if (generic instanceof ParameterizedType parameterized
+ && parameterized.getRawType() ==
variable.getGenericDeclaration()) {
+ TypeVariable<?>[] parameters = ((Class<?>)
parameterized.getRawType()).getTypeParameters();
+ for (int i = 0; i < parameters.length; i++) {
+ if (parameters[i].equals(variable)) {
+ return parameterized.getActualTypeArguments()[i];
+ }
+ }
+ }
+ Class<?> raw = generic instanceof ParameterizedType parameterized
+ ? (Class<?>) parameterized.getRawType() : (Class<?>)
generic;
+ Type found = bind(raw, variable);
+ if (found != null) {
+ return found;
+ }
+ }
+ return null;
+ }
+}