This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 4468687fb26d CAMEL-27322: camel-file - a cluster view no longer
releases the file lock of another view in the same JVM (#30576)
4468687fb26d is described below
commit 4468687fb26da56fc478f86249bc442426c8eb7f
Author: allthingssecurity <[email protected]>
AuthorDate: Sun Nov 22 21:32:12 2026 +0530
CAMEL-27322: camel-file - a cluster view no longer releases the file lock
of another view in the same JVM (#30576)
The file locks of FileLockClusterView are locks of the process (POSIX fcntl
locks on Linux and macOS): closing any channel of the lock file releases all
the locks of the process on it, while the FileLock of the holder still says
it
is valid. A follower view in the same JVM (two CamelContexts with a
FileLockClusterService on the same root) that found the leader stale opened
the lock file, got an OverlappingFileLockException and closed its channel,
which dropped the lock of the leader view. A member in another JVM could
then
take the lock and become a second leader.
The views of a JVM now register the lock files they hold or try, and a view
does not open a lock file that another view of its JVM has registered.
The key of the lock file in the JVM-wide registry resolves the real path of
the lock file directory, which reads the file system. It ran directly on the
scheduler thread of the service, so a hung network mount could block the
leadership checks of all its views for good. It now runs through the cluster
task executor with the cluster data task timeout and attempts, like the
other
file I/O of the leadership check (CAMEL-25089): the attempt fails and the
check is scheduled again. The key is resolved before it is registered, so a
failed attempt leaves nothing in the registry.
If the directory cannot be resolved, the attempt now fails too, instead of
using the unresolved path, which may differ from the key of another view of
the JVM (opening the lock file fails without the directory anyway).
New tests: a check whose key resolution hangs times out and is scheduled
again, and the lock file is not opened while the key cannot be resolved.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../file/cluster/FileLockClusterView.java | 78 ++++++-
.../cluster/FileLockClusterViewSameJvmTest.java | 258 +++++++++++++++++++++
2 files changed, 332 insertions(+), 4 deletions(-)
diff --git
a/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
b/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
index eb13f444c41b..b12af1dede9e 100644
---
a/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
+++
b/components/camel-file/src/main/java/org/apache/camel/component/file/cluster/FileLockClusterView.java
@@ -26,9 +26,11 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -48,6 +50,12 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
// It isolates the critical section making sure only one service creates
the files.
private static final ReentrantLock contextStartLock = new ReentrantLock();
private static final Logger LOGGER =
LoggerFactory.getLogger(FileLockClusterView.class);
+ // The lock files that a view of this JVM has locked or is trying to lock,
with that view. The file locks are
+ // locks of the process (POSIX fcntl locks on Linux and macOS), and
closing any channel of a file releases all the
+ // locks of the process on it (see FileLock), while the FileLock of the
holder still says it is valid. So a view
+ // must not open (and then close) a channel on a lock file that another
view of the JVM has locked: it does not try
+ // the lock while the path is registered here.
+ private static final Map<Path, FileLockClusterView> LOCK_PATHS = new
ConcurrentHashMap<>();
private final ClusterMember localMember;
private final Path leaderLockPath;
@@ -58,6 +66,8 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
private volatile RandomAccessFile leaderLockFile;
private volatile RandomAccessFile leaderDataFile;
private volatile FileLock lock;
+ // the key of the lock file in LOCK_PATHS while the view holds the lock
(written with lock, under stateLock)
+ private volatile Path lockPathKey;
private int heartbeatTimeoutMultiplier;
private long acquireLockIntervalMilliseconds;
private FileLockClusterTaskExecutor clusterTaskExecutor;
@@ -154,6 +164,7 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
final FileLock heldLock;
final RandomAccessFile heldLockFile;
final RandomAccessFile heldDataFile;
+ final Path heldLockPathKey;
stateLock.lock();
try {
// A leadership check that is running now can no longer publish a
lock it acquires: it re-checks the
@@ -165,9 +176,11 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
heldLock = lock;
heldLockFile = leaderLockFile;
heldDataFile = leaderDataFile;
+ heldLockPathKey = lockPathKey;
lock = null;
leaderLockFile = null;
leaderDataFile = null;
+ lockPathKey = null;
} finally {
stateLock.unlock();
}
@@ -201,6 +214,7 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
releaseFileLock(heldLock);
closeFile(heldLockFile);
closeFile(heldDataFile);
+ unregisterLockPath(heldLockPathKey);
clusterLeaderInfoRef.set(null);
}
}
@@ -208,11 +222,12 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
/**
* The lock, lock file and data file taken over from this view by {@link
#takeOver(long)}.
*/
- private record HeldLock(FileLock lock, RandomAccessFile lockFile,
RandomAccessFile dataFile) {
- void release() {
+ private record HeldLock(FileLock lock, RandomAccessFile lockFile,
RandomAccessFile dataFile, Path lockPathKey) {
+ void release(FileLockClusterView view) {
releaseFileLock(lock);
closeFile(lockFile);
closeFile(dataFile);
+ view.unregisterLockPath(lockPathKey);
}
}
@@ -227,10 +242,11 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
if (gen != generation) {
return null;
}
- HeldLock held = new HeldLock(lock, leaderLockFile, leaderDataFile);
+ HeldLock held = new HeldLock(lock, leaderLockFile, leaderDataFile,
lockPathKey);
lock = null;
leaderLockFile = null;
leaderDataFile = null;
+ lockPathKey = null;
localMember.setStatus(ClusterMemberStatus.FOLLOWER);
return held;
} finally {
@@ -241,6 +257,44 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
private void closeInternal() {
releaseFileLock();
closeLockFiles();
+ // Safe without stateLock: closeInternal() is only called from
doStart() under contextStartLock,
+ // before the first scheduleTryLock(), so no concurrent acquireLock()
writes lockPathKey here.
+ unregisterLockPath(lockPathKey);
+ lockPathKey = null;
+ }
+
+ /**
+ * The key of the lock file in the JVM-wide registry (see {@link
#resolveLockPathKey()}). Resolving it reads the
+ * file system, so it runs through the cluster task executor with the
cluster data task timeout and attempts, like
+ * the other file I/O of the leadership check: on a hung network mount the
attempt fails instead of blocking the
+ * check (and the other checks of the service, which share its scheduler
thread). Returns null if the check was
+ * interrupted.
+ */
+ private Path lockPathKey() throws ExecutionException, TimeoutException {
+ return clusterTaskExecutor.run(ThrowingHelper.wrapAsSupplier(new
ThrowingSupplier<Path, Throwable>() {
+ @Override
+ public Path get() throws Throwable {
+ return resolveLockPathKey();
+ }
+ }));
+ }
+
+ /**
+ * The absolute path of the lock file, with the symbolic links of its
directory resolved, so that two services
+ * configured with different paths to one directory share the key. If the
directory cannot be resolved, the attempt
+ * fails rather than use a key that may differ from the key of another
view (opening the lock file would fail
+ * without the directory anyway).
+ */
+ Path resolveLockPathKey() throws IOException {
+ Path path = leaderLockPath.toAbsolutePath().normalize();
+ Path parent = path.getParent();
+ return parent != null ?
parent.toRealPath().resolve(path.getFileName()) : path;
+ }
+
+ private void unregisterLockPath(Path key) {
+ if (key != null) {
+ LOCK_PATHS.remove(key, this);
+ }
}
private void closeLockFiles() {
@@ -325,7 +379,7 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
localMember.getUuid());
fireLeadershipChangedEvent((CamelClusterMember) null);
clusterLeaderInfoRef.set(null);
- held.release();
+ held.release(this);
}
return;
}
@@ -394,6 +448,20 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
// stopped while the cluster data was read: do not take the lock,
even briefly
return false;
}
+ // resolved before the key is registered, so a timeout leaves nothing
in LOCK_PATHS
+ Path key = lockPathKey();
+ if (key == null) {
+ // interrupted while resolving the key
+ return false;
+ }
+ FileLockClusterView holder = LOCK_PATHS.putIfAbsent(key, this);
+ if (holder != null) {
+ // another view of this JVM holds (or is trying) the lock: trying
it here would fail anyway, and closing
+ // the channel afterwards would release the lock of that view at
OS level (see LOCK_PATHS)
+ LOGGER.debug("Lock on file {} is held by another cluster view of
this JVM (cluster-member-id={}), not trying it",
+ leaderLockPath, holder.localMember.getUuid());
+ return false;
+ }
RandomAccessFile newLockFile = null;
RandomAccessFile newDataFile = null;
FileLock newLock = null;
@@ -412,6 +480,7 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
lock = newLock;
leaderLockFile = newLockFile;
leaderDataFile = newDataFile;
+ lockPathKey = key;
localMember.setStatus(ClusterMemberStatus.LEADER);
clusterLeaderInfoRef.set(null);
published = true;
@@ -430,6 +499,7 @@ public class FileLockClusterView extends
AbstractCamelClusterView {
releaseFileLock(newLock);
closeFile(newLockFile);
closeFile(newDataFile);
+ unregisterLockPath(key);
}
}
return published;
diff --git
a/core/camel-core/src/test/java/org/apache/camel/component/file/cluster/FileLockClusterViewSameJvmTest.java
b/core/camel-core/src/test/java/org/apache/camel/component/file/cluster/FileLockClusterViewSameJvmTest.java
new file mode 100644
index 000000000000..edee24dc60a0
--- /dev/null
+++
b/core/camel-core/src/test/java/org/apache/camel/component/file/cluster/FileLockClusterViewSameJvmTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.cluster;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.CamelContext;
+import org.apache.camel.cluster.CamelClusterView;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Two cluster views of one JVM on the same lock file (two CamelContexts with
a FileLockClusterService on the same
+ * root). The file locks are locks of the process, and closing any channel of
the lock file releases them all at OS
+ * level (see {@link java.nio.channels.FileLock}): the follower must not open
(and close) a channel on the lock file
+ * while the leader of its JVM holds the lock, as a member in another JVM
could then take the lock too.
+ */
+class FileLockClusterViewSameJvmTest {
+
+ private static final String NAMESPACE = "ns";
+
+ @TempDir
+ Path root;
+
+ private final List<CamelContext> contexts = new ArrayList<>();
+
+ @AfterEach
+ void stopContexts() {
+ contexts.forEach(CamelContext::stop);
+ }
+
+ @Test
+ void followerDoesNotOpenTheLockFileHeldInItsJvm() throws Exception {
+ CamelContext contextA = startContext();
+ FileLockClusterService serviceA = newService(new
FileLockClusterService(), "A");
+ contextA.addService(serviceA);
+ CamelClusterView viewA = serviceA.getView(NAMESPACE);
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
viewA.getLocalMember().isLeader()
+ &&
FileLockClusterUtils.readClusterLeaderInfo(root.resolve(NAMESPACE + ".dat")) !=
null);
+
+ // the leadership checks of A stall (as in a long GC pause), so its
heartbeat gets stale: this is the case where
+ // the lock must keep the members of other JVMs out
+ CountDownLatch stall = new CountDownLatch(1);
+ serviceA.getExecutor().execute(() -> {
+ try {
+ stall.await(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+
+ try {
+ CountingFileLockClusterService serviceB = newService(new
CountingFileLockClusterService(), "B");
+ CamelContext contextB = startContext();
+ contextB.addService(serviceB);
+ CamelClusterView viewB = serviceB.getView(NAMESPACE);
+
+ // B finds the leader stale and tries to take the leadership. Once
a later check has started, that attempt
+ // is over
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
serviceB.staleChecks.get() >= 2);
+
+ assertEquals(0, serviceB.lockFileOpens.get(),
+ "the follower opened the lock file that a view of its JVM
holds");
+ assertTrue(viewA.getLocalMember().isLeader());
+ assertFalse(viewB.getLocalMember().isLeader());
+ } finally {
+ stall.countDown();
+ }
+ }
+
+ @Test
+ void followerTakesOverWhenTheLeaderOfItsJvmStops() throws Exception {
+ CamelContext contextA = startContext();
+ FileLockClusterService serviceA = newService(new
FileLockClusterService(), "A");
+ contextA.addService(serviceA);
+ CamelClusterView viewA = serviceA.getView(NAMESPACE);
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
viewA.getLocalMember().isLeader());
+
+ CountingFileLockClusterService serviceB = newService(new
CountingFileLockClusterService(), "B");
+ CamelContext contextB = startContext();
+ contextB.addService(serviceB);
+ CamelClusterView viewB = serviceB.getView(NAMESPACE);
+ await().atMost(10, TimeUnit.SECONDS).until(() -> serviceB.checks.get()
>= 2);
+ assertFalse(viewB.getLocalMember().isLeader());
+
+ // the leader releases the lock, so the follower of the same JVM can
take it
+ contextA.stop();
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
viewB.getLocalMember().isLeader());
+ assertFalse(viewA.getLocalMember().isLeader());
+ }
+
+ @Test
+ void leadershipCheckGoesOnWhileResolvingTheLockPathHangs() throws
Exception {
+ // resolving the real path of the lock file directory hangs, as on a
hung NFS mount
+ CountDownLatch hang = new CountDownLatch(1);
+ AtomicInteger resolutions = new AtomicInteger();
+ FileLockClusterService service = newService(new
FileLockClusterService() {
+ @Override
+ protected FileLockClusterView createView(String namespace) {
+ return new FileLockClusterView(this, namespace) {
+ @Override
+ Path resolveLockPathKey() throws IOException {
+ resolutions.incrementAndGet();
+ try {
+ hang.await(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return super.resolveLockPathKey();
+ }
+ };
+ }
+ }, "A");
+ service.setClusterDataTaskTimeout(200, TimeUnit.MILLISECONDS);
+ service.setClusterDataTaskMaxAttempts(1);
+ CamelContext context = startContext();
+ context.addService(service);
+ CamelClusterView view = service.getView(NAMESPACE);
+
+ try {
+ // each check times out resolving the key and schedules the next
check, instead of blocking for good. Each
+ // hung resolution keeps one of the 5 threads of the cluster data
task pool, so wait for fewer than 5
+ await().atMost(10, TimeUnit.SECONDS).until(() -> resolutions.get()
>= 3);
+ assertFalse(view.getLocalMember().isLeader());
+ } finally {
+ hang.countDown();
+ }
+
+ // the checks that timed out registered nothing, so the view takes the
lock once the key resolves
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
view.getLocalMember().isLeader());
+ }
+
+ @Test
+ void lockFileIsNotOpenedWhileTheKeyCannotBeResolved() throws Exception {
+ // a key from the unresolved path could differ from the key of another
view of the JVM, so the view must not
+ // open the lock file without a resolved key
+ AtomicBoolean failing = new AtomicBoolean(true);
+ AtomicInteger resolutions = new AtomicInteger();
+ AtomicInteger lockFileOpens = new AtomicInteger();
+ FileLockClusterService service = newService(new
FileLockClusterService() {
+ @Override
+ protected FileLockClusterView createView(String namespace) {
+ return new FileLockClusterView(this, namespace) {
+ @Override
+ Path resolveLockPathKey() throws IOException {
+ resolutions.incrementAndGet();
+ if (failing.get()) {
+ throw new IOException("Simulated failure resolving
the lock file directory");
+ }
+ return super.resolveLockPathKey();
+ }
+
+ @Override
+ RandomAccessFile createRandomAccessFile(Path path) throws
ExecutionException, TimeoutException {
+ if (path.getFileName().toString().equals(NAMESPACE)) {
+ lockFileOpens.incrementAndGet();
+ }
+ return super.createRandomAccessFile(path);
+ }
+ };
+ }
+ }, "A");
+ service.setClusterDataTaskMaxAttempts(1);
+ CamelContext context = startContext();
+ context.addService(service);
+ CamelClusterView view = service.getView(NAMESPACE);
+
+ // a check that opened the lock file anyway would become leader and
stop resolving the key
+ await().atMost(10, TimeUnit.SECONDS).until(() -> resolutions.get() >=
2 || lockFileOpens.get() > 0);
+ assertEquals(0, lockFileOpens.get(), "the lock file was opened without
a resolved key");
+ assertFalse(view.getLocalMember().isLeader());
+
+ failing.set(false);
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
view.getLocalMember().isLeader());
+ }
+
+ private CamelContext startContext() {
+ CamelContext context = new DefaultCamelContext();
+ context.disableJMX();
+ contexts.add(context);
+ context.start();
+ return context;
+ }
+
+ private <T extends FileLockClusterService> T newService(T service, String
id) {
+ service.setId("node-" + id);
+ service.setRoot(root.toString());
+ service.setAcquireLockDelay(100, TimeUnit.MILLISECONDS);
+ service.setAcquireLockInterval(100, TimeUnit.MILLISECONDS);
+ service.setHeartbeatTimeoutMultiplier(2);
+ service.setClusterDataTaskTimeout(30, TimeUnit.SECONDS);
+ return service;
+ }
+
+ /**
+ * Counts the leadership checks of its view, the checks that found the
leader stale, and the times the view opened
+ * the lock file.
+ */
+ private static final class CountingFileLockClusterService extends
FileLockClusterService {
+ final AtomicInteger checks = new AtomicInteger();
+ final AtomicInteger staleChecks = new AtomicInteger();
+ final AtomicInteger lockFileOpens = new AtomicInteger();
+
+ @Override
+ protected FileLockClusterView createView(String namespace) {
+ return new FileLockClusterView(this, namespace) {
+ @Override
+ boolean isLeaderStale(FileLockClusterLeaderInfo info,
FileLockClusterLeaderInfo previous) {
+ checks.incrementAndGet();
+ boolean stale = super.isLeaderStale(info, previous);
+ if (stale) {
+ staleChecks.incrementAndGet();
+ }
+ return stale;
+ }
+
+ @Override
+ RandomAccessFile createRandomAccessFile(Path path) throws
ExecutionException, TimeoutException {
+ if (path.getFileName().toString().equals(NAMESPACE)) {
+ lockFileOpens.incrementAndGet();
+ }
+ return super.createRandomAccessFile(path);
+ }
+ };
+ }
+ }
+}