commit: 7622da95fd97c07d7a51f14dfcfb07048db687c1 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Tue Oct 3 14:59:17 2023 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Tue Oct 3 14:59:17 2023 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=7622da95
_get_lock_fn: support multiprocessing spawn start method (bug 915119) Ensure that _get_lock_fn arguments to multiprocessing.Process will successfully pickle, as required by the spawn start method, which is the default for macOS since Python 3.8. Since file descriptors are not inherited unless the fork start method is used, the subprocess should only try to close an inherited file descriptor for the fork start method. Bug: https://bugs.gentoo.org/915119 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/portage/locks.py | 50 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/lib/portage/locks.py b/lib/portage/locks.py index 1c3e13ce4f..ee40451b12 100644 --- a/lib/portage/locks.py +++ b/lib/portage/locks.py @@ -79,6 +79,11 @@ class _lock_manager: del _open_inodes[self.inode_key] +def _lockf_test_lock_fn(path, fd, flags): + fcntl.lockf(fd, flags) + return functools.partial(unlockfile, (path, fd, flags, fcntl.lockf)) + + def _get_lock_fn(): """ Returns fcntl.lockf if proven to work, and otherwise returns fcntl.flock. @@ -88,10 +93,7 @@ def _get_lock_fn(): if _lock_fn is not None: return _lock_fn - if _test_lock_fn( - lambda path, fd, flags: fcntl.lockf(fd, flags) - and functools.partial(unlockfile, (path, fd, flags, fcntl.lockf)) - ): + if _test_lock_fn(_lockf_test_lock_fn): _lock_fn = fcntl.lockf return _lock_fn @@ -103,19 +105,6 @@ def _get_lock_fn(): def _test_lock_fn( lock_fn: typing.Callable[[str, int, int], typing.Callable[[], None]] ) -> bool: - def _test_lock(fd, lock_path): - os.close(fd) - try: - with open(lock_path, "a") as f: - lock_fn(lock_path, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except (TryAgain, OSError) as e: - if isinstance(e, TryAgain) or e.errno == errno.EAGAIN: - # Parent process holds lock, as expected. - sys.exit(0) - - # Something went wrong. - sys.exit(1) - fd, lock_path = tempfile.mkstemp() unlock_fn = None try: @@ -125,7 +114,17 @@ def _test_lock_fn( pass else: _lock_manager(fd, os.fstat(fd), lock_path) - proc = multiprocessing.Process(target=_test_lock, args=(fd, lock_path)) + proc = multiprocessing.Process( + target=_subprocess_test_lock, + args=( + # Since file descriptors are not inherited unless the fork start + # method is used, the subprocess should only try to close an + # inherited file descriptor for the fork start method. + fd if multiprocessing.get_start_method() == "fork" else None, + lock_fn, + lock_path, + ), + ) proc.start() proc.join() if proc.exitcode == os.EX_OK: @@ -141,6 +140,21 @@ def _test_lock_fn( return False +def _subprocess_test_lock(fd, lock_fn, lock_path): + if fd is not None: + os.close(fd) + try: + with open(lock_path, "a") as f: + lock_fn(lock_path, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (TryAgain, OSError) as e: + if isinstance(e, TryAgain) or e.errno == errno.EAGAIN: + # Parent process holds lock, as expected. + sys.exit(0) + + # Something went wrong. + sys.exit(1) + + def _close_fds(): """ This is intended to be called after a fork, in order to close file
