commit: f65afc5e394bd584f68fa29b149bb833a791ccd3 Author: Brian Harring <ferringb <AT> gmail <DOT> com> AuthorDate: Wed Dec 17 20:24:18 2025 +0000 Commit: Brian Harring <ferringb <AT> gmail <DOT> com> CommitDate: Wed Dec 17 20:31:09 2025 +0000 URL: https://gitweb.gentoo.org/proj/pkgcore/snakeoil.git/commit/?id=f65afc5e
pull all contexts.SplitExec usage and contexts.Namespace Fact is, this may work, but it's deeply intrinsic knowledge of the target python environment. I personally can follow the code- and maintain it- but why *should* snakeoil? This functionality should only be exposed if we're leveraging a 3rd party that wants to maintain compatibility for this sort of intrinsic "sidestep around the python implementation". This is being pulled due to those concerns, but also since it randomly fails out in the 0.11.0 release path. The only change between the last release and that one for this code is literally running the test from a tarball release rather than from git- IE, none that could be found, even in replicating in local envs. The likely issue here is the pypi env runs in some fashion slightly different to flex things and expose the issue, but it was reproducible in GH. So as said: have someone else maintain this since it's not simple, and remove it until it becomes necessary to have that. Removing this removes `contexts.Namespace` which runs code w/in a context block in a namespaced environment. That's blatantly cool and very ergonomic, but it's built on SplitExec, it's unused in the ecosystem, so it's being removed in parallel. This may be revisited; the ergonomic pattern *is* stupidly fluid, but the maintenance burden has to be less. Signed-off-by: Brian Harring <ferringb <AT> gmail.com> src/snakeoil/contexts.py | 279 --------------------------------------------- src/snakeoil/decorators.py | 27 ----- tests/test_contexts.py | 106 +---------------- tests/test_decorators.py | 55 +-------- tests/test_osutils.py | 62 +--------- 5 files changed, 7 insertions(+), 522 deletions(-) diff --git a/src/snakeoil/contexts.py b/src/snakeoil/contexts.py index 2d00dba..eab380a 100644 --- a/src/snakeoil/contexts.py +++ b/src/snakeoil/contexts.py @@ -1,298 +1,19 @@ """Various with-statement context utilities.""" import contextlib -import errno -import inspect import os -import pickle -import signal import subprocess import sys -import threading -import traceback from contextlib import AbstractContextManager, contextmanager from contextlib import chdir as _contextlib_chdir from importlib import import_module -from multiprocessing.connection import Pipe from snakeoil._internals import deprecated from snakeoil.python_namespaces import protect_imports from .cli.exceptions import UserException -from .process import namespaces from .sequences import predicate_split -# Ideas and code for SplitExec have been borrowed from withhacks -# (https://pypi.python.org/pypi/withhacks) governed by the MIT license found -# below. -# -# Copyright (c) 2010 Ryan Kelly -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - - -class SplitExec: - """Context manager separating code execution across parent/child processes. - - This is done by forking and doing some magic on the stack so the contents - of the context are executed only on the forked child. Exceptions are - pickled and passed back to the parent. - """ - - def __init__(self): - self.__trace_lock = threading.Lock() - self.__orig_sys_trace = None - self.__orig_trace_funcs = {} - self.__injected_trace_funcs = {} - self.__pipe = None - self.childpid = None - self.exit_status = -1 - self.locals = {} - - def _parent_handler(self, signum, frame): - """Signal handler for the parent process. - - By default this runs the parent cleanup and then resends the original - signal to the parent process. - """ - self._cleanup() - signal.signal(signum, signal.SIG_DFL) - os.kill(os.getpid(), signum) - - def _parent_setup(self): - """Initialization for parent process.""" - try: - signal.signal(signal.SIGINT, self._parent_handler) - signal.signal(signal.SIGTERM, self._parent_handler) - except ValueError: - # skip if we're not in the main thread - pass - - def _child_setup(self): - """Initialization for child process.""" - - def _cleanup(self): - """Parent process clean up on termination of the child.""" - - def _exception_cleanup(self): - """Parent process clean up after the child throws an exception.""" - self._cleanup() - - def _child_exit(self, exc): - # pass back changed local scope vars from the child that can be pickled - frame = self.__get_context_frame() - local_vars = {} - for k, v in frame.f_locals.items(): - if k not in self.__child_orig_locals or v != self.__child_orig_locals[k]: - try: - pickle.dumps(v) - local_vars[k] = v - except (AttributeError, TypeError, pickle.PicklingError): - continue - exc._locals = local_vars - - try: - self.__pipe.send(exc) - except BrokenPipeError as e: - if e.errno in (errno.EPIPE, errno.ESHUTDOWN): - pass - else: - raise - - if isinstance(exc, SystemExit) and exc.code is not None: - code = exc.code - else: - code = 0 - os._exit(code) # pylint: disable=W0212 - - def __enter__(self): - parent_pipe, child_pipe = Pipe() - - if pid := os.fork(): - self.childpid = pid - self._parent_setup() - self.__pipe = parent_pipe - frame = self.__get_context_frame() - self.__inject_trace_func(frame, self.__exit_context) - return self - else: - frame = self.__get_context_frame() - self.__child_orig_locals = dict(frame.f_locals) - self.__pipe = child_pipe - - try: - self._child_setup() - # pylint: disable=W0703 - # need to catch all exceptions here since we are passing them to - # the parent process - except Exception as exc: - exc.__traceback_list__ = traceback.format_exc() - self._child_exit(exc) - - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - if self.childpid is not None: - # make sure system tracing function is reset - self.__revert_tracing(inspect.currentframe()) - # re-raise unknown exceptions from the parent - if exc_type is not self.ParentException: - raise exc_value - - # get exception from the child - try: - exc = self.__pipe.recv() - self.locals = exc._locals - except EOFError as e: - exc = SystemExit(e) - - # handle child exiting abnormally - if not isinstance(exc, SystemExit): - os.waitpid(self.childpid, 0) - self._exception_cleanup() - sys.excepthook = self.__excepthook - raise exc - else: - if exc_value is not None: - exc = exc_value - # Unfortunately, traceback objects can't be pickled so the relevant - # traceback from the code executing within the chroot context is - # placed in the __traceback_list__ attribute and printed by a - # custom exception hook. - exc.__traceback_list__ = traceback.format_exc() - else: - exc = SystemExit() - - self._child_exit(exc) - - # wait for child process to exit - _pid, exit_status = os.waitpid(self.childpid, 0) - self.exit_status = exit_status >> 8 - self._cleanup() - - return True - - @staticmethod - def __excepthook(_exc_type, exc_value, exc_traceback): - """Output the proper traceback information from the chroot context.""" - if hasattr(exc_value, "__traceback_list__"): - sys.stderr.write(exc_value.__traceback_list__) - else: - traceback.print_tb(exc_traceback) - - @staticmethod - def __dummy_sys_trace(frame, event, arg): - """Dummy trace function used to enable tracing.""" - - class ParentException(Exception): - """Exception used to detect when the child terminates.""" - - def __enable_tracing(self): - """Enable system-wide tracing via a dummy method.""" - self.__orig_sys_trace = sys.gettrace() - sys.settrace(self.__dummy_sys_trace) - - def __revert_tracing(self, frame=None): - """Revert to previous system trace setting.""" - sys.settrace(self.__orig_sys_trace) - if frame is not None: - frame.f_trace = self.__orig_sys_trace - - def __exit_context(self, frame, event, arg): - """Simple function to throw a ParentException.""" - raise self.ParentException() - - def __inject_trace_func(self, frame, func): - """Inject a trace function for a frame. - - The given trace function will be executed immediately when the frame's - execution resumes. - """ - with self.__trace_lock: - if frame.f_trace is not self.__invoke_trace_funcs: - self.__orig_trace_funcs[frame] = frame.f_trace - frame.f_trace = self.__invoke_trace_funcs - self.__injected_trace_funcs[frame] = [] - if len(self.__orig_trace_funcs) == 1: - self.__enable_tracing() - self.__injected_trace_funcs[frame].append(func) - - def __invoke_trace_funcs(self, frame, event, arg): - """Invoke all trace funcs that have been injected. - - Once the injected functions have been executed all trace hooks are - removed in order to minimize overhead. - """ - try: - for func in self.__injected_trace_funcs[frame]: - func(frame, event, arg) - finally: - del self.__injected_trace_funcs[frame] - with self.__trace_lock: - if len(self.__orig_trace_funcs) == 1: - self.__revert_tracing() - frame.f_trace = self.__orig_trace_funcs.pop(frame) - - def __get_context_frame(self): - """Get the frame object for the with-statement context. - - This is designed to work from within superclass method call. It finds - the first frame where the local variable "self" doesn't exist. - """ - try: - return self.__frame - except AttributeError: - # an offset of two accounts for this method and its caller - frame = inspect.stack(0)[2][0] - while frame.f_locals.get("self") is self: - frame = frame.f_back - self.__frame = frame # pylint: disable=W0201 - return frame - - -class Namespace(SplitExec): - """Context manager that provides Linux namespace support.""" - - def __init__( - self, - mount=False, - uts=True, - ipc=False, - net=False, - pid=False, - user=False, - hostname=None, - ): - self._hostname = hostname - self._namespaces = { - "mount": mount, - "uts": uts, - "ipc": ipc, - "net": net, - "pid": pid, - "user": user, - } - super().__init__() - - def _child_setup(self): - namespaces.simple_unshare(hostname=self._hostname, **self._namespaces) - class GitStash(AbstractContextManager): """Context manager for stashing untracked or modified/uncommitted files.""" diff --git a/src/snakeoil/decorators.py b/src/snakeoil/decorators.py index 1d23abe..97f4416 100644 --- a/src/snakeoil/decorators.py +++ b/src/snakeoil/decorators.py @@ -2,33 +2,6 @@ from functools import wraps -from . import contexts - - -def splitexec(func): - """Run the decorated function in another process.""" - - @wraps(func) - def wrapper(*args, **kwargs): - with contexts.SplitExec(): - return func(*args, **kwargs) - - return wrapper - - -def namespace(**namespaces): - """Run the decorated function in a specified namespace.""" - - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - with contexts.Namespace(**namespaces): - return func(*args, **kwargs) - - return wrapper - - return decorator - def coroutine(func): """Prime a coroutine for input.""" diff --git a/tests/test_contexts.py b/tests/test_contexts.py index a25124e..89d48ac 100644 --- a/tests/test_contexts.py +++ b/tests/test_contexts.py @@ -1,15 +1,9 @@ -import errno import os -import platform -import random -import socket import sys from contextlib import chdir -import pytest - from snakeoil._internals import deprecated -from snakeoil.contexts import Namespace, SplitExec, syspath +from snakeoil.contexts import syspath @deprecated.suppress_deprecations() @@ -47,101 +41,3 @@ def test_syspath(tmpdir): # dir isn't added again due to condition with syspath(tmpdir, condition=(tmpdir not in sys.path)): assert mangled_syspath == tuple(sys.path) - - [email protected]( - reason="this currently is broken: https://github.com/pkgcore/snakeoil/issues/68 is the GH side first incidence of it" -) -class TestSplitExec: - def test_context_process(self): - # code inside the with statement is run in a separate process - pid = os.getpid() - with SplitExec() as c: - pass - assert c.childpid is not None - assert pid != c.childpid - - def test_context_exit_status(self): - # exit status of the child process is available as a context attr - with SplitExec() as c: - pass - assert c.exit_status == 0 - - exit_status = random.randint(1, 255) - with SplitExec() as c: - sys.exit(exit_status) - assert c.exit_status == exit_status - - def test_context_locals(self): - # code inside the with statement returns modified, pickleable locals - # via 'locals' attr of the context manager - a = 1 - with SplitExec() as c: - assert a == 1 - a = 2 - assert a == 2 - b = 3 - # changes to locals aren't propagated back - assert a == 1 - assert "b" not in locals() - # but they're accessible via the 'locals' attr - expected = {"a": 2, "b": 3} - for k, v in expected.items(): - assert c.locals[k] == v - - # make sure unpickleables don't cause issues - with SplitExec() as c: - func = lambda x: x - from sys import implementation - - a = 4 - assert c.locals == {"a": 4} - - def test_context_exceptions(self): - # exceptions in the child process are sent back to the parent and re-raised - with pytest.raises(IOError) as e: - with SplitExec() as c: - raise IOError(errno.EBUSY, "random error") - assert e.value.errno == errno.EBUSY - - def test_child_setup_raises_exception(self): - class ChildSetupException(SplitExec): - def _child_setup(self): - raise IOError(errno.EBUSY, "random error") - - with pytest.raises(IOError) as e: - with ChildSetupException() as c: - pass - assert e.value.errno == errno.EBUSY - - [email protected]( - not sys.platform.startswith("linux"), reason="supported on Linux only" -) [email protected](platform.python_implementation() == "PyPy", reason="Fails on PyPy") -class TestNamespace: - @pytest.mark.skipif( - not os.path.exists("/proc/self/ns/user"), - reason="user namespace support required", - ) - def test_user_namespace(self): - try: - with Namespace(user=True) as ns: - assert os.getuid() == 0 - except PermissionError: - pytest.skip("No permission to use user namespace") - - @pytest.mark.skipif( - not ( - os.path.exists("/proc/self/ns/user") and os.path.exists("/proc/self/ns/uts") - ), - reason="user and uts namespace support required", - ) - def test_uts_namespace(self): - try: - with Namespace(user=True, uts=True, hostname="host") as ns: - ns_hostname, _, ns_domainname = socket.getfqdn().partition(".") - assert ns_hostname == "host" - assert ns_domainname == "" - except PermissionError: - pytest.skip("No permission to use user and uts namespace") diff --git a/tests/test_decorators.py b/tests/test_decorators.py index ef63e27..da3854c 100644 --- a/tests/test_decorators.py +++ b/tests/test_decorators.py @@ -1,59 +1,6 @@ -import contextlib -import os -import socket -import sys -from unittest import mock - import pytest -from snakeoil.decorators import coroutine, namespace, splitexec - - [email protected]("flaky test, needs rework") -class TestSplitExecDecorator: - def setup_method(self, method): - self.pid = os.getpid() - - @splitexec - def test_separate_func_process(self): - # code inside the decorated func is run in a different process - assert self.pid != os.getpid() - - [email protected]( - not sys.platform.startswith("linux"), reason="supported on Linux only" -) -class TestNamespaceDecorator: - @contextlib.contextmanager - def capture_call(self): - @contextlib.contextmanager - def fake_namespace(): - yield - - with mock.patch("snakeoil.contexts.Namespace") as m: - m.side_effect = lambda *a, **kw: fake_namespace() - yield object(), m - - def test_user_namespace(self): - with self.capture_call() as (unique, m): - - @namespace(user=True) - def do_test(): - return unique - - # do_test() - assert unique is do_test() - m.assert_called_once_with(user=True) - - def test_uts_namespace(self): - with self.capture_call() as (unique, m): - - @namespace(user=True, uts=True, hostname="host") - def do_test(): - return unique - - assert unique is do_test() - m.assert_called_once_with(user=True, uts=True, hostname="host") +from snakeoil.decorators import coroutine class TestCoroutineDecorator: diff --git a/tests/test_osutils.py b/tests/test_osutils.py index cc3ba9d..964dbd2 100644 --- a/tests/test_osutils.py +++ b/tests/test_osutils.py @@ -11,9 +11,8 @@ import pytest from snakeoil import osutils from snakeoil._internals import deprecated -from snakeoil.contexts import Namespace from snakeoil.osutils import listdir_dirs, listdir_files, sizeof_fmt, supported_systems -from snakeoil.osutils.mount import MNT_DETACH, MS_BIND, mount, umount +from snakeoil.osutils.mount import MS_BIND, mount, umount class ReaddirCommon: @@ -327,62 +326,11 @@ class TestMount: umount(str(target)) assert cm.value.errno in (errno.EPERM, errno.EINVAL) - @pytest.mark.skipif( - not ( - os.path.exists("/proc/self/ns/mnt") and os.path.exists("/proc/self/ns/user") - ), - reason="user and mount namespace support required", - ) - def test_bind_mount(self, source, target): - src_file = source / "file" - bind_file = target / "file" - src_file.touch() + @pytest.mark.skip("test must be rewritten due to reliance on SplitExec") + def test_bind_mount(self): ... - try: - with Namespace(user=True, mount=True): - assert not bind_file.exists() - mount(str(source), str(target), None, MS_BIND) - assert bind_file.exists() - umount(str(target)) - assert not bind_file.exists() - except PermissionError: - pytest.skip("No permission to use user and mount namespace") - - @pytest.mark.skipif( - not ( - os.path.exists("/proc/self/ns/mnt") and os.path.exists("/proc/self/ns/user") - ), - reason="user and mount namespace support required", - ) - def test_lazy_unmount(self, source, target): - src_file = source / "file" - bind_file = target / "file" - src_file.touch() - src_file.write_text("foo") - - try: - with Namespace(user=True, mount=True): - mount(str(source), str(target), None, MS_BIND) - assert bind_file.exists() - - with bind_file.open() as f: - # can't unmount the target due to the open file - with pytest.raises(OSError) as cm: - umount(str(target)) - assert cm.value.errno == errno.EBUSY - # lazily unmount instead - umount(str(target), MNT_DETACH) - # confirm the file doesn't exist in the bind mount anymore - assert not bind_file.exists() - # but the file is still accessible to the process - assert f.read() == "foo" - - # trying to reopen causes IOError - with pytest.raises(IOError) as cm: - f = bind_file.open() - assert cm.value.errno == errno.ENOENT - except PermissionError: - pytest.skip("No permission to use user and mount namespace") + @pytest.mark.skip("test must be rewritten due to reliance on SplitExec") + def test_lazy_unmount(self): ... class TestSizeofFmt:
