commit: b9a85ff987ea677e350fb70e2c3355197ee6c39b Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Sun Oct 22 08:08:02 2023 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Sun Oct 22 21:29:49 2023 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9a85ff9
MergeProcess: Support QueryCommand with spawn start method Since QueryCommand._db does not propagate to the child process in MergeProcess with the multiprocessing spawn start method, handle it by passing QueryCommand._db as an explicit parameter. Since self.mydbapi can be a bindbapi instance, override the bindbapi __getstate__ method so that it is picklable (omit the unpicklable _aux_cache_slot_dict attribute and regenerate it after unpickling). Bug: https://bugs.gentoo.org/916106 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/portage/dbapi/_MergeProcess.py | 19 ++++++++++++++++--- lib/portage/dbapi/bintree.py | 20 ++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index cf8e6513d1..dd5ad71cf8 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -9,6 +9,7 @@ import platform import fcntl import portage from portage import os, _unicode_decode +from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage.util._ctypes import find_library import portage.elog.messages from portage.util._async.ForkProcess import ForkProcess @@ -180,6 +181,15 @@ class MergeProcess(ForkProcess): self._dblink = mylink self._elog_reader_fd = elog_reader_fd + # Since the entire QueryCommand._db is not required, only pass + # in tree types that QueryCommand specifically requires. + child_db = {} + parent_db = portage.db if QueryCommand._db is None else QueryCommand._db + for root in parent_db: + child_db[root] = {} + for tree_type in ("vartree", "porttree"): + child_db[root][tree_type] = parent_db[root][tree_type] + self.target = functools.partial( self._target, self._counter, @@ -192,6 +202,7 @@ class MergeProcess(ForkProcess): self.settings, self.unmerge, self.vartree.dbapi, + child_db, ) pids = super()._spawn(args, fd_pipes, **kwargs) @@ -223,10 +234,12 @@ class MergeProcess(ForkProcess): settings, unmerge, vardb, + db, ): - """ - TODO: Make all arguments picklable for the multiprocessing spawn start method. - """ + if QueryCommand._db is None: + # Initialize QueryCommand._db for AbstractEbuildProcess/EbuildIpcDaemon + # when not using the multiprocessing fork start method. + QueryCommand._db = db portage.output.havecolor = not no_color(settings) # Avoid wastful updates of the vdb cache. vardb._flush_cache_enabled = False diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 421863f3e8..3f1bf9d5d2 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2021 Gentoo Authors +# Copyright 1998-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["bindbapi", "binarytree"] @@ -129,8 +129,24 @@ class bindbapi(fakedbapi): "USE", "_mtime_", } - self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} + self._aux_cache_slot_dict_cache = None + + @property + def _aux_cache_slot_dict(self): + if self._aux_cache_slot_dict_cache is None: + self._aux_cache_slot_dict_cache = slot_dict_class(self._aux_cache_keys) + return self._aux_cache_slot_dict_cache + + def __getstate__(self): + # This attribute is not picklable, but it automatically + # regenerates after unpickling when set to None. + _aux_cache_slot_dict = self._aux_cache_slot_dict_cache + self._aux_cache_slot_dict_cache = None + try: + return super().__getstate__() + finally: + self._aux_cache_slot_dict_cache = _aux_cache_slot_dict @property def writable(self):
