commit: 8835140f9da1f43fa0e7360aaf403bc635ab7398 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Mon Oct 16 03:58:40 2023 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Mon Oct 16 04:41:04 2023 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8835140f
slot_dict_class: Make instances picklable This improves compatibility with the multiprocessing spawn start method, by eliminating this error for MergeProcess: AttributeError: Can't pickle local object 'slot_dict_class.<locals>.LocalSlotDict' Bug: https://bugs.gentoo.org/915834 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/portage/cache/mappings.py | 23 +++++++++++++++++ lib/portage/tests/process/meson.build | 1 + lib/portage/tests/process/test_pickle.py | 43 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/lib/portage/cache/mappings.py b/lib/portage/cache/mappings.py index 1aacad99a4..e7ee7df5cd 100644 --- a/lib/portage/cache/mappings.py +++ b/lib/portage/cache/mappings.py @@ -292,6 +292,16 @@ class _SlotDict: if kwargs: self.update(kwargs) + def __reduce__(self): + return _PickledSlotDict, ( + self._prefix, + self.allowed_keys, + dict(self), + ) + + def __eq__(self, other): + return dict(self) == dict(other) + def __iter__(self): for k, v in self.iteritems(): yield k @@ -417,6 +427,19 @@ class _SlotDict: values = itervalues +class _PickledSlotDict(_SlotDict): + """ + Since LocalSlotDict instances are not directly picklable, this + class exists as a way to express pickled LocalSlotDict instances, + using a plain __dict__ instead of custom __slots__. + """ + + def __init__(self, prefix, allowed_keys, *args, **kwargs): + self._prefix = prefix + self.allowed_keys = allowed_keys + super().__init__(*args, **kwargs) + + _slot_dict_classes = weakref.WeakValueDictionary() diff --git a/lib/portage/tests/process/meson.build b/lib/portage/tests/process/meson.build index 1b34e57e33..1cef1cc4a2 100644 --- a/lib/portage/tests/process/meson.build +++ b/lib/portage/tests/process/meson.build @@ -4,6 +4,7 @@ py.install_sources( 'test_PipeLogger.py', 'test_PopenProcessBlockingIO.py', 'test_PopenProcess.py', + 'test_pickle.py', 'test_poll.py', 'test_spawn_fail_e2big.py', 'test_spawn_warn_large_env.py', diff --git a/lib/portage/tests/process/test_pickle.py b/lib/portage/tests/process/test_pickle.py new file mode 100644 index 0000000000..9b7d9ef423 --- /dev/null +++ b/lib/portage/tests/process/test_pickle.py @@ -0,0 +1,43 @@ +# Copyright 2023 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import pickle + +from portage.tests import TestCase +from _emerge.Package import _PackageMetadataWrapperBase +from _emerge.FifoIpcDaemon import FifoIpcDaemon + + +class PickleTestCase(TestCase): + def test_PackageMetadataWrapperBase(self): + """ + Verify that instances of slot_dict_class, like + PackageMetadataWrapperBase, are picklable for + compatibility with the multiprocessing spawn + start method. + """ + obj = _PackageMetadataWrapperBase(EAPI="8") + self.assertEqual(obj["EAPI"], "8") + serialized = pickle.dumps(obj) + obj_copy = pickle.loads(serialized) + self.assertEqual(len(obj_copy), len(obj)) + self.assertEqual(obj_copy["EAPI"], obj["EAPI"]) + self.assertEqual(obj_copy, obj) + + def test_FifoIpcDaemon_files_dict(self): + """ + Verify that FifoIpcDaemon._files_dict instances are picklable for + compatibility with the multiprocessing spawn start method. + """ + obj = FifoIpcDaemon._files_dict( + (k, "test-value") for k in FifoIpcDaemon._file_names + ) + self.assertEqual(obj["pipe_in"], "test-value") + # Attributes of same name exist because of slot_dict_class prefix="" argument. + self.assertEqual(obj.pipe_in, obj["pipe_in"]) + serialized = pickle.dumps(obj) + obj_copy = pickle.loads(serialized) + self.assertEqual(len(obj_copy), len(obj)) + self.assertEqual(obj_copy["pipe_in"], obj["pipe_in"]) + self.assertEqual(obj_copy.pipe_in, obj["pipe_in"]) + self.assertEqual(obj_copy, obj)
