commit: 4978b913df39abbb3369739e370856cf8292a203 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Thu Oct 12 18:05:29 2023 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Thu Oct 12 18:37:26 2023 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=4978b913
Detect and handle unnecessary package reinstall Compare package rebuilds/reinstalls to installed packages of the same exact version, and eliminate unecessary rebuilds/reinstalls triggered solely by the @__auto_slot_operator_replace_installed__ set. This is careful to obey the user's wishes if they have explicitly requested for a package to be rebuilt or reinstalled for some reason. The SonameSkipUpdateTestCase::testSonameSkipUpdateNoPruneRebuilds test case shows that the new depgraph._eliminate_rebuilds method eliminates a backtracking run that is needed for the testSonameSkipUpdate test case to succeed via prune_rebuilds backtracking which was added for bug 439688. Bug: https://bugs.gentoo.org/915494 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/_emerge/depgraph.py | 206 ++++++++++++++++++--- lib/portage/dep/_slot_operator.py | 8 +- .../tests/resolver/soname/test_skip_update.py | 19 +- 3 files changed, 199 insertions(+), 34 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index ad835ac06a..4e4452dad1 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -53,6 +53,7 @@ from portage.package.ebuild.config import _get_feature_flags from portage.package.ebuild.getmaskingstatus import _getmaskingstatus, _MaskReason from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet +from portage.dep._slot_operator import evaluate_slot_operator_equal_deps from portage.util import ConfigProtect, shlex_split, new_protect_filename from portage.util import cmp_sort_key, writemsg, writemsg_stdout from portage.util import ensure_dirs, normalize_path @@ -448,6 +449,8 @@ class _dynamic_depgraph_config: """ + _ENABLE_PRUNE_REBUILDS = True + def __init__(self, depgraph, myparams, allow_backtracking, backtrack_parameters): self.myparams = myparams.copy() self._vdb_loaded = False @@ -459,7 +462,7 @@ class _dynamic_depgraph_config: self._filtered_trees = {} # Contains installed packages and new packages that have been added # to the graph. - self._graph_trees = {} + self._graph_trees = portage._trees_dict() # Caches visible packages returned from _select_package, for use in # depgraph._iter_atoms_for_pkg() SLOT logic. self._visible_pkgs = {} @@ -570,6 +573,12 @@ class _dynamic_depgraph_config: graph_tree.dbapi = fakedb self._graph_trees[myroot] = {} + self._graph_trees._running_eroot = ( + depgraph._frozen_config._trees_orig._running_eroot + ) + self._graph_trees._target_eroot = ( + depgraph._frozen_config._trees_orig._target_eroot + ) self._filtered_trees[myroot] = {} # Substitute the graph tree for the vartree in dep_check() since we # want atom selections to be consistent with package selections @@ -3588,7 +3597,148 @@ class depgraph: return False return True - def _remove_pkg(self, pkg): + def _eliminate_rebuilds(self): + """ + Compare package rebuilds/reinstalls to installed packages of the same + exact version, and eliminate unnecessary rebuilds/reinstalls triggered + solely by the @__auto_slot_operator_replace_installed__ set. This is + careful to obey the user's wishes if they have explicitly requested + for a package to be rebuilt or reinstalled for some reason. + """ + modified = False + selective = "selective" in self._dynamic_config.myparams + for root, atom in self._dynamic_config._slot_operator_replace_installed: + for pkg in self._dynamic_config._package_tracker.match( + root, atom, installed=False + ): + installed_instance = self._frozen_config.trees[root][ + "vartree" + ].dbapi.match_pkgs(pkg.slot_atom) + if not installed_instance: + continue + installed_instance = installed_instance[0] + if installed_instance.cpv != pkg.cpv: + continue + if pkg in self._dynamic_config._reinstall_nodes: + # --newuse, --changed-use + continue + + if self._dynamic_config.myparams.get("changed_slot") and ( + self._changed_slot(pkg) or self._changed_slot(installed_instance) + ): + continue + + unsatisfied_parent = any( + not atom.match(installed_instance) + for parent, atom in self._dynamic_config._parent_atoms[pkg] + ) + if unsatisfied_parent: + continue + + have_arg = False + if not selective: + for parent, atom in self._dynamic_config._parent_atoms[pkg]: + if isinstance(parent, AtomArg): + have_arg = True + break + elif ( + isinstance(parent, SetArg) + and parent.name + != "__auto_slot_operator_replace_installed__" + ): + have_arg = True + break + if have_arg: + continue + if (installed_instance.slot, installed_instance.sub_slot) != ( + pkg.slot, + pkg.sub_slot, + ): + continue + if pkg.built: + if pkg.provides != installed_instance.provides: + continue + if pkg.requires != installed_instance.requires: + continue + + depvars = Package._dep_keys + try: + installed_deps = [] + for k in depvars: + dep_struct = portage.dep.use_reduce( + installed_instance._raw_metadata[k], + uselist=pkg.use.enabled, + eapi=pkg.eapi, + token_class=Atom, + ) + installed_deps.append(dep_struct) + except InvalidDependString: + continue + else: + new_deps = [] + new_use = self._pkg_use_enabled(pkg) + if pkg.built: + pkg_metadata = pkg._raw_metadata + else: + pkgsettings = self._frozen_config.pkgsettings[root] + pkgsettings.setcpv(pkg) + evaluate_slot_operator_equal_deps( + pkgsettings, new_use, self._dynamic_config._graph_trees + ) + pkg_metadata = pkgsettings.configdict["pkg"] + + for k in depvars: + dep_struct = portage.dep.use_reduce( + pkg_metadata[k], + uselist=new_use, + eapi=pkg.eapi, + token_class=Atom, + ) + new_deps.append(dep_struct) + + if new_deps != installed_deps: + continue + + modified = True + parent_atoms = [] + for parent, parent_atom in self._dynamic_config._parent_atoms[pkg]: + priorities = self._dynamic_config.digraph.nodes[pkg][1][parent][:] + parent_atoms.append((parent, parent_atom, priorities)) + child_parents = {} + for child in self._dynamic_config.digraph.child_nodes(pkg): + priorities = self._dynamic_config.digraph.nodes[child][1][pkg][:] + child_parents[child] = ( + [ + atom + for parent, atom in self._dynamic_config._parent_atoms[ + child + ] + if parent is pkg + ], + priorities, + ) + self._remove_pkg(pkg, remove_orphans=False) + for parent, atom, priorities in parent_atoms: + self._add_parent_atom(installed_instance, (parent, atom)) + for priority in priorities: + self._dynamic_config.digraph.add( + installed_instance, + parent, + priority=priority, + ) + for child, (atoms, priorities) in child_parents.items(): + for child_atom in atoms: + self._add_parent_atom(child, (installed_instance, child_atom)) + for priority in priorities: + self._dynamic_config.digraph.add( + child, + installed_instance, + priority=priority, + ) + + return modified + + def _remove_pkg(self, pkg, remove_orphans=True): """ Remove a package and all its then parentless digraph children from all depgraph datastructures. @@ -3643,12 +3793,13 @@ class depgraph: self._dynamic_config._blocked_pkgs.discard(pkg) self._dynamic_config._blocked_world_pkgs.pop(pkg, None) - for child in children: - if ( - child in self._dynamic_config.digraph - and not self._dynamic_config.digraph.parent_nodes(child) - ): - self._remove_pkg(child) + if remove_orphans: + for child in children: + if ( + child in self._dynamic_config.digraph + and not self._dynamic_config.digraph.parent_nodes(child) + ): + self._remove_pkg(child) # Clear caches. self._dynamic_config._filtered_trees[pkg.root]["porttree"].dbapi._clear_cache() @@ -5173,25 +5324,6 @@ class depgraph: ): return False, myfavorites - if ( - not self._dynamic_config._prune_rebuilds - and self._dynamic_config._slot_operator_replace_installed - and self._get_missed_updates() - ): - # When there are missed updates, we might have triggered - # some unnecessary rebuilds (see bug #439688). So, prune - # all the rebuilds and backtrack with the problematic - # updates masked. The next backtrack run should pull in - # any rebuilds that are really needed, and this - # prune_rebuilds path should never be entered more than - # once in a series of backtracking nodes (in order to - # avoid a backtracking loop). - backtrack_infos = self._dynamic_config._backtrack_infos - config = backtrack_infos.setdefault("config", {}) - config["prune_rebuilds"] = True - self._dynamic_config._need_restart = True - return False, myfavorites - if self.need_restart(): # want_restart_for_use_change triggers this return False, myfavorites @@ -5243,15 +5375,29 @@ class depgraph: self._dynamic_config._skip_restart = True return False, myfavorites - if ( - not self._dynamic_config._prune_rebuilds - and self._ignored_binaries_autounmask_backtrack() + if not self._dynamic_config._prune_rebuilds and ( + self._ignored_binaries_autounmask_backtrack() + or ( + self._dynamic_config._ENABLE_PRUNE_REBUILDS + and self._dynamic_config._slot_operator_replace_installed + and self._get_missed_updates() + ) ): config = self._dynamic_config._backtrack_infos.setdefault("config", {}) config["prune_rebuilds"] = True self._dynamic_config._need_restart = True return False, myfavorites + if ( + self._dynamic_config._slot_operator_replace_installed + and self._eliminate_rebuilds() + ): + self._dynamic_config._serialized_tasks_cache = None + try: + self.altlist() + except self._unknown_internal_error: + return False, myfavorites + # Any failures except those due to autounmask *alone* should return # before this point, since the success_without_autounmask flag that's # set below is reserved for cases where there are *zero* other diff --git a/lib/portage/dep/_slot_operator.py b/lib/portage/dep/_slot_operator.py index 8ce72956f9..82dd7d66c9 100644 --- a/lib/portage/dep/_slot_operator.py +++ b/lib/portage/dep/_slot_operator.py @@ -1,4 +1,4 @@ -# Copyright 2012-2018 Gentoo Foundation +# Copyright 2012-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.dep import Atom, paren_enclose, use_reduce @@ -101,7 +101,11 @@ def _eval_deps(dep_struct, vardbs): if best_version: best_version = best_version[-1] try: - best_version = vardb._pkg_str(best_version, None) + best_version = ( + best_version + if hasattr(best_version, "slot") + else vardb._pkg_str(best_version, None) + ) except (KeyError, InvalidData): pass else: diff --git a/lib/portage/tests/resolver/soname/test_skip_update.py b/lib/portage/tests/resolver/soname/test_skip_update.py index 510020fda9..407c16a548 100644 --- a/lib/portage/tests/resolver/soname/test_skip_update.py +++ b/lib/portage/tests/resolver/soname/test_skip_update.py @@ -1,7 +1,8 @@ -# Copyright 2015 Gentoo Foundation +# Copyright 2015-2023 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys +from unittest.mock import patch from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS from portage.tests import TestCase @@ -13,7 +14,19 @@ from portage.output import colorize class SonameSkipUpdateTestCase(TestCase): - def testSonameSkipUpdate(self): + def testSonameSkipUpdateNoPruneRebuilds(self): + """ + Make sure that there are fewer backtracking runs required if we + disable prune_rebuilds backtracking, which shows that + _eliminate_rebuilds works for the purposes of bug 915494. + """ + with patch( + "_emerge.depgraph._dynamic_depgraph_config._ENABLE_PRUNE_REBUILDS", + new=False, + ): + self.testSonameSkipUpdate(backtrack=2) + + def testSonameSkipUpdate(self, backtrack=3): binpkgs = { "app-misc/A-1": { "RDEPEND": "dev-libs/B", @@ -52,6 +65,7 @@ class SonameSkipUpdateTestCase(TestCase): "--ignore-soname-deps": "y", "--update": True, "--usepkgonly": True, + "--backtrack": backtrack, }, success=True, mergelist=[ @@ -68,6 +82,7 @@ class SonameSkipUpdateTestCase(TestCase): "--ignore-soname-deps": "n", "--update": True, "--usepkgonly": True, + "--backtrack": backtrack, }, success=True, mergelist=[],
