commit: 3f12ddaf34db88e2708d164ef871fe4de3d81434 Author: Jethro Donaldson <devel <AT> jro <DOT> nz> AuthorDate: Thu Dec 4 11:13:52 2025 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Mon Feb 16 06:54:41 2026 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=3f12ddaf
config: add getbinpkg-exclude and getbinpkg-include to binrepos.conf Allow configuration in binrepos.conf of specific packages which are to be satisfied (or not) using remote binary packages. These attributes appear under the repository section and are specific to that binary package host, with the behaviour otherwise identical to the command line arguments of the same name. Where atoms specified with getbinpkg-exclude or getbinpkg-include in binrepos.conf match those for the opposing command line options then emit a warning and override the binrepos.conf atoms. Signed-off-by: Jethro Donaldson <devel <AT> jro.nz> Suggested-by: Sam James <sam <AT> gentoo.org> Part-of: https://github.com/gentoo/portage/pull/1527 Closes: https://github.com/gentoo/portage/pull/1527 Signed-off-by: Sam James <sam <AT> gentoo.org> lib/portage/binrepo/config.py | 38 ++ lib/portage/dbapi/bintree.py | 38 +- .../tests/resolver/test_binpackage_selection.py | 491 +++++++++++++++++++++ man/portage.5 | 14 + 4 files changed, 579 insertions(+), 2 deletions(-) diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py index d167a9b5ca..604246d15e 100644 --- a/lib/portage/binrepo/config.py +++ b/lib/portage/binrepo/config.py @@ -5,7 +5,9 @@ from collections import OrderedDict from collections.abc import Mapping from hashlib import md5 +from portage._sets.base import WildcardPackageSet from portage.localization import _ +from portage.repository.config import _find_bad_atoms from portage.util import _recursive_file_list, writemsg from portage.util.configparser import SafeConfigParser, ConfigParserError, read_configs @@ -17,6 +19,8 @@ class BinRepoConfig: "name", "name_fallback", "fetchcommand", + "getbinpkg_exclude", + "getbinpkg_include", "location", "priority", "resumecommand", @@ -35,6 +39,40 @@ class BinRepoConfig: if isinstance(getattr(self, k, None), str): setattr(self, k, getattr(self, k).lower() in ("true", "yes")) + # getbinpkg-exclude and getbinpkg-include validation + for opt in ("getbinpkg-exclude", "getbinpkg-include"): + attr = opt.replace("-", "_") + if self.name == "DEFAULT": + setattr(self, attr, None) + continue + getbinpkg_atoms = opts.get(opt, "").split() + bad_atoms = _find_bad_atoms(getbinpkg_atoms) + if bad_atoms: + writemsg( + "\n!!! The following atoms are invalid in %s attribute for " + "binrepo [%s] (only package names and slot atoms allowed):\n" + "\n %s\n" % (opt, self.name, "\n ".join(bad_atoms)) + ) + for a in bad_atoms: + getbinpkg_atoms.remove(a) + getbinpkg_set = WildcardPackageSet(getbinpkg_atoms, allow_repo=True) + setattr(self, attr, getbinpkg_set) + conflicted_atoms = ( + self.getbinpkg_exclude + and self.getbinpkg_exclude.getAtoms().intersection( + self.getbinpkg_include.getAtoms() + ) + ) + if conflicted_atoms: + writemsg( + "\n!!! The following atoms appear in both the getbinpkg-exclude " + "getbinpkg-include lists for binrepo [%s]:\n" + "\n %s\n" % (self.name, "\n ".join(conflicted_atoms)) + ) + for a in conflicted_atoms: + self.getbinpkg_exclude.remove(a) + self.getbinpkg_include.remove(a) + def info_string(self): """ Returns a formatted string containing information about the repository. diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index b6056896e7..d0a70bcbd1 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1434,14 +1434,48 @@ class binarytree: # Order by descending priority. for repo in reversed(list(self._binrepos_conf.values())): + excluded = repo.getbinpkg_exclude or [] + getbinpkg_exclude_repo = WildcardPackageSet(excluded) + included = repo.getbinpkg_include or [] + getbinpkg_include_repo = WildcardPackageSet(included) + + getbinpkg_exclude_repo.update(getbinpkg_exclude) + getbinpkg_include_repo.update(getbinpkg_include) + + # --getbinpkg-include overrides getbinpkg-exclude in binrepos.conf + conflicted_exclude = getbinpkg_exclude_repo.getAtoms().intersection( + getbinpkg_include.getAtoms() + ) + if conflicted_exclude: + writemsg( + "\n!!! The following getbinpkg-exclude atoms for [%s] have " + "been overridden by the --getbinpkg-include option:\n" + "\n %s\n" % (repo.name, "\n ".join(conflicted_exclude)) + ) + for a in conflicted_exclude: + getbinpkg_exclude_repo.remove(a) + + # --getbinpkg-exclude overrides getbinpkg-include in binrepos.conf + conflicted_include = getbinpkg_include_repo.getAtoms().intersection( + getbinpkg_exclude.getAtoms() + ) + if conflicted_include: + writemsg( + "\n!!! The following getbinpkg-include atoms for [%s] have " + "been overridden by the --getbinpkg-exclude option:\n" + "\n %s\n" % (repo.name, "\n ".join(conflicted_include)) + ) + for a in conflicted_include: + getbinpkg_include_repo.remove(a) + self._populate_remote_repo( repo, getbinpkg_refresh, pretend, verbose, gpkg_only, - getbinpkg_exclude, - getbinpkg_include, + getbinpkg_exclude_repo, + getbinpkg_include_repo, ) def _populate_remote_repo( diff --git a/lib/portage/tests/resolver/test_binpackage_selection.py b/lib/portage/tests/resolver/test_binpackage_selection.py index ce1f57d9c7..1205dfe599 100644 --- a/lib/portage/tests/resolver/test_binpackage_selection.py +++ b/lib/portage/tests/resolver/test_binpackage_selection.py @@ -353,6 +353,266 @@ class GetBinPkgExcludeTestCase(BinPkgSelectionTestCase): test_cases, binpkgs=binpkgs, binrepos=binrepos, ebuilds=ebuilds ) + def testGetBinPkgExcludeBinreposConf(self): + binpkgs = self.pkgs_no_deps + ebuilds = self.pkgs_no_deps + + binrepos = {"test_binrepo": self.pkgs_no_deps} + + test_cases = { + ( + "[test_binrepo]", + "getbinpkg-exclude = foo", + ): ( + # binrepos.conf attributes to have no effect without --getbinpkg + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + mergelist=[ + "app-misc/foo-1.0", + "app-misc/bar-1.0", + "app-misc/baz-1.0", + ], + ), + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--usepkgonly": True}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + # request all packages with getbinpkg-exclude in binrepos.conf + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary,remote]app-misc/bar-1.0", + "[binary,remote]app-misc/baz-1.0", + ], + ), + # suppliment binrepos.conf with --getbinpkg-exclude on command line + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-exclude": ["bar"]}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary,remote]app-misc/baz-1.0", + ], + ), + # override binrepos.conf with --getbinpkg-include on command line + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-include": ["foo"]}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + ), + ( + "[test_binrepo]", + "getbinpkg-exclude = foo", + "getbinpkg-include = foo", + ): ( + # conflicted repos.conf attributes to have no effect + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0", + "[binary,remote]app-misc/bar-1.0", + "[binary,remote]app-misc/baz-1.0", + ], + ), + ), + ( + "[test_binrepo]", + "getbinpkg-exclude = foo bar", + "getbinpkg-include = foo", + ): ( + # conflicted binrepos.conf attributes to not interfere with + # non-overlapping usepkg-exclude + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary,remote]app-misc/baz-1.0", + ], + ), + # remaining atoms in binrepos.conf sourced lists after conflicting + # attributes have been filtered still to be overridable using + # --getbinpkg-include on command line + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-include": ["bar"]}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary,remote]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + ), + } + + self.runBinPkgSelectionTestUserConfig( + "binrepos.conf", + test_cases, + binpkgs=binpkgs, + binrepos=binrepos, + ebuilds=ebuilds, + ) + + def testGetBinPkgExcludeMultiBinrepo(self): + # here things get a bit more complicated... to interpret test results we + # need to know from which binrepo a package came, which is not something + # the resolver can tell us. instead we will put binaries with build-id + # of 1 into the binhost 'test_binrepo', and binaries with build-id of 2 + # into the binhost 'other_binrepo', the build-id being something we can + # check in the mergelist of each test case. the binaries are otherwise + # identical in each binrepo. + # + # to make tests outcomes easier to follow we also want both binhosts to + # fallback on local binaries rather than ebuilds when fetching is blocked + # by getbinpkg-exclude, which means pkgdir needs to contain the packages + # from *both* binhosts. + pkgs_no_deps_b1 = with_build_id(self.pkgs_no_deps, "1") + pkgs_no_deps_b2 = with_build_id(self.pkgs_no_deps, "2") + + ebuilds = self.pkgs_no_deps + binpkgs = pkgs_no_deps_b1 + pkgs_no_deps_b2 + + binrepos = { + "test_binrepo": pkgs_no_deps_b1, + "other_binrepo": pkgs_no_deps_b2, + } + + user_config = { + "binrepos.conf": ( + "[test_binrepo]", + "getbinpkg-exclude = foo", + "[other_binrepo]", + "getbinpkg-exclude = bar", + ), + } + + test_cases = ( + # no build-id allows the resolver to work around explicit exclusions + # by using the binrepo with no applicable getbinpkg-exclude as these + # builds are identical in the test environment. + # + # still requesting a specific build-id for app-misc/baz as how the + # resolver disambiguates equivalent binaries is not under test here. + ResolverPlaygroundTestCase( + ["app-misc/foo", "app-misc/bar", "=app-misc/baz-1.0-2"], + success=True, + ignore_mergelist_order=True, + check_repo_names=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-2", + "[binary,remote]app-misc/bar-1.0-1", + "[binary,remote]app-misc/baz-1.0-2", + ], + ), + # explicitly request build-id 1 from test_binrepo such that the + # getbinpkg-exclude=foo entry in binrepos.conf applies + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-1", "=app-misc/bar-1.0-1", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary]app-misc/foo-1.0-1", + "[binary,remote]app-misc/bar-1.0-1", + "[binary,remote]app-misc/baz-1.0-1", + ], + ), + # explicitly request build-id 2 from other_binrepo such that the + # getbinpkg-exclude=bar entry in binrepos.conf applies + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-2", "=app-misc/bar-1.0-2", "=app-misc/baz-1.0-2"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-2", + "[binary]app-misc/bar-1.0-2", + "[binary,remote]app-misc/baz-1.0-2", + ], + ), + # explicitly request the build-ids for the repositories where each + # of app-misc/foo and app-misc/bar are excluded + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-1", "=app-misc/bar-1.0-2", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary]app-misc/foo-1.0-1", + "[binary]app-misc/bar-1.0-2", + "[binary,remote]app-misc/baz-1.0-1", + ], + ), + # explicitly request the build-ids for the repositories where neither + # of app-misc/foo and app-misc/bar are excluded + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-2", "=app-misc/bar-1.0-1", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-2", + "[binary,remote]app-misc/bar-1.0-1", + "[binary,remote]app-misc/baz-1.0-1", + ], + ), + # explicitly request the build-ids for the repositories where each + # of app-misc/foo and app-misc/bar are excluded, but override this + # for app-misc/foo using --getbinpkg-include. this will implicitly + # exclude app-misc/baz. + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-1", "=app-misc/bar-1.0-2", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-include": ["foo"]}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-1", + "[binary]app-misc/bar-1.0-2", + "[binary]app-misc/baz-1.0-1", + ], + ), + ) + + self.runBinPkgSelectionTest( + test_cases, + binpkgs=binpkgs, + binrepos=binrepos, + ebuilds=ebuilds, + user_config=user_config, + ) + # test --getbinpkg-include option class GetBinPkgIncludeTestCase(BinPkgSelectionTestCase): @@ -599,6 +859,237 @@ class GetBinPkgIncludeTestCase(BinPkgSelectionTestCase): test_cases, binpkgs=binpkgs, binrepos=binrepos, ebuilds=ebuilds ) + def testGetBinPkgIncludeBinreposConf(self): + binpkgs = self.pkgs_no_deps + ebuilds = self.pkgs_no_deps + + binrepos = {"test_binrepo": self.pkgs_no_deps} + + test_cases = { + ( + "[test_binrepo]", + "getbinpkg-include = foo", + ): ( + # binrepos.conf attributes to have no effect without --getbinpkg + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + mergelist=[ + "app-misc/foo-1.0", + "app-misc/bar-1.0", + "app-misc/baz-1.0", + ], + ), + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--usepkgonly": True}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + # request all packages with getbinpkg-include in binrepos.conf + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + # suppliment binrepos.conf with --getbinpkg-include on command line + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-include": ["bar"]}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0", + "[binary,remote]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + # override binrepos.conf with --getbinpkg-exclude on command line + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-exclude": ["foo"]}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary,remote]app-misc/bar-1.0", + "[binary,remote]app-misc/baz-1.0", + ], + ), + ), + ( + "[test_binrepo]", + "getbinpkg-exclude = foo", + "getbinpkg-include = foo bar", + ): ( + # conflicted binrepos.conf attributes to not interfere with + # non-overlapping usepkg-include + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary]app-misc/foo-1.0", + "[binary,remote]app-misc/bar-1.0", + "[binary]app-misc/baz-1.0", + ], + ), + # remaining atoms in binrepos.conf sourced lists after conflicting + # attributes have been filtered still to be overridable using + # --getbinpkg-exclude on command line + ResolverPlaygroundTestCase( + self.pkg_atoms, + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-exclude": ["bar"]}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0", + "[binary]app-misc/bar-1.0", + "[binary,remote]app-misc/baz-1.0", + ], + ), + ), + } + + self.runBinPkgSelectionTestUserConfig( + "binrepos.conf", + test_cases, + binpkgs=binpkgs, + binrepos=binrepos, + ebuilds=ebuilds, + ) + + def testGetBinPkgIncludeMultiBinrepo(self): + # see commentary in testGetBinPkgExcludeMultiBinrepo() + pkgs_no_deps_b1 = with_build_id(self.pkgs_no_deps, "1") + pkgs_no_deps_b2 = with_build_id(self.pkgs_no_deps, "2") + + ebuilds = self.pkgs_no_deps + binpkgs = pkgs_no_deps_b1 + pkgs_no_deps_b2 + + binrepos = { + "test_binrepo": pkgs_no_deps_b1, + "other_binrepo": pkgs_no_deps_b2, + } + + user_config = { + "binrepos.conf": ( + "[test_binrepo]", + "getbinpkg-include = foo", + "[other_binrepo]", + "getbinpkg-include = bar", + ), + } + + test_cases = ( + # no build-id allows the resolver to work around implicit exclusions + # by using the binrepo with no applicable getbinpkg-include as these + # builds are identical in the test environment. + # + # still requesting a specific build-id for app-misc/baz as how the + # resolver disambiguates equivalent binaries is not under test here. + ResolverPlaygroundTestCase( + ["app-misc/foo", "app-misc/bar", "=app-misc/baz-1.0-2"], + success=True, + ignore_mergelist_order=True, + check_repo_names=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-1", + "[binary,remote]app-misc/bar-1.0-2", + "[binary]app-misc/baz-1.0-2", + ], + ), + # explicitly request build-id 1 from test_binrepo such that the + # getbinpkg-include=foo entry in binrepos.conf applies + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-1", "=app-misc/bar-1.0-1", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-1", + "[binary]app-misc/bar-1.0-1", + "[binary]app-misc/baz-1.0-1", + ], + ), + # explicitly request build-id 2 from other_binrepo such that the + # getbinpkg-include=bar entry in binrepos.conf applies + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-2", "=app-misc/bar-1.0-2", "=app-misc/baz-1.0-2"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary]app-misc/foo-1.0-2", + "[binary,remote]app-misc/bar-1.0-2", + "[binary]app-misc/baz-1.0-2", + ], + ), + # explicitly request the build-ids for the repositories where each + # of app-misc/foo and app-misc/bar are included + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-1", "=app-misc/bar-1.0-2", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary,remote]app-misc/foo-1.0-1", + "[binary,remote]app-misc/bar-1.0-2", + "[binary]app-misc/baz-1.0-1", + ], + ), + # explicitly request the build-ids for the repositories where neither + # of app-misc/foo and app-misc/bar are included + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-2", "=app-misc/bar-1.0-1", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True}, + mergelist=[ + "[binary]app-misc/foo-1.0-2", + "[binary]app-misc/bar-1.0-1", + "[binary]app-misc/baz-1.0-1", + ], + ), + # explicitly request the build-ids for the repositories where each + # of app-misc/foo and app-misc/bar are included, but override this + # for app-misc/foo using --getbinpkg-exclude. this will implicitly + # include app-misc/baz. + ResolverPlaygroundTestCase( + ["=app-misc/foo-1.0-1", "=app-misc/bar-1.0-2", "=app-misc/baz-1.0-1"], + success=True, + ignore_mergelist_order=True, + options={"--getbinpkg": True, "--getbinpkg-exclude": ["foo"]}, + mergelist=[ + "[binary]app-misc/foo-1.0-1", + "[binary,remote]app-misc/bar-1.0-2", + "[binary,remote]app-misc/baz-1.0-1", + ], + ), + ) + + self.runBinPkgSelectionTest( + test_cases, + binpkgs=binpkgs, + binrepos=binrepos, + ebuilds=ebuilds, + user_config=user_config, + ) + # test --usepkg-exclude option class UsePkgExcludeTestCase(BinPkgSelectionTestCase): diff --git a/man/portage.5 b/man/portage.5 index 41c68ab961..fbcafd96d5 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -669,6 +669,20 @@ overriding the value from \fBmake.conf\fR(5). Portage will check if the named package has an update available after syncing and notify the user to update it immediately if so. .TP +.B getbinpkg\-exclude +A space separated list of package names or slot atoms. Emerge will not +fetch matching remote binary packages except where overridden by the +\-\-getbinpkg\-include command line option (see \fBemerge\fR(1)). + +This is an experimental option subject to change. +.TP +.B getbinpkg\-include +A space separated list of package names or slot atoms. Emerge will not +fetch non-matching remote binary packages except where overridden by the +\-\-getbinpkg\-exclude command line option (see \fBemerge\fR(1)). + +This is an experimental option subject to change. +.TP .B priority Specifies priority of given repository. When a package exists in multiple repositories, those with higher priority are preferred.
