commit: 5ebc8a249b08318da5a2ca89cee2eed604f7e639 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Mon Sep 7 00:13:13 2020 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Tue Sep 8 01:38:08 2020 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=5ebc8a24
binrepos.conf: support fetchcommand customization (bug 668302) Support customization of fetchcommand and resumecommand in binrepos.conf, allowing customized authentication mechanisms for each repository. Bug: https://bugs.gentoo.org/668302 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/_emerge/BinpkgFetcher.py | 29 +++++++++++++++++++---------- lib/portage/binrepo/config.py | 2 ++ lib/portage/dbapi/bintree.py | 34 +++++++++++++++++++++++++--------- man/portage.5 | 14 ++++++++++++++ 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 218d4d2ab..9a96bde28 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -96,14 +96,17 @@ class _BinpkgFetcherProcess(SpawnProcess): # urljoin doesn't work correctly with # unrecognized protocols like sftp + fetchcommand = None + resumecommand = None if bintree._remote_has_index: - instance_key = bintree.dbapi._instance_key(pkg.cpv) - rel_uri = bintree._remotepkgs[instance_key].get("PATH") + remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)] + rel_uri = remote_metadata.get("PATH") if not rel_uri: rel_uri = pkg.cpv + ".tbz2" - remote_base_uri = bintree._remotepkgs[ - instance_key]["BASE_URI"] + remote_base_uri = remote_metadata["BASE_URI"] uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") + fetchcommand = remote_metadata.get('FETCHCOMMAND') + resumecommand = remote_metadata.get('RESUMECOMMAND') else: uri = settings["PORTAGE_BINHOST"].rstrip("/") + \ "/" + pkg.pf + ".tbz2" @@ -114,13 +117,19 @@ class _BinpkgFetcherProcess(SpawnProcess): self._async_wait() return - protocol = urllib_parse_urlparse(uri)[0] - fcmd_prefix = "FETCHCOMMAND" + fcmd = None if resume: - fcmd_prefix = "RESUMECOMMAND" - fcmd = settings.get(fcmd_prefix + "_" + protocol.upper()) - if not fcmd: - fcmd = settings.get(fcmd_prefix) + fcmd = resumecommand + else: + fcmd = fetchcommand + if fcmd is None: + protocol = urllib_parse_urlparse(uri)[0] + fcmd_prefix = "FETCHCOMMAND" + if resume: + fcmd_prefix = "RESUMECOMMAND" + fcmd = settings.get(fcmd_prefix + "_" + protocol.upper()) + if not fcmd: + fcmd = settings.get(fcmd_prefix) fcmd_vars = { "DISTDIR" : os.path.dirname(pkg_path), diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py index a4bce9073..6ba1a3e9f 100644 --- a/lib/portage/binrepo/config.py +++ b/lib/portage/binrepo/config.py @@ -15,7 +15,9 @@ class BinRepoConfig: __slots__ = ( 'name', 'name_fallback', + 'fetchcommand', 'priority', + 'resumecommand', 'sync_uri', ) def __init__(self, opts): diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 97018db6e..e4393e06d 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -382,10 +382,10 @@ class binarytree: self._pkgindex_keys.update(["CPV", "SIZE"]) self._pkgindex_aux_keys = \ ["BASE_URI", "BDEPEND", "BUILD_ID", "BUILD_TIME", "CHOST", - "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI", + "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI", "FETCHCOMMAND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PKGINDEX_URI", "PROPERTIES", "PROVIDES", - "RDEPEND", "repository", "REQUIRES", "RESTRICT", + "RDEPEND", "repository", "REQUIRES", "RESTRICT", "RESUMECOMMAND", "SIZE", "SLOT", "USE"] self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) self._pkgindex_use_evaluated_keys = \ @@ -979,7 +979,7 @@ class binarytree: # Don't use urlopen for https, unless # PEP 476 is supported (bug #469888). - if parsed_url.scheme not in ('https',) or _have_pep_476(): + if repo.fetchcommand is None and (parsed_url.scheme not in ('https',) or _have_pep_476()): try: f = _urlopen(url, if_modified_since=local_timestamp, proxies=proxies) if hasattr(f, 'headers') and f.headers.get('timestamp', ''): @@ -1004,7 +1004,7 @@ class binarytree: path = parsed_url.path.rstrip("/") + "/Packages" - if parsed_url.scheme == 'ssh': + if repo.fetchcommand is None and parsed_url.scheme == 'ssh': # Use a pipe so that we can terminate the download # early if we detect that the TIMESTAMP header # matches that of the cached Packages file. @@ -1023,12 +1023,15 @@ class binarytree: stdout=subprocess.PIPE) f = proc.stdout else: - setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() - fcmd = self.settings.get(setting) - if not fcmd: - fcmd = self.settings.get('FETCHCOMMAND') + if repo.fetchcommand is None: + setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() + fcmd = self.settings.get(setting) if not fcmd: - raise EnvironmentError("FETCHCOMMAND is unset") + fcmd = self.settings.get('FETCHCOMMAND') + if not fcmd: + raise EnvironmentError("FETCHCOMMAND is unset") + else: + fcmd = repo.fetchcommand fd, tmp_filename = tempfile.mkstemp() tmp_dirname, tmp_basename = os.path.split(tmp_filename) @@ -1142,6 +1145,19 @@ class binarytree: d["CPV"] = cpv d["BASE_URI"] = remote_base_uri d["PKGINDEX_URI"] = url + # FETCHCOMMAND and RESUMECOMMAND may be specified + # by binrepos.conf, and otherwise ensure that they + # do not propagate from the Packages index since + # it may be unsafe to execute remotely specified + # commands. + if repo.fetchcommand is None: + d.pop('FETCHCOMMAND', None) + else: + d['FETCHCOMMAND'] = repo.fetchcommand + if repo.resumecommand is None: + d.pop('RESUMECOMMAND', None) + else: + d['RESUMECOMMAND'] = repo.resumecommand self._remotepkgs[self.dbapi._instance_key(cpv)] = d self.dbapi.cpv_inject(cpv) diff --git a/man/portage.5 b/man/portage.5 index 4f183654c..82dd8a509 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -634,6 +634,20 @@ is intended to be used as a replacement for the \fBmake.conf\fR(5) \- attributes are specified in "${attribute} = ${value}" format .fi +.RS +.I Attributes supported in DEFAULT section: +.RS +.TP +.B fetchcommand +Specifies a \fBFETCHCOMMAND\fR used to fetch files from a repository, +overriding the value from \fBmake.conf\fR(5). +.TP +.B resumecommand +Specifies a \fBRESUMECOMMAND\fR used to fetch files from a repository, +overriding the value from \fBmake.conf\fR(5). +.RE +.RE + .RS .I Attributes supported in sections of repositories: .RS
