commit: 20cd76664c11991e59b7d72b782fea96259ff9af Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Wed Jul 17 02:14:34 2024 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Wed Jul 17 02:14:34 2024 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=20cd7666
binrepos.conf: Support custom download location Download packages to a custom location if it is configured in binrepos.conf, instead of PKGDIR. If a custom download location is not configured then inject downloaded packages into PKGDIR as usual. The binarytree download_required method should now be used instead of the isremote method to check if download is required, since a remote package may or may not be cached in the custom location. The get_local_repo_location method should be used to check if there is a custom download location, and if there is then downloaded packages must not be injected into PKGDIR. If any packages from a repo with a custom download location were injected into PKGDIR in the past, their identity will be recognized and will not be re-downloaded to the custom location. Bug: https://bugs.gentoo.org/934784 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/_emerge/Binpkg.py | 7 +++-- lib/_emerge/BinpkgFetcher.py | 8 ++++- lib/_emerge/BinpkgPrefetcher.py | 7 +++++ lib/_emerge/Scheduler.py | 8 +++-- lib/_emerge/actions.py | 2 +- lib/portage/binrepo/config.py | 3 ++ lib/portage/dbapi/bintree.py | 70 +++++++++++++++++++++++++++++++++++++++-- lib/portage/versions.py | 6 +++- man/portage.5 | 7 ++++- 9 files changed, 106 insertions(+), 12 deletions(-) diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py index 299ae7fbc9..437111fa10 100644 --- a/lib/_emerge/Binpkg.py +++ b/lib/_emerge/Binpkg.py @@ -170,7 +170,7 @@ class Binpkg(CompositeTask): pkg_count = self.pkg_count fetcher = None - if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): + if self.opts.getbinpkg and self._bintree.download_required(pkg.cpv): fetcher = BinpkgFetcher( background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), @@ -245,7 +245,10 @@ class Binpkg(CompositeTask): pkg = self.pkg pkg_count = self.pkg_count - if self._fetched_pkg: + if self._fetched_pkg and self._bintree.get_local_repo_location(pkg.cpv): + os.rename(self._fetched_pkg, self._pkg_allocated_path) + pkg_path = self._pkg_allocated_path + elif self._fetched_pkg: stdout_orig = sys.stdout stderr_orig = sys.stderr out = io.StringIO() diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 19d08359f0..a357bac82d 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -34,8 +34,14 @@ class BinpkgFetcher(CompositeTask): ) binpkg_format = get_binpkg_format(binpkg_path) + getname_kwargs = {} + if not bintree.get_local_repo_location(pkg.cpv): + getname_kwargs.update( + dict(allocate_new=True, remote_binpkg_format=binpkg_format) + ) + self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname( - pkg.cpv, allocate_new=True, remote_binpkg_format=binpkg_format + pkg.cpv, **getname_kwargs ) self.pkg_path = self.pkg_allocated_path + ".partial" diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py index a8af30ca80..f7204bcc17 100644 --- a/lib/_emerge/BinpkgPrefetcher.py +++ b/lib/_emerge/BinpkgPrefetcher.py @@ -51,6 +51,13 @@ class BinpkgPrefetcher(CompositeTask): self.wait() return + if self._bintree.get_local_repo_location(self.pkg.cpv): + os.rename(self.pkg_path, self.pkg_allocated_path) + self._current_task = None + self.returncode = os.EX_OK + self.wait() + return + injected_pkg = None stdout_orig = sys.stdout stderr_orig = sys.stderr diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 614df9e783..e23ebeb7ac 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -830,7 +830,7 @@ class Scheduler(PollScheduler): elif ( pkg.type_name == "binary" and "--getbinpkg" in self.myopts - and pkg.root_config.trees["bintree"].isremote(pkg.cpv) + and pkg.root_config.trees["bintree"].download_required(pkg.cpv) ): prefetcher = BinpkgPrefetcher( background=True, pkg=pkg, scheduler=self._sched_iface @@ -939,7 +939,7 @@ class Scheduler(PollScheduler): # Display fetch on stdout, so that it's always clear what # is consuming time here. - if bintree.isremote(x.cpv): + if bintree.download_required(x.cpv): fetcher = self._get_prefetcher(x) if fetcher is not None and not fetcher.isAlive(): # Cancel it because it hasn't started yet. @@ -983,7 +983,9 @@ class Scheduler(PollScheduler): continue current_task = None - if fetched: + if fetched and bintree.get_local_repo_location(x.cpv): + os.rename(fetched, fetcher.pkg_allocated_path) + elif fetched: if not bintree.inject( x.cpv, current_pkg_path=fetched, diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index 43d936fd14..4554445999 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -1882,7 +1882,7 @@ def action_info(settings, trees, myopts, myfiles): matches.reverse() for match in matches: if pkg_type == "binary": - if db.bintree.isremote(match): + if db.bintree.download_required(match): continue auxkeys = ["EAPI", "DEFINED_PHASES"] metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys))) diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py index c744d1012f..97207eb24b 100644 --- a/lib/portage/binrepo/config.py +++ b/lib/portage/binrepo/config.py @@ -16,6 +16,7 @@ class BinRepoConfig: "name", "name_fallback", "fetchcommand", + "location", "priority", "resumecommand", "sync_uri", @@ -40,6 +41,8 @@ class BinRepoConfig: indent = " " * 4 repo_msg = [] repo_msg.append(self.name or self.name_fallback) + if self.location: + repo_msg.append(indent + "location: " + self.location) if self.priority is not None: repo_msg.append(indent + "priority: " + str(self.priority)) repo_msg.append(indent + "sync-uri: " + self.sync_uri) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index b32dea1eae..6099bab968 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -468,7 +468,7 @@ class bindbapi(fakedbapi): pkg = getattr(pkg, "cpv", pkg) filesdict = {} - if not self.bintree.isremote(pkg): + if not self.bintree.download_required(pkg): pass else: metadata = self.bintree._remotepkgs[self._instance_key(pkg)] @@ -1630,7 +1630,11 @@ class binarytree: remote_base_uri = pkgindex.header.get("URI", base_url) for d in pkgindex.packages: cpv = _pkg_str( - d["CPV"], metadata=d, settings=self.settings, db=self.dbapi + d["CPV"], + metadata=d, + settings=self.settings, + db=self.dbapi, + repoconfig=repo, ) # Local package instances override remote instances # with the same instance_key. @@ -2265,6 +2269,14 @@ class binarytree: path = self._pkg_paths.get(instance_key) if path is not None: filename = os.path.join(self.pkgdir, path) + elif instance_key in self._remotepkgs: + remote_metadata = self._remotepkgs[instance_key] + location = self.get_local_repo_location(cpv) + if location: + return ( + os.path.join(location, remote_metadata["PATH"]), + int(remote_metadata["BUILD_ID"]), + ) if filename is None and not allocate_new: try: @@ -2422,7 +2434,10 @@ class binarytree: def isremote(self, pkgname): """Returns true if the package is kept remotely and it has not been - downloaded (or it is only partially downloaded).""" + downloaded (or it is only partially downloaded), or if the package + is cached in a binrepo location (use download_required to check if + cached file has correct size and mtime). + """ if self._remotepkgs is None: return False instance_key = self.dbapi._instance_key(pkgname) @@ -2434,6 +2449,55 @@ class binarytree: # package is downloaded, state is updated by self.inject(). return True + def download_required(self, pkgname): + """Returns True if package is remote and download is required.""" + if not self._remotepkgs: + return False + + instance_key = self.dbapi._instance_key(pkgname) + remote_metadata = self._remotepkgs.get(instance_key) + if remote_metadata is None: + return False + + if not remote_metadata["CPV"]._repoconfig.location: + # In this case the package would have been removed from + # self._remotepkgs if it was already downloaded. + return True + + pkg_path = self.getname(pkgname) + try: + st = os.stat(pkg_path) + except OSError: + return True + + return ( + int(remote_metadata["SIZE"]) != st.st_size + or int(remote_metadata["_mtime_"]) != st[stat.ST_MTIME] + ) + + def get_local_repo_location(self, pkgname): + """Returns local repo location associated with pkgname or None + if a location is not associated.""" + # Since pkgname._repoconfig is not guaranteed to be present + # here, retrieve it from the remote metadata. + if not self._remotepkgs: + return None + instance_key = self.dbapi._instance_key(pkgname) + remote_metadata = self._remotepkgs.get(instance_key) + if remote_metadata is None: + return False + repoconfig = remote_metadata["CPV"]._repoconfig + if repoconfig is None: + return None + if repoconfig.location: + location = normalize_path(repoconfig.location) + if location == self.pkgdir: + # If the cache location is set to the same location as + # PKGDIR, then behave as though it is unset so that + # packages will be correctly injected into PKGDIR. + return None + return repoconfig.location + def get_pkgindex_uri(self, cpv): """Returns the URI to the Packages file for a given package.""" uri = None diff --git a/lib/portage/versions.py b/lib/portage/versions.py index 0e515ba5c8..c7eb91b384 100644 --- a/lib/portage/versions.py +++ b/lib/portage/versions.py @@ -1,5 +1,5 @@ # versions.py -- core Portage functionality -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -386,6 +386,7 @@ class _pkg_str(str): file_size: Optional[int] = None, mtime: Optional[int] = None, db: Any = None, + repoconfig: Any = None, ): return str.__new__(cls, cpv) @@ -402,6 +403,7 @@ class _pkg_str(str): file_size: Optional[int] = None, mtime: Optional[int] = None, db: Any = None, + repoconfig: Any = None, ): if not isinstance(cpv, str): # Avoid TypeError from str.__init__ with PyPy. @@ -420,6 +422,8 @@ class _pkg_str(str): self.__dict__["_settings"] = settings if db is not None: self.__dict__["_db"] = db + if repoconfig is not None: + self.__dict__["_repoconfig"] = repoconfig if eapi is not None: self.__dict__["eapi"] = eapi diff --git a/man/portage.5 b/man/portage.5 index 66437d8f8a..f5cbe0ad4e 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1,4 +1,4 @@ -.TH "PORTAGE" "5" "May 2024" "Portage @VERSION@" "Portage" +.TH "PORTAGE" "5" "Jul 2024" "Portage @VERSION@" "Portage" .SH NAME portage \- the heart of Gentoo .SH "DESCRIPTION" @@ -667,6 +667,11 @@ overriding the value from \fBmake.conf\fR(5). Specifies priority of given repository. When a package exists in multiple repositories, those with higher priority are preferred. .TP +.B location +Specify a cache location which serves as a partial local mirror of a +remote repository. If unset then the effective default is PKGDIR +(see \fBmake.conf\fR(5)). +.TP .B sync\-uri Specifies URI of repository used for `emerge \-\-getbinpkg`. .RE
