commit: fa901a6510c4a5f72dec6aad86db6fe7efd6e7b3 Author: Sheng Yu <syu.os <AT> protonmail <DOT> com> AuthorDate: Tue Sep 6 18:38:27 2022 +0000 Commit: Michał Górny <mgorny <AT> gentoo <DOT> org> CommitDate: Fri Sep 9 10:16:06 2022 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa901a65
Add BUILD_ID to metadata during binary packaging Also create placeholder for new multi-instance package before actually packaging to avoid race. Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com> Closes: https://github.com/gentoo/portage/pull/893 Signed-off-by: Michał Górny <mgorny <AT> gentoo.org> bin/gpkg-helper.py | 4 +- bin/misc-functions.sh | 6 +- bin/quickpkg | 2 +- lib/_emerge/Binpkg.py | 2 +- lib/_emerge/BinpkgFetcher.py | 27 ++-- lib/_emerge/BinpkgPrefetcher.py | 8 +- lib/_emerge/EbuildBinpkg.py | 35 +++-- lib/_emerge/Scheduler.py | 2 +- lib/portage/dbapi/bintree.py | 145 +++++++++++---------- .../package/ebuild/_config/special_env_vars.py | 1 + lib/portage/package/ebuild/doebuild.py | 3 +- 11 files changed, 126 insertions(+), 109 deletions(-) diff --git a/bin/gpkg-helper.py b/bin/gpkg-helper.py index d45177f3e..4752c84ee 100755 --- a/bin/gpkg-helper.py +++ b/bin/gpkg-helper.py @@ -19,7 +19,7 @@ def command_compose(args): sys.stderr.write("4 arguments are required, got %s\n" % len(args)) return 1 - cpv, binpkg_path, metadata_dir, image_dir = args + basename, binpkg_path, metadata_dir, image_dir = args if not os.path.isdir(metadata_dir): sys.stderr.write(usage) @@ -31,7 +31,7 @@ def command_compose(args): sys.stderr.write("Argument 4 is not a directory: '%s'\n" % image_dir) return 1 - gpkg_file = portage.gpkg.gpkg(portage.settings, cpv, binpkg_path) + gpkg_file = portage.gpkg.gpkg(portage.settings, basename, binpkg_path) metadata = gpkg_file._generate_metadata_from_dir(metadata_dir) gpkg_file.compress(image_dir, metadata) return os.EX_OK diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index faa8184c6..170e60d1c 100755 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -512,6 +512,10 @@ __dyn_package() { die "PORTAGE_BINPKG_TMPFILE is unset" mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed" + if [[ ! -z "${BUILD_ID}" ]]; then + echo -n "${BUILD_ID}" > "${PORTAGE_BUILDDIR}"/build-info/BUILD_ID + fi + if [[ "${BINPKG_FORMAT}" == "xpak" ]]; then local tar_options="" @@ -547,7 +551,7 @@ __dyn_package() { elif [[ "${BINPKG_FORMAT}" == "gpkg" ]]; then PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}"/gpkg-helper.py compress \ - "${CATEGORY}/${PF}" "${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}" + "${PF}${BUILD_ID:+-${BUILD_ID}}" "${PORTAGE_BINPKG_TMPFILE}" "${PORTAGE_BUILDDIR}/build-info" "${D}" if [[ $? -ne 0 ]]; then rm -f "${PORTAGE_BINPKG_TMPFILE}" die "Failed to create binpkg file" diff --git a/bin/quickpkg b/bin/quickpkg index 773c1c07e..eef5f912f 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -206,7 +206,7 @@ def quickpkg_atom(options, infos, arg, eout): finally: if have_lock: dblnk.unlockdb() - pkg_info = bintree.inject(cpv, filename=binpkg_tmpfile) + pkg_info = bintree.inject(cpv, current_pkg_path=binpkg_tmpfile) # The pkg_info value ensures that the following getname call # returns the correct path when FEATURES=binpkg-multi-instance # is enabled, but fallback to cpv in case the inject call diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py index 15eb56092..949ac8ee7 100644 --- a/lib/_emerge/Binpkg.py +++ b/lib/_emerge/Binpkg.py @@ -246,7 +246,7 @@ class Binpkg(CompositeTask): if self._fetched_pkg: pkg_path = self._bintree.getname( - self._bintree.inject(pkg.cpv, filename=self._fetched_pkg), + self._bintree.inject(pkg.cpv, current_pkg_path=self._fetched_pkg), allocate_new=False, ) else: diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index d5275ea11..b7021e276 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -12,6 +12,7 @@ import sys import portage from portage import os from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS +from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS from portage.exception import FileNotFound, InvalidBinaryPackageFormat from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._pty import _create_pty_or_pipe @@ -19,27 +20,31 @@ from portage.util._pty import _create_pty_or_pipe class BinpkgFetcher(CompositeTask): - __slots__ = ("pkg", "pretend", "logfile", "pkg_path") + __slots__ = ("pkg", "pretend", "logfile", "pkg_path", "pkg_allocated_path") def __init__(self, **kwargs): CompositeTask.__init__(self, **kwargs) pkg = self.pkg bintree = pkg.root_config.trees["bintree"] - binpkg_path = None + instance_key = bintree.dbapi._instance_key(pkg.cpv) + binpkg_format = bintree._remotepkgs[instance_key].get("BINPKG_FORMAT", None) - if bintree._remote_has_index: - instance_key = bintree.dbapi._instance_key(pkg.cpv) + if binpkg_format is None: binpkg_path = bintree._remotepkgs[instance_key].get("PATH") - if binpkg_path: - self.pkg_path = binpkg_path + ".partial" + if binpkg_path.endswith(SUPPORTED_XPAK_EXTENSIONS): + binpkg_format = "xpak" + elif binpkg_path.endswith(SUPPORTED_GPKG_EXTENSIONS): + binpkg_format = "gpkg" else: - self.pkg_path = ( - pkg.root_config.trees["bintree"].getname(pkg.cpv, allocate_new=True) - + ".partial" + raise InvalidBinaryPackageFormat( + f"Unsupported binary package format from '{binpkg_path}'" ) - else: - raise FileNotFound("Binary packages index not found") + + self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname( + pkg.cpv, allocate_new=True, remote_binpkg_format=binpkg_format + ) + self.pkg_path = self.pkg_allocated_path + ".partial" def _start(self): fetcher = _BinpkgFetcherProcess( diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py index 3df9ebd76..912673bd1 100644 --- a/lib/_emerge/BinpkgPrefetcher.py +++ b/lib/_emerge/BinpkgPrefetcher.py @@ -11,6 +11,7 @@ class BinpkgPrefetcher(CompositeTask): __slots__ = ("pkg",) + ( "pkg_path", + "pkg_allocated_path", "_bintree", ) @@ -23,6 +24,7 @@ class BinpkgPrefetcher(CompositeTask): scheduler=self.scheduler, ) self.pkg_path = fetcher.pkg_path + self.pkg_allocated_path = fetcher.pkg_allocated_path self._start_task(fetcher, self._fetcher_exit) def _fetcher_exit(self, fetcher): @@ -45,7 +47,11 @@ class BinpkgPrefetcher(CompositeTask): self.wait() return - self._bintree.inject(self.pkg.cpv, filename=self.pkg_path) + self._bintree.inject( + self.pkg.cpv, + current_pkg_path=self.pkg_path, + allocated_pkg_path=self.pkg_allocated_path, + ) self._current_task = None self.returncode = os.EX_OK diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py index ccdd30f7b..04a17f283 100644 --- a/lib/_emerge/EbuildBinpkg.py +++ b/lib/_emerge/EbuildBinpkg.py @@ -6,8 +6,6 @@ from _emerge.EbuildPhase import EbuildPhase import portage from portage import os -from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS -from portage.exception import InvalidBinaryPackageFormat class EbuildBinpkg(CompositeTask): @@ -15,30 +13,27 @@ class EbuildBinpkg(CompositeTask): This assumes that src_install() has successfully completed. """ - __slots__ = ("pkg", "settings") + ("_binpkg_tmpfile", "_binpkg_info") + __slots__ = ("pkg", "settings") + ( + "_binpkg_tmpfile", + "_binpkg_info", + "pkg_allocated_path", + ) def _start(self): pkg = self.pkg root_config = pkg.root_config bintree = root_config.trees["bintree"] - binpkg_format = self.settings.get( - "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] + pkg_allocated_path, build_id = bintree.getname_build_id( + pkg.cpv, allocate_new=True ) - if binpkg_format == "xpak": - binpkg_tmpfile = os.path.join( - bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid()) - ) - elif binpkg_format == "gpkg": - binpkg_tmpfile = os.path.join( - bintree.pkgdir, pkg.cpv + ".gpkg.tar." + str(portage.getpid()) - ) - else: - raise InvalidBinaryPackageFormat(binpkg_format) - bintree._ensure_dir(os.path.dirname(binpkg_tmpfile)) - self._binpkg_tmpfile = binpkg_tmpfile + self.pkg_allocated_path = pkg_allocated_path + self._binpkg_tmpfile = self.pkg_allocated_path + "." + str(portage.getpid()) self.settings["PORTAGE_BINPKG_TMPFILE"] = self._binpkg_tmpfile + if "binpkg-multi-instance" in self.settings.features: + self.settings["BUILD_ID"] = str(build_id) + package_phase = EbuildPhase( background=self.background, phase="package", @@ -61,7 +56,11 @@ class EbuildBinpkg(CompositeTask): pkg = self.pkg bintree = pkg.root_config.trees["bintree"] - self._binpkg_info = bintree.inject(pkg.cpv, filename=self._binpkg_tmpfile) + self._binpkg_info = bintree.inject( + pkg.cpv, + current_pkg_path=self._binpkg_tmpfile, + allocated_pkg_path=self.pkg_allocated_path, + ) self._current_task = None self.returncode = os.EX_OK diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index bc3531627..9e210f182 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -975,7 +975,7 @@ class Scheduler(PollScheduler): continue if fetched: - bintree.inject(x.cpv, filename=fetched) + bintree.inject(x.cpv, current_pkg_path=fetched) infloc = os.path.join(build_dir_path, "build-info") ensure_dirs(infloc) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 814e6627c..7f5dc051c 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -47,6 +47,7 @@ from portage.exception import ( from portage.localization import _ from portage.output import colorize from portage.package.ebuild.profile_iuse import iter_iuse_vars +from portage.util import ensure_dirs from portage.util.file_copy import copyfile from portage.util.futures import asyncio from portage.util.futures.executor.fork import ForkExecutor @@ -733,7 +734,8 @@ class binarytree: # assuming that it will be deleted by eclean-pkg when its # time comes. mynewcpv = _pkg_str(mynewcpv, metadata=metadata, db=self.dbapi) - update_path = self.getname(mynewcpv, allocate_new=True) + ".partial" + allocated_pkg_path = self.getname(mynewcpv, allocate_new=True) + update_path = allocated_pkg_path + ".partial" self._ensure_dir(os.path.dirname(update_path)) update_path_lock = None try: @@ -747,7 +749,11 @@ class binarytree: mybinpkg.update_metadata(mydata, new_basename=mynewcpv) else: raise InvalidBinaryPackageFormat(binpkg_format) - self.inject(mynewcpv, filename=update_path) + self.inject( + mynewcpv, + current_pkg_path=update_path, + allocated_pkg_path=allocated_pkg_path, + ) finally: if update_path_lock is not None: try: @@ -1590,13 +1596,17 @@ class binarytree: self._additional_pkgs[instance_key] = pkg self.dbapi.cpv_inject(pkg) - def inject(self, cpv, filename=None): + def inject(self, cpv, current_pkg_path=None, allocated_pkg_path=None): """Add a freshly built package to the database. This updates $PKGDIR/Packages with the new package metadata (including MD5). @param cpv: The cpv of the new package to inject @type cpv: string - @param filename: File path of the package to inject, or None if it's - already in the location returned by getname() + @param current_pkg_path: File path of the package to inject, + or None if it's already in the location returned by getname() + @type filename: string + @rtype: _pkg_str or None + @param allocated_pkg_path: File path of the package that was newly + allocated or None if it's not allocated. @type filename: string @rtype: _pkg_str or None @return: A _pkg_str instance on success, or None on failure. @@ -1604,10 +1614,10 @@ class binarytree: mycat, mypkg = catsplit(cpv) if not self.populated: self.populate() - if filename is None: + if current_pkg_path is None: full_path = self.getname(cpv) else: - full_path = filename + full_path = current_pkg_path try: s = os.stat(full_path) except OSError as e: @@ -1615,7 +1625,7 @@ class binarytree: raise del e writemsg( - _("!!! Binary package does not exist: '%s'\n") % full_path, + f"!!! Binary package does not exist: '{full_path}'\n", noiselevel=-1, ) return @@ -1664,48 +1674,19 @@ class binarytree: try: os.makedirs(self.pkgdir, exist_ok=True) pkgindex_lock = lockfile(self._pkgindex_file, wantnewlockfile=1) - if filename is not None: - new_filename = self.getname(cpv, allocate_new=True) + if current_pkg_path is not None: + if allocated_pkg_path is not None: + new_path = allocated_pkg_path + else: + new_path = self.getname(cpv, allocate_new=True) try: - samefile = os.path.samefile(filename, new_filename) + samefile = os.path.samefile(current_pkg_path, new_path) except OSError: samefile = False if not samefile: - self._ensure_dir(os.path.dirname(new_filename)) - _movefile(filename, new_filename, mysettings=self.settings) - full_path = new_filename - - basename = os.path.basename(full_path) - pf = catsplit(cpv)[1] - if (build_id is None) and (not fetched) and binpkg_format: - # Apply the newly assigned BUILD_ID. This is intended - # to occur only for locally built packages. If the - # package was fetched, we want to preserve its - # attributes, so that we can later distinguish that it - # is identical to its remote counterpart. - build_id = self._parse_build_id(basename) - if build_id > 0: - metadata["BUILD_ID"] = str(build_id) - cpv = _pkg_str( - cpv, metadata=metadata, settings=self.settings, db=self.dbapi - ) - if binpkg_format == "xpak": - if basename.endswith(".xpak"): - binpkg = portage.xpak.tbz2(full_path) - binary_data = binpkg.get_data() - binary_data[b"BUILD_ID"] = _unicode_encode( - metadata["BUILD_ID"] - ) - binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data)) - elif binpkg_format == "gpkg": - binpkg = portage.gpkg.gpkg(self.settings, cpv, full_path) - binpkg_metadata = binpkg.get_metadata() - binpkg_metadata["BUILD_ID"] = _unicode_encode( - metadata["BUILD_ID"] - ) - binpkg.update_metadata(binpkg_metadata) - else: - raise InvalidBinaryPackageFormat(basename) + self._ensure_dir(os.path.dirname(new_path)) + _movefile(current_pkg_path, new_path, mysettings=self.settings) + full_path = new_path self._file_permissions(full_path) pkgindex = self._load_pkgindex() @@ -2055,7 +2036,12 @@ class binarytree: return "" return mymatch - def getname(self, cpv, allocate_new=None): + def getname(self, cpv, allocate_new=None, remote_binpkg_format=None): + return self.getname_build_id( + cpv, allocate_new=allocate_new, remote_binpkg_format=remote_binpkg_format + )[0] + + def getname_build_id(self, cpv, allocate_new=None, remote_binpkg_format=None): """Returns a file location for this package. If cpv has both build_time and build_id attributes, then the path to the specific corresponding instance is returned. @@ -2072,8 +2058,9 @@ class binarytree: cpv = _pkg_str(cpv) filename = None + build_id = None if allocate_new: - filename = self._allocate_filename(cpv) + filename, build_id = self._allocate_filename(cpv) elif self._is_specific_instance(cpv): instance_key = self.dbapi._instance_key(cpv) path = self._pkg_paths.get(instance_key) @@ -2090,7 +2077,7 @@ class binarytree: if filename is not None: filename = os.path.join(self.pkgdir, filename) elif instance_key in self._additional_pkgs: - return None + return (None, None) if filename is None: try: @@ -2133,7 +2120,7 @@ class binarytree: else: raise InvalidBinaryPackageFormat(binpkg_format) - return filename + return (filename, build_id) def _is_specific_instance(self, cpv): specific = True @@ -2154,22 +2141,26 @@ class binarytree: max_build_id = x.build_id return max_build_id - def _allocate_filename(self, cpv): - try: - binpkg_format = cpv.binpkg_format - except AttributeError: - binpkg_format = self.settings.get( - "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] - ) + def _allocate_filename(self, cpv, remote_binpkg_format=None): + if remote_binpkg_format is None: + try: + binpkg_format = cpv.binpkg_format + except AttributeError: + binpkg_format = self.settings.get( + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] + ) + else: + binpkg_format = remote_binpkg_format + # Do not create a new placeholder to avoid overwriting existing binpkgs. if binpkg_format == "xpak": - return os.path.join(self.pkgdir, cpv + ".tbz2") + return (os.path.join(self.pkgdir, cpv + ".tbz2"), None) elif binpkg_format == "gpkg": - return os.path.join(self.pkgdir, cpv + ".gpkg.tar") + return (os.path.join(self.pkgdir, cpv + ".gpkg.tar"), None) else: raise InvalidBinaryPackageFormat(binpkg_format) - def _allocate_filename_multi(self, cpv): + def _allocate_filename_multi(self, cpv, remote_binpkg_format=None): # First, get the max build_id found when _populate was # called. @@ -2181,29 +2172,39 @@ class binarytree: pf = catsplit(cpv)[1] build_id = max_build_id + 1 - try: - binpkg_format = cpv.binpkg_format - except AttributeError: - binpkg_format = self.settings.get( - "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] - ) + if remote_binpkg_format is None: + try: + binpkg_format = cpv.binpkg_format + except AttributeError: + binpkg_format = self.settings.get( + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] + ) + else: + binpkg_format = remote_binpkg_format if binpkg_format == "xpak": - filename_format = "%s-%s.xpak" + binpkg_suffix = "xpak" elif binpkg_format == "gpkg": - filename_format = "%s-%s.gpkg.tar" + binpkg_suffix = "gpkg.tar" else: raise InvalidBinaryPackageFormat(binpkg_format) while True: - filename = filename_format % ( - os.path.join(self.pkgdir, cpv.cp, pf), - build_id, + filename = ( + f"{os.path.join(self.pkgdir, cpv.cp, pf)}-{build_id}.{binpkg_suffix}" ) if os.path.exists(filename): build_id += 1 else: - return filename + try: + # Avoid races + ensure_dirs(os.path.dirname(filename)) + with open(filename, "x") as f: + pass + except FileExistsError: + build_id += 1 + continue + return (filename, build_id) @staticmethod def _parse_build_id(filename): diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py index 04e4c5b9b..1de62e421 100644 --- a/lib/portage/package/ebuild/_config/special_env_vars.py +++ b/lib/portage/package/ebuild/_config/special_env_vars.py @@ -88,6 +88,7 @@ environ_whitelist += [ "BASH_FUNC____in_portage_iuse%%", "BINPKG_FORMAT", "BROOT", + "BUILD_ID", "BUILD_PREFIX", "COLUMNS", "D", diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 8ee9f73c6..d0d134b39 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -1463,7 +1463,8 @@ def doebuild( if retval == os.EX_OK: if mydo == "package" and bintree is not None: pkg = bintree.inject( - mysettings.mycpv, filename=mysettings["PORTAGE_BINPKG_TMPFILE"] + mysettings.mycpv, + current_pkg_path=mysettings["PORTAGE_BINPKG_TMPFILE"], ) if pkg is not None: infoloc = os.path.join(
