commit: 7575302346d29b18cd5d90b7816cc79b86f1cc84 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Sun Nov 23 06:43:48 2025 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Mon Nov 24 21:44:00 2025 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=75753023
emaint binhost: compress-index fixes Check and fix a missing or stale Packages.gz, and delete Packages.gz if compress-index is disabled. Bug: https://bugs.gentoo.org/966251 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/portage/dbapi/bintree.py | 6 + lib/portage/emaint/modules/binhost/binhost.py | 51 +++++- lib/portage/tests/emaint/__init__.py | 2 + lib/portage/tests/emaint/__test__.py | 0 lib/portage/tests/emaint/meson.build | 9 ++ lib/portage/tests/emaint/test_emaint_binhost.py | 207 ++++++++++++++++++++++++ 6 files changed, 273 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 8f5499eafe..dd8072b837 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -2140,6 +2140,12 @@ class binarytree: fileobj, ) ) + else: + try: + os.unlink(self._pkgindex_file + ".gz") + except OSError as e: + if e.errno != errno.ENOENT: + raise for f, fname, f_close in output_files: f.write(contents) diff --git a/lib/portage/emaint/modules/binhost/binhost.py b/lib/portage/emaint/modules/binhost/binhost.py index ece6865918..6d78138be4 100644 --- a/lib/portage/emaint/modules/binhost/binhost.py +++ b/lib/portage/emaint/modules/binhost/binhost.py @@ -1,4 +1,4 @@ -# Copyright 2005-2020 Gentoo Authors +# Copyright 2005-2025 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -87,10 +87,57 @@ class BinhostHandler: errors = [f"'{cpv}' is not in Packages" for cpv in missing] for cpv in stale: errors.append(f"'{cpv}' is not in the repository") + + errors.extend(self._check_compressed_index()) + if errors: return (False, errors) return (True, None) + def _check_compressed_index(self): + """Check the compressed index file for consistency. Return a list of error + messages, or an empty list if no errors were found. + """ + errors = [] + bintree = self._bintree + timestamps = {} + for suffix in ("", ".gz"): + pkgindex_file = self._pkgindex_file + suffix + try: + st = os.stat(pkgindex_file) + except OSError as e: + timestamps[suffix] = None + if e.errno == errno.ENOENT: + if suffix == "": + errors.append(f"Missing index file: {pkgindex_file}") + elif ( + suffix == ".gz" + and "compress-index" in bintree.settings.features + ): + errors.append(f"Missing index file: {pkgindex_file}") + else: + raise + else: + timestamps[suffix] = st[stat.ST_MTIME] + + if "compress-index" in bintree.settings.features and len(timestamps) == 2: + if ( + timestamps[""] is not None + and timestamps[".gz"] is not None + and timestamps[""] != timestamps[".gz"] + ): + errors.append( + f"Uncompressed index timestamp '{timestamps['']}' is not equal to compressed index timestamp '{timestamps['.gz']}'" + ) + + if "compress-index" not in bintree.settings.features: + if timestamps[".gz"] is not None: + errors.append( + f"Compressed index exists but 'compress-index' feature is disabled: {self._pkgindex_file}.gz" + ) + + return errors + def fix(self, **kwargs): onProgress = kwargs.get("onProgress", None) bintree = self._bintree @@ -119,7 +166,7 @@ class BinhostHandler: if not d or self._need_update(cpv, d): missing.append(cpv) - if missing or stale: + if missing or stale or self._check_compressed_index(): from portage import locks pkgindex_lock = locks.lockfile(self._pkgindex_file, wantnewlockfile=1) diff --git a/lib/portage/tests/emaint/__init__.py b/lib/portage/tests/emaint/__init__.py new file mode 100644 index 0000000000..0ce6dfbec4 --- /dev/null +++ b/lib/portage/tests/emaint/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2025 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 diff --git a/lib/portage/tests/emaint/__test__.py b/lib/portage/tests/emaint/__test__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/portage/tests/emaint/meson.build b/lib/portage/tests/emaint/meson.build new file mode 100644 index 0000000000..293d27e636 --- /dev/null +++ b/lib/portage/tests/emaint/meson.build @@ -0,0 +1,9 @@ +py.install_sources( + [ + 'test_emaint_binhost.py', + '__init__.py', + '__test__.py', + ], + subdir : 'portage/tests/emaint', + pure : not native_extensions +) diff --git a/lib/portage/tests/emaint/test_emaint_binhost.py b/lib/portage/tests/emaint/test_emaint_binhost.py new file mode 100644 index 0000000000..8df3576db4 --- /dev/null +++ b/lib/portage/tests/emaint/test_emaint_binhost.py @@ -0,0 +1,207 @@ +# Copyright 2025 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import os +import subprocess +import sys +import time +from dataclasses import dataclass +from typing import Any, Callable, Optional + +import portage +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground + + +@dataclass +class CommandStep: + returncode: int + command: tuple[str, ...] + env: Optional[dict] = None + cwd: Optional[str] = None + + +@dataclass +class FunctionStep: + function: Callable[[int], Any] # called with step index as argument + + +class EmainBinhostTestCase(TestCase): + def testCompressedIndex(self): + debug = False + + user_config = {"make.conf": ('FEATURES="-compress-index"',)} + + binpkgs = { + "app-misc/A-1": { + "EAPI": "8", + "DEPEND": "app-misc/B", + "RDEPEND": "app-misc/C", + }, + } + + playground = ResolverPlayground( + binpkgs=binpkgs, + user_config=user_config, + debug=debug, + ) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + bintree = playground.trees[eroot]["bintree"] + + cmds = {} + for cmd in ("emaint",): + for bindir in (self.bindir, self.sbindir): + path = os.path.join(str(bindir), cmd) + if os.path.exists(path): + cmds[cmd] = (portage._python_interpreter, "-b", "-Wd", path) + break + else: + raise AssertionError( + f"{cmd} binary not found in {self.bindir} or {self.sbindir}" + ) + + env = settings.environ() + env.update( + { + "PORTAGE_OVERRIDE_EPREFIX": eprefix, + "HOME": eprefix, + "PYTHONDONTWRITEBYTECODE": os.environ.get( + "PYTHONDONTWRITEBYTECODE", "" + ), + } + ) + + def current_time(offset=0): + t = time.time() + offset + return (t, t) + + steps = ( + FunctionStep( + function=lambda i: self.assertTrue( + os.path.exists(bintree._pkgindex_file), f"step {i}" + ), + ), + # The compressed index should not exist yet becuase compress-index is disabled in make.conf. + FunctionStep( + function=lambda i: self.assertFalse( + os.path.exists(bintree._pkgindex_file + ".gz"), f"step {i}" + ) + ), + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--fix"), + ), + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--check"), + ), + FunctionStep( + function=lambda i: self.assertTrue( + os.path.exists(bintree._pkgindex_file + ".gz"), f"step {i}" + ), + ), + FunctionStep( + function=lambda i: os.unlink(bintree._pkgindex_file + ".gz"), + ), + # It should report an error for a missing Packages.gz here. + CommandStep( + returncode=1, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--check"), + ), + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--fix"), + ), + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--check"), + ), + # Bump the timestamp of Packages so that Packages.gz becomes stale. + FunctionStep( + function=lambda i: os.utime( + bintree._pkgindex_file, current_time(offset=2) + ), + ), + # It should report an error for stale Packages.gz here. + CommandStep( + returncode=1, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--check"), + ), + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--fix"), + ), + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "compress-index"}, + command=cmds["emaint"] + ("binhost", "--check"), + ), + # It should delete the unwanted Packages.gz here when compress-index is disabled. + CommandStep( + returncode=os.EX_OK, + env={"FEATURES": "-compress-index"}, + command=cmds["emaint"] + ("binhost", "--fix"), + ), + FunctionStep( + function=lambda i: self.assertFalse( + os.path.exists(bintree._pkgindex_file + ".gz"), f"step {i}" + ) + ), + ) + + try: + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for i, step in enumerate(steps): + if isinstance(step, FunctionStep): + try: + step.function(i) + except Exception as e: + if isinstance(e, AssertionError) and f"step {i}" in str(e): + raise + raise AssertionError( + f"step {i} raised {e.__class__.__name__}" + ) from e + continue + + proc = subprocess.Popen( + step.command, + env=dict(env.items(), **(step.env or {})), + cwd=step.cwd, + stdout=stdout, + ) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != step.returncode: + for line in output: + sys.stderr.write(portage._unicode_decode(line)) + + self.assertEqual( + step.returncode, + proc.returncode, + f"{step.command} (step {i}) failed with exit code {proc.returncode}", + ) + + finally: + playground.cleanup()
