commit: 80445d9b00bfcd1eb4955cf3ecb397b4c02663ba Author: Alexey Gladkov <legion <AT> kernel <DOT> org> AuthorDate: Mon Feb 12 13:59:40 2024 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Sun Apr 28 00:04:07 2024 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=80445d9b
sync: Add method to download zip archives Add a simple method for synchronizing repository as a snapshot in a zip archive. The implementation does not require external utilities to download and unpack archive. This makes the method very cheap. The main usecase being considered is obtaining snapshots of github repositories, but many other web interfaces for git also support receiving snapshots in zip format. For example, to get a snapshot of the master branch: https://github.com/gentoo/portage/archive/refs/heads/master.zip https://gitweb.gentoo.org/proj/portage.git/snapshot/portage-master.zip or a link to a snapshot of the tag: https://github.com/gentoo/portage/archive/refs/tags/portage-3.0.61.zip Signed-off-by: Alexey Gladkov <legion <AT> kernel.org> Signed-off-by: Sam James <sam <AT> gentoo.org> lib/portage/sync/modules/zipfile/__init__.py | 33 +++++++++++ lib/portage/sync/modules/zipfile/zipfile.py | 82 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/lib/portage/sync/modules/zipfile/__init__.py b/lib/portage/sync/modules/zipfile/__init__.py new file mode 100644 index 0000000000..19fe3af412 --- /dev/null +++ b/lib/portage/sync/modules/zipfile/__init__.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2024 Alexey Gladkov <[email protected]> + +doc = """Zipfile plug-in module for portage. +Performs a http download of a portage snapshot and unpacks it to the repo +location.""" +__doc__ = doc[:] + + +import os + +from portage.sync.config_checks import CheckSyncConfig + + +module_spec = { + "name": "zipfile", + "description": doc, + "provides": { + "zipfile-module": { + "name": "zipfile", + "sourcefile": "zipfile", + "class": "ZipFile", + "description": doc, + "functions": ["sync"], + "func_desc": { + "sync": "Performs an archived http download of the " + + "repository, then unpacks it.", + }, + "validate_config": CheckSyncConfig, + "module_specific_options": (), + }, + }, +} diff --git a/lib/portage/sync/modules/zipfile/zipfile.py b/lib/portage/sync/modules/zipfile/zipfile.py new file mode 100644 index 0000000000..1762d2c8f1 --- /dev/null +++ b/lib/portage/sync/modules/zipfile/zipfile.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2024 Alexey Gladkov <[email protected]> + +import os +import os.path +import logging +import zipfile +import shutil +import tempfile +import urllib.request + +import portage +from portage.util import writemsg_level +from portage.sync.syncbase import SyncBase + + +class ZipFile(SyncBase): + """ZipFile sync module""" + + short_desc = "Perform sync operations on GitHub repositories" + + @staticmethod + def name(): + return "ZipFile" + + def __init__(self): + SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3") + + def sync(self, **kwargs): + """Sync the repository""" + if kwargs: + self._kwargs(kwargs) + + # initial checkout + zip_uri = self.repo.sync_uri + + with urllib.request.urlopen(zip_uri) as response: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + shutil.copyfileobj(response, tmp_file) + zip_file = tmp_file.name + + if not zipfile.is_zipfile(zip_file): + msg = "!!! file is not a zip archive." + self.logger(self.xterm_titles, msg) + writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR) + + os.unlink(zip_file) + + return (1, False) + + # Drop previous tree + shutil.rmtree(self.repo.location) + + with zipfile.ZipFile(zip_file) as archive: + strip_comp = 0 + + for f in archive.namelist(): + f = os.path.normpath(f) + if os.path.basename(f) == "profiles": + strip_comp = f.count("/") + break + + for n in archive.infolist(): + p = os.path.normpath(n.filename) + + if os.path.isabs(p): + continue + + parts = p.split("/") + dstpath = os.path.join(self.repo.location, *parts[strip_comp:]) + + if n.is_dir(): + os.makedirs(dstpath, mode=0o755, exist_ok=True) + continue + + with archive.open(n) as srcfile: + with open(dstpath, "wb") as dstfile: + shutil.copyfileobj(srcfile, dstfile) + + os.unlink(zip_file) + + return (os.EX_OK, True)
