commit: f197db0e9318f946b4085bb11c92da28158d5c19 Author: Ben Torkington <prmera <AT> gmail <DOT> com> AuthorDate: Wed Mar 4 00:58:47 2026 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Wed Mar 4 03:56:30 2026 +0000 URL: https://gitweb.gentoo.org/proj/mirrorselect.git/commit/?id=f197db0e
implement GLEP75 compliant path resolution Closes: https://bugs.gentoo.org/921759 Signed-off-by: Ben Torkington <prmera <AT> gmail.com> Part-of: https://codeberg.org/gentoo/mirrorselect/pulls/3 Merges: https://codeberg.org/gentoo/mirrorselect/pulls/3 Signed-off-by: Sam James <sam <AT> gentoo.org> mirrorselect/main.py | 2 +- mirrorselect/selectors.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ setup.py | 8 ++++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/mirrorselect/main.py b/mirrorselect/main.py index 54e3164..3a418a7 100755 --- a/mirrorselect/main.py +++ b/mirrorselect/main.py @@ -266,7 +266,7 @@ class MirrorSelect: "-f", "--file", action="store", - default="f2/mirrorselect-test", + default="mirrorselect-test", help="An alternate file to download for deep testing. " "Please choose the file carefully as to not abuse the system " "by selecting an overly large size file. You must also " diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py index 3437309..1041d82 100644 --- a/mirrorselect/selectors.py +++ b/mirrorselect/selectors.py @@ -37,11 +37,23 @@ import subprocess import sys import time import hashlib +import itertools import urllib.request import urllib.parse import urllib.error +from mirrorselect.extractor import USERAGENT +from portage.package.ebuild.fetch import ( + MirrorLayoutConfig, + FlatLayout, + get_mirror_url, + ) +from configparser import ( + ConfigParser, + Error as ConfigParseError + ) + url_parse = urllib.parse.urlparse url_unparse = urllib.parse.urlunparse url_open = urllib.request.urlopen @@ -297,6 +309,43 @@ class Deep: ) self.urls = rethosts + def get_distfile_structure(self, distfiles_url): + """ + Obtain the GLEP 75 Mirror Layout from layout.conf + + This is a modification of that found in portage + in package/ebuild/fetch.py, however that implementation + requires storing the layout.conf as a temporary file. + + GLEP 75 explains the mechanism for mirrors to communicate + the path schema they use. + See: https://www.gentoo.org/glep/glep-0075.html + """ + config_parser = ConfigParser() + config_url = Deep._urljoin(distfiles_url, "layout.conf") + + self.output.write(f"_get_distfile_structure(): config_url = {config_url}\n", 2) + + response = url_open(config_url, None, self._connect_timeout) + + if response.status == 404: + self.output.write("_get_distfile_structure(): no layout.conf, assuming flat\n", 2) + # mirrors lacking a layout.conf are assume to use a flat layout + return FlatLayout() + + config_parser.read_string(response.read().decode('utf-8')) + vals = [] + + for i in itertools.count(): + try: + vals.append(tuple(config_parser.get("structure", "%d" % i).split())) + except ConfigParseError: + break + + mlc = MirrorLayoutConfig() + mlc.deserialize(vals) + return mlc.get_best_supported_layout() + def deeptime(self, url, maxtime): """ Takes a single url and fetch command, and downloads the test file. @@ -307,8 +356,21 @@ class Deep: dist_url = Deep._urljoin(url, "distfiles") + try: + structure = self.get_distfile_structure(dist_url) + except OSError as e: + self.output.write( + f"deeptime(): unable to connect to host {url}\n", + 2, + ) + return (None, True) + + path = structure.get_path(self.test_file) + url = self._urljoin(dist_url, path) url_parts = url_parse(url) + self.output.write(f"_deeptime(): testfile url = {url}\n", 1) + signal.signal(signal.SIGALRM, timeout_handler) ips = [] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b71fdca --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +portage diff --git a/setup.py b/setup.py index de41dca..b5d0d42 100755 --- a/setup.py +++ b/setup.py @@ -30,6 +30,13 @@ python_scripts = [os.path.join(cwd, path) for path in ("mirrorselect/version.py" manpage = [os.path.join(cwd, path) for path in ("mirrorselect.8",)] +# requirements.txt +lib_folder = os.path.dirname(os.path.realpath(__file__)) +requirements_path = f"{lib_folder}/requirements.txt" +install_requires = [] +if os.path.isfile(requirements_path): + with open(requirements_path) as f: + install_requires = f.read().splitlines() class set_version(core.Command): """Set python version to our __version__.""" @@ -110,6 +117,7 @@ core.setup( download_url="http://distfiles.gentoo.org/distfiles/mirrorselect-%s.tar.gz"; % __version__, packages=["mirrorselect"], + install_requires=install_requires, # package_data = test_data, scripts=(["bin/mirrorselect"]), data_files=(
