commit: 98ca3429f44098c3822b59cb5ffb208dfa1fbc75
Author: Brian Harring <ferringb <AT> gmail <DOT> com>
AuthorDate: Wed Dec 3 16:55:42 2025 +0000
Commit: Brian Harring <ferringb <AT> gmail <DOT> com>
CommitDate: Wed Dec 3 18:52:46 2025 +0000
URL:
https://gitweb.gentoo.org/proj/pkgcore/snakeoil.git/commit/?id=98ca3429
chore(python_namespaces): address that python suffixes can intersect other
python suffixes
Signed-off-by: Brian Harring <ferringb <AT> gmail.com>
src/snakeoil/dist/generate_docs.py | 2 ++
src/snakeoil/python_namespaces.py | 38 ++++++++++++++++++++++++--------------
tests/test_python_namespaces.py | 31 ++++++++++++++++++++++++-------
3 files changed, 50 insertions(+), 21 deletions(-)
diff --git a/src/snakeoil/dist/generate_docs.py
b/src/snakeoil/dist/generate_docs.py
index 9def23e..8534aed 100644
--- a/src/snakeoil/dist/generate_docs.py
+++ b/src/snakeoil/dist/generate_docs.py
@@ -6,6 +6,8 @@ import subprocess
from importlib import import_module
from io import StringIO
+from snakeoil import python_namespaces
+
from ..contexts import syspath
from .generate_man_rsts import ManConverter
diff --git a/src/snakeoil/python_namespaces.py
b/src/snakeoil/python_namespaces.py
index ea21b1c..244a17d 100644
--- a/src/snakeoil/python_namespaces.py
+++ b/src/snakeoil/python_namespaces.py
@@ -1,11 +1,10 @@
__all__ = ("import_submodules_of", "get_submodules_of")
-import importlib
-import importlib.machinery
import os
-import pathlib
import types
import typing
+from importlib import import_module, machinery
+from pathlib import Path
T_class_filter = typing.Callable[[str], bool]
@@ -17,7 +16,7 @@ def get_submodules_of(
ignore_import_failures: T_class_filter | typing.Container[str] | bool =
False,
include_root=False,
) -> typing.Iterable[types.ModuleType]:
- """Visit all submodules of the target via walking the underlying filesystem
+ """Visit all submodules of the target finding modules ending with PEP3147
suffixes.
This currently cannot work against a frozen python exe (for example), nor
source only contained within an egg; it currently just walks the FS.
:param root: the module to trace
@@ -49,7 +48,7 @@ def get_submodules_of(
if current is not root or include_root:
yield current
- base = pathlib.Path(os.path.abspath(current.__file__))
+ base = Path(os.path.abspath(current.__file__))
# if it's not the root of a module, there's nothing to do- return it..
if not base.name.startswith("__init__."):
continue
@@ -62,14 +61,9 @@ def get_submodules_of(
if potential.is_dir():
if name == "__pycache__":
continue
- else:
- for ext in importlib.machinery.all_suffixes():
- if name.endswith(ext):
- name = name[: -len(ext)]
- break
- else:
- # it's not a python source.
- continue
+ elif remove_py_extension(name) is None:
+ # it's not a python source.
+ continue
if dont_import(qualname):
continue
@@ -80,7 +74,7 @@ def get_submodules_of(
# extended for working from .whl or .egg directly, we will
want that
# logic in one spot. TL;DR: this is intentionally not
optimized for the
# common case.
- to_scan.append(importlib.import_module(qualname))
+ to_scan.append(import_module(qualname))
except ImportError as e:
if not ignore_import_failures(qualname):
raise ImportError(f"failed importing {qualname}: {e}")
from e
@@ -93,3 +87,19 @@ def import_submodules_of(target: types.ModuleType, **kwargs)
-> None:
"""
for _ in get_submodules_of(target, **kwargs):
pass
+
+
+def remove_py_extension(path: Path | str) -> str | None:
+ """Return the stem of a path or None, if the extension is PEP3147 (.py,
.pyc, etc)
+
+ This accounts for the fact certain extensions in
importlib.machinery.all_suffixes()
+ intersect each other. This will give you the resultant package name
irregardless of
+ PEP3147 conflicts.
+
+ If it's not a valid extension, None is returned.
+ """
+ name = Path(path).name
+ for ext in sorted(machinery.all_suffixes(), key=lambda x: x.split(".")):
+ if name.endswith(ext):
+ return name[: -len(ext)]
+ return None
diff --git a/tests/test_python_namespaces.py b/tests/test_python_namespaces.py
index 184f3b6..2ad1bdc 100644
--- a/tests/test_python_namespaces.py
+++ b/tests/test_python_namespaces.py
@@ -1,17 +1,18 @@
-import importlib
import pathlib
import sys
from contextlib import contextmanager
+from importlib import import_module, invalidate_caches, machinery
import pytest
from snakeoil.python_namespaces import (
get_submodules_of,
import_submodules_of,
+ remove_py_extension,
)
-class TestNamespaceCollector:
+class test_python_namespaces:
def write_tree(self, base: pathlib.Path, *paths: str | pathlib.Path):
base.mkdir(exist_ok=True)
for path in sorted(paths):
@@ -26,12 +27,12 @@ class TestNamespaceCollector:
modules = sys.modules.copy()
try:
sys.path.append(str(base))
- importlib.invalidate_caches()
+ invalidate_caches()
yield (modules.copy())
finally:
sys.path[:] = python_path
sys.modules = modules
- importlib.invalidate_caches()
+ invalidate_caches()
def test_it(self, tmp_path):
self.write_tree(
@@ -44,7 +45,7 @@ class TestNamespaceCollector:
)
def get_it(target, *args, **kwargs):
- target = importlib.import_module(target)
+ target = import_module(target)
return list(
sorted(x.__name__ for x in get_submodules_of(target, *args,
**kwargs))
)
@@ -80,7 +81,7 @@ class TestNamespaceCollector:
"extra.py",
)
with self.protect_modules(tmp_path):
- assert None is
import_submodules_of(importlib.import_module("_ns_test"))
+ assert None is import_submodules_of(import_module("_ns_test"))
assert set(["_ns_test.blah", "_ns_test.foon", "_ns_test.extra"])
== set(
x for x in sys.modules if x.startswith("_ns_test.")
)
@@ -95,7 +96,7 @@ class TestNamespaceCollector:
f.write("raise ImportError('bad2')")
with self.protect_modules(tmp_path):
- mod = importlib.import_module("_ns_test")
+ mod = import_module("_ns_test")
with pytest.raises(ImportError) as capture:
import_submodules_of(mod,
ignore_import_failures=["_ns_test.bad2"])
assert "bad1" in " ".join(tuple(capture.value.args))
@@ -110,3 +111,19 @@ class TestNamespaceCollector:
assert ["_ns_test.blah"] == [
x.__name__ for x in get_submodules_of(mod,
ignore_import_failures=True)
]
+
+
+def test_remove_py_extension():
+ # no need to mock, python standards intersect.
+ cpy = [x for x in machinery.all_suffixes() if x.startswith(".cpython")]
+ assert cpy, "couldn't find an extension of .cpython per PEP3147. Is this
pypy?"
+ cpy = cpy[0]
+ suffix = f".{cpy.rsplit('.')[-1]}"
+ assert suffix in machinery.all_suffixes() # confirm .so or .dylib is in
there
+ assert "blah" == remove_py_extension(f"blah{cpy}")
+ assert "blah" == remove_py_extension(f"blah{suffix}")
+ assert f"blah{suffix}" == remove_py_extension(f"blah{suffix}.py"), (
+ "the code is double stripping suffixes"
+ )
+ assert None is remove_py_extension("asdf")
+ assert None is remove_py_extension("asdf.txt")
