commit: 18e5a8170c69aecd10f162918de571d85055ae81 Author: Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com> AuthorDate: Fri Mar 25 22:06:22 2022 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Sun Mar 27 23:06:48 2022 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=18e5a817
Miscellaneous refactors and cleanups Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com> Closes: https://github.com/gentoo/portage/pull/796 Signed-off-by: Sam James <sam <AT> gentoo.org> lib/_emerge/main.py | 10 ++-- lib/portage/__init__.py | 14 ++--- lib/portage/_global_updates.py | 36 ++++++------- lib/portage/checksum.py | 17 ++---- lib/portage/data.py | 34 +++++------- lib/portage/dispatch_conf.py | 2 +- lib/portage/getbinpkg.py | 115 ++++++++++++++++++----------------------- lib/portage/glsa.py | 4 +- 8 files changed, 91 insertions(+), 141 deletions(-) diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index 01dc1b419..d22da1926 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -1235,12 +1235,10 @@ def emerge_main(args=None): 1: dev_null.fileno(), 2: dev_null.fileno(), } - if ( - portage.process.spawn_bash( - "[[ $(< <(echo foo) ) == foo ]]", fd_pipes=fd_pipes - ) - != 0 - ): + exit_code = portage.process.spawn_bash( + "[[ $(< <(echo foo) ) == foo ]]", fd_pipes=fd_pipes + ) + if exit_code != 0: writemsg_level( "Failed to validate a sane '/dev'.\n" "bash process substitution doesn't work; this may be an " diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py index e6dff28ee..3f1f29f08 100644 --- a/lib/portage/__init__.py +++ b/lib/portage/__init__.py @@ -260,7 +260,6 @@ class _unicode_func_wrapper: self._encoding = encoding def _process_args(self, args, kwargs): - encoding = self._encoding wrapped_args = [ _unicode_encode(x, encoding=encoding, errors="strict") for x in args @@ -276,7 +275,6 @@ class _unicode_func_wrapper: return (wrapped_args, wrapped_kwargs) def __call__(self, *args, **kwargs): - encoding = self._encoding wrapped_args, wrapped_kwargs = self._process_args(args, kwargs) @@ -360,7 +358,6 @@ class _eintr_func_wrapper: self._func = func def __call__(self, *args, **kwargs): - while True: try: rval = self._func(*args, **kwargs) @@ -383,10 +380,7 @@ _os_overrides = { } -try: - _os_overrides[id(_os.mkfifo)] = _os.mkfifo -except AttributeError: - pass # Jython +_os_overrides[id(_os.mkfifo)] = _os.mkfifo if hasattr(_os, "statvfs"): _os_overrides[id(_os.statvfs)] = _os.statvfs @@ -498,9 +492,9 @@ if platform.system() in ("FreeBSD",): def load_mod(name): - modname = ".".join(name.split(".")[:-1]) - mod = __import__(modname) components = name.split(".") + modname = ".".join(components[:-1]) + mod = __import__(modname) for comp in components[1:]: mod = getattr(mod, comp) return mod @@ -573,7 +567,6 @@ def _eapi_is_deprecated(eapi): def eapi_is_supported(eapi): eapi = str(eapi).strip() - return eapi in _supported_eapis @@ -794,7 +787,6 @@ _legacy_global_var_names = ( def _reset_legacy_globals(): - global _legacy_globals_constructed _legacy_globals_constructed = set() for k in _legacy_global_var_names: diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py index d7117d36b..59e0b263e 100644 --- a/lib/portage/_global_updates.py +++ b/lib/portage/_global_updates.py @@ -89,7 +89,7 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): if not update_notice_printed: update_notice_printed = True writemsg_stdout("\n") - writemsg_stdout(colorize("GOOD", _("Performing Global Updates\n"))) + writemsg_stdout(colorize("GOOD", "Performing Global Updates\n")) writemsg_stdout( _( "(Could take a couple of minutes if you have a lot of binary packages.)\n" @@ -236,38 +236,32 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): # until after _all_ of the above updates have # been processed because the mtimedb will # automatically commit when killed by ctrl C. - for mykey, mtime in timestamps.items(): - prev_mtimes[mykey] = mtime + prev_mtimes.update(timestamps) - do_upgrade_packagesmessage = False # We gotta do the brute force updates for these now. - if True: + def onUpdate(_maxval, curval): + if curval > 0: + writemsg_stdout("#") - def onUpdate(_maxval, curval): - if curval > 0: - writemsg_stdout("#") + if quiet: + onUpdate = None - if quiet: - onUpdate = None - vardb.update_ents(repo_map, onUpdate=onUpdate) - if bindb: + vardb.update_ents(repo_map, onUpdate=onUpdate) - def onUpdate(_maxval, curval): - if curval > 0: - writemsg_stdout("*") + if bindb: - if quiet: - onUpdate = None - bindb.update_ents(repo_map, onUpdate=onUpdate) - else: - do_upgrade_packagesmessage = 1 + def onUpdate(_maxval, curval): + if curval > 0: + writemsg_stdout("*") + + bindb.update_ents(repo_map, onUpdate=onUpdate) # Update progress above is indicated by characters written to stdout so # we print a couple new lines here to separate the progress output from # what follows. writemsg_stdout("\n\n") - if do_upgrade_packagesmessage and bindb and bindb.cpv_all(): + if bindb and bindb.cpv_all(): writemsg_stdout( _( " ** Skipping packages. Run 'fixpackages' or set it in FEATURES to fix the tbz2's in the packages directory.\n" diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py index 72d20525a..b4644da04 100644 --- a/lib/portage/checksum.py +++ b/lib/portage/checksum.py @@ -344,11 +344,8 @@ if os.path.exists(PRELINK_BINARY): def is_prelinkable_elf(filename): - f = _open_file(filename) - try: + with _open_file(filename) as f: magic = f.read(17) - finally: - f.close() return ( len(magic) == 17 and magic.startswith(b"\x7fELF") @@ -444,11 +441,11 @@ def _apply_hash_filter(digests, hash_filter): @type hash_filter: callable """ - verifiable_hash_types = set(digests).intersection(hashfunc_keys) - verifiable_hash_types.discard("size") + verifiable_hash_types = frozenset(digests).intersection(hashfunc_keys) modified = False if len(verifiable_hash_types) > 1: - for k in list(verifiable_hash_types): + verifiable_hash_types.discard("size") + for k in verifiable_hash_types: if not hash_filter(k): modified = True verifiable_hash_types.remove(k) @@ -456,11 +453,7 @@ def _apply_hash_filter(digests, hash_filter): break if modified: - digests = { - k: v - for (k, v) in digests.items() - if k == "size" or k in verifiable_hash_types - } + digests = {k: v for k, v in digests.items() if k in verifiable_hash_types} return digests diff --git a/lib/portage/data.py b/lib/portage/data.py index b73fa8882..c783d76d0 100644 --- a/lib/portage/data.py +++ b/lib/portage/data.py @@ -8,6 +8,7 @@ import platform import pwd import portage +from portage.localization import _ portage.proxy.lazyimport.lazyimport( globals(), @@ -16,14 +17,11 @@ portage.proxy.lazyimport.lazyimport( "portage.util.path:first_existing", "subprocess", ) -from portage.localization import _ ostype = platform.system() -userland = None +userland = "GNU" if ostype == "DragonFly" or ostype.endswith("BSD"): userland = "BSD" -else: - userland = "GNU" lchown = getattr(os, "lchown", None) @@ -140,7 +138,6 @@ def _get_global(k): return globals()[k] if k == "secpass": - unprivileged = False if hasattr(portage, "settings"): unprivileged = "unprivileged" in portage.settings.features @@ -254,11 +251,10 @@ def _get_global(k): # Avoid instantiating portage.settings when the desired # variable is set in os.environ. elif k in ("_portage_grpname", "_portage_username"): - v = None + v = "portage" + env_key = "PORTAGE_USERNAME" if k == "_portage_grpname": env_key = "PORTAGE_GRPNAME" - else: - env_key = "PORTAGE_USERNAME" if env_key in os.environ: v = os.environ[env_key] @@ -290,9 +286,6 @@ def _get_global(k): pass else: v = pwd_struct.pw_name - - if v is None: - v = "portage" else: raise AssertionError("unknown name: %s" % k) @@ -302,7 +295,6 @@ def _get_global(k): class _GlobalProxy(portage.proxy.objectproxy.ObjectProxy): - __slots__ = ("_name",) def __init__(self, name): @@ -340,17 +332,15 @@ def _init(settings): # from grp.getgrnam() with PyPy native_string = platform.python_implementation() == "PyPy" - v = settings.get("PORTAGE_GRPNAME", "portage") - if native_string: - v = portage._native_string(v) - globals()["_portage_grpname"] = v - _initialized_globals.add("_portage_grpname") - - v = settings.get("PORTAGE_USERNAME", "portage") if native_string: - v = portage._native_string(v) - globals()["_portage_username"] = v - _initialized_globals.add("_portage_username") + grpname = settings.get("PORTAGE_GRPNAME", "portage") + grpname = portage._native_string(grpname) + globals()["_portage_grpname"] = grpname + _initialized_globals.add("_portage_grpname") + username = settings.get("PORTAGE_USERNAME", "portage") + username = portage._native_string(username) + globals()["_portage_username"] = username + _initialized_globals.add("_portage_username") if "secpass" not in _initialized_globals: v = 0 diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py index d682a9ad0..6dbb1885e 100644 --- a/lib/portage/dispatch_conf.py +++ b/lib/portage/dispatch_conf.py @@ -40,7 +40,7 @@ def diffstatusoutput(cmd, file1, file2): args = (portage._unicode_encode(x, errors="strict") for x in args) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) - if output and output[-1] == "\n": + if output and output.endswith("\n"): # getstatusoutput strips one newline output = output[:-1] return (proc.wait(), output) diff --git a/lib/portage/getbinpkg.py b/lib/portage/getbinpkg.py index f357a91df..ea9ee1d0a 100644 --- a/lib/portage/getbinpkg.py +++ b/lib/portage/getbinpkg.py @@ -21,11 +21,11 @@ import tempfile import base64 import warnings -_all_errors = [NotImplementedError, ValueError, socket.error] - from html.parser import HTMLParser as html_parser_HTMLParser from urllib.parse import unquote as urllib_parse_unquote +_all_errors = [NotImplementedError, ValueError, socket.error] + try: import ftplib except ImportError as e: @@ -47,7 +47,6 @@ _all_errors = tuple(_all_errors) def make_metadata_dict(data): - warnings.warn( "portage.getbinpkg.make_metadata_dict() is deprecated", DeprecationWarning, @@ -82,7 +81,6 @@ class ParseLinks(html_parser_HTMLParser): page and provide suffix and prefix limitors""" def __init__(self): - warnings.warn( "portage.getbinpkg.ParseLinks is deprecated", DeprecationWarning, @@ -96,19 +94,15 @@ class ParseLinks(html_parser_HTMLParser): return self.PL_anchors def get_anchors_by_prefix(self, prefix): - newlist = [] - for x in self.PL_anchors: - if x.startswith(prefix): - if x not in newlist: - newlist.append(x[:]) + newlist = [ + x for x in self.PL_anchors if x.startswith(prefix) and x not in newlist + ] return newlist def get_anchors_by_suffix(self, suffix): - newlist = [] - for x in self.PL_anchors: - if x.endswith(suffix): - if x not in newlist: - newlist.append(x[:]) + newlist = [ + x for x in self.PL_anchors if x.endswith(suffix) and x not in newlist + ] return newlist def handle_endtag(self, tag): @@ -140,16 +134,13 @@ def create_conn(baseurl, conn=None): raise ValueError( _("Provided URI does not " "contain protocol identifier. '%s'") % baseurl ) - protocol, url_parts = parts + protocol, url = parts del parts - url_parts = url_parts.split("/") - host = url_parts[0] - if len(url_parts) < 2: - address = "/" - else: - address = "/" + "/".join(url_parts[1:]) - del url_parts + url_split = url.split("/", 1) + host = url_split[0] + address = f"/{url_split[1]}" + del url, url_split userpass_host = host.split("@", 1) if len(userpass_host) == 1: @@ -160,13 +151,14 @@ def create_conn(baseurl, conn=None): userpass = userpass_host[0].split(":") del userpass_host - if len(userpass) > 2: + userpass_size = len(userpass) + if userpass_size > 2: raise ValueError(_("Unable to interpret username/password provided.")) - elif len(userpass) == 2: - username = userpass[0] + + username = userpass[0] + if userpass_size == 2: password = userpass[1] - elif len(userpass) == 1: - username = userpass[0] + elif userpass_size == 1: password = None del userpass @@ -243,7 +235,6 @@ def make_ftp_request(conn, address, rest=None, dest=None): ) try: - if dest: fstart_pos = dest.tell() @@ -268,10 +259,9 @@ def make_ftp_request(conn, address, rest=None, dest=None): else: break + data_size = len(mydata) if dest: data_size = fstart_pos - dest.tell() - else: - data_size = len(mydata) mysocket.close() conn.voidresp() @@ -296,7 +286,7 @@ def make_http_request(conn, address, _params={}, headers={}, dest=None): rc = 0 response = None - while (rc == 0) or (rc == 301) or (rc == 302): + while rc in (0, 301, 302): try: if rc != 0: conn = create_conn(address)[0] @@ -309,9 +299,9 @@ def make_http_request(conn, address, _params={}, headers={}, dest=None): rc = response.status # 301 means that the page address is wrong. - if (rc == 301) or (rc == 302): - ignored_data = response.read() - del ignored_data + if rc in (301, 302): + # This response reading is ignored on purpose. + _ = response.read() for x in str(response.msg).split("\n"): parts = x.split(": ", 1) if parts[0] == "Location": @@ -326,7 +316,7 @@ def make_http_request(conn, address, _params={}, headers={}, dest=None): address = parts[1] break - if (rc != 200) and (rc != 206): + if rc not in (200, 206): return ( None, rc, @@ -355,26 +345,28 @@ def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0): for x in array: add_p = 0 - if prefix and (len(x) >= len(prefix)) and (x[: len(prefix)] == prefix): + x_size = len(x) + prefix_size = len(prefix) + if prefix and x_size >= prefix_size and x[:prefix_size] == prefix: add_p = 1 if match_both: if prefix and not add_p: # Require both, but don't have first one. continue - else: - if add_p: # Only need one, and we have it. - myarray.append(x[:]) - continue + elif add_p: # Only need one, and we have it. + myarray.append(x[:]) + continue + suffix_size = len(suffix) if not allow_overlap: # Not allow to overlap prefix and suffix - if len(x) >= (len(prefix) + len(suffix)): + if x_size >= (prefix_size + suffix_size): pass else: continue # Too short to match. else: pass # Do whatever... We're overlapping. - if suffix and (len(x) >= len(suffix)) and (x[-len(suffix) :] == suffix): + if suffix and x_size >= suffix_size and x[-len(suffix) :] == suffix: myarray.append(x) # It matches else: continue # Doesn't match. @@ -393,9 +385,8 @@ def dir_get_list(baseurl, conn=None): stacklevel=2, ) - if not conn: - keepconnection = 0 - else: + keepconnection = 0 + if conn: keepconnection = 1 conn, protocol, address, params, headers = create_conn(baseurl, conn) @@ -419,7 +410,7 @@ def dir_get_list(baseurl, conn=None): raise portage.exception.PortageException( _("Unable to get listing: %s %s") % (rc, msg) ) - elif protocol in ["ftp"]: + elif protocol == "ftp": if address[-1] == "/": olddir = conn.pwd() conn.cwd(address) @@ -450,10 +441,9 @@ def file_get_metadata(baseurl, conn=None, chunk_size=3000): stacklevel=2, ) - if not conn: + keepconnection = 1 + if conn: keepconnection = 0 - else: - keepconnection = 1 conn, protocol, address, params, headers = create_conn(baseurl, conn) @@ -532,7 +522,7 @@ def file_get( variables["URI"] = baseurl if "FILE" not in variables: - if filename is None: + if not filename: filename = os.path.basename(variables["URI"]) variables["FILE"] = filename @@ -565,9 +555,8 @@ def file_get_lib(baseurl, dest, conn=None): stacklevel=2, ) - if not conn: - keepconnection = 0 - else: + keepconnection = 0 + if conn: keepconnection = 1 conn, protocol, address, params, headers = create_conn(baseurl, conn) @@ -608,22 +597,20 @@ def file_get_lib(baseurl, dest, conn=None): def dir_get_metadata( baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None ): - warnings.warn( "portage.getbinpkg.dir_get_metadata() is deprecated", DeprecationWarning, stacklevel=2, ) - if not conn: + keepconnection = 1 + if conn: keepconnection = 0 - else: - keepconnection = 1 cache_path = "/var/cache/edb" metadatafilename = os.path.join(cache_path, "remote_metadata.pickle") - if makepickle is None: + if not makepickle: makepickle = "/var/cache/edb/metadata.idx.most_recent" try: @@ -896,7 +883,7 @@ class PackageIndex: ): self._pkg_slot_dict = None - if allowed_pkg_keys is not None: + if allowed_pkg_keys: self._pkg_slot_dict = slot_dict_class(allowed_pkg_keys) self._default_header_data = default_header_data @@ -914,11 +901,9 @@ class PackageIndex: self.modified = True def _readpkgindex(self, pkgfile, pkg_entry=True): - + d = {} allowed_keys = None - if self._pkg_slot_dict is None or not pkg_entry: - d = {} - else: + if self._pkg_slot_dict and pkg_entry: d = self._pkg_slot_dict() allowed_keys = d.allowed_keys @@ -964,7 +949,7 @@ class PackageIndex: if self._inherited_keys: for k in self._inherited_keys: v = self.header.get(k) - if v is not None: + if v: d.setdefault(k, v) self.packages.append(d) @@ -982,7 +967,7 @@ class PackageIndex: if self._inherited_keys: for k in self._inherited_keys: v = self.header.get(k) - if v is not None and v == metadata.get(k): + if v and v == metadata.get(k): del metadata[k] if self._default_pkg_data: for k, v in self._default_pkg_data.items(): diff --git a/lib/portage/glsa.py b/lib/portage/glsa.py index 1b68bc0e9..d61ad7e59 100644 --- a/lib/portage/glsa.py +++ b/lib/portage/glsa.py @@ -115,12 +115,10 @@ def get_glsa_list(myconfig): @rtype: List of Strings @return: a list of GLSA IDs in this repository """ - rValue = [] + repository = os.path.join(myconfig["PORTDIR"], "metadata", "glsa") if "GLSA_DIR" in myconfig: repository = myconfig["GLSA_DIR"] - else: - repository = os.path.join(myconfig["PORTDIR"], "metadata", "glsa") if not os.access(repository, os.R_OK): return []
