commit: 13255057f276b7ec1fe14528490502b5916ed879
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 17 15:29:11 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Sep 20 01:54:09 2015 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=13255057
repoman: Move the primary checks loop to it's own class and file
Only minimal changes were done for this initial move.
The _scan_ebuilds() needs major hacking up into manageable chunks.
Clean out code separation demarcation lines
These lines were originally used to mark places where code was removed.
And replaced with a class instance and/or function call.
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
pym/repoman/main.py | 756 ++-----------------------------------------------
pym/repoman/scanner.py | 715 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 743 insertions(+), 728 deletions(-)
diff --git a/pym/repoman/main.py b/pym/repoman/main.py
index e3d0472..2b2f91d 100755
--- a/pym/repoman/main.py
+++ b/pym/repoman/main.py
@@ -4,7 +4,6 @@
from __future__ import print_function, unicode_literals
-import copy
import errno
import io
import logging
@@ -15,7 +14,6 @@ import sys
import tempfile
import platform
from itertools import chain
-from pprint import pformat
from os import path as osp
if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))),
".portage_not_installed")):
@@ -30,14 +28,12 @@ portage._disable_legacy_globals()
from portage import os
from portage import _encodings
from portage import _unicode_encode
-from _emerge.Package import Package
from _emerge.UserQuery import UserQuery
import portage.checksum
import portage.const
import portage.repository.config
-from portage import cvstree, normalize_path
+from portage import cvstree
from portage import util
-from portage.dep import Atom
from portage.process import find_binary, spawn
from portage.output import (
bold, create_color_func, green, nocolor, red)
@@ -47,40 +43,18 @@ from portage.util import writemsg_level
from portage.package.ebuild.digestgen import digestgen
from repoman.argparser import parse_args
-from repoman.checks.directories.files import FileChecks
-from repoman.checks.ebuilds.checks import run_checks, checks_init
-from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks
-from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks
-from repoman.checks.ebuilds.fetches import FetchChecks
-from repoman.checks.ebuilds.keywords import KeywordChecks
-from repoman.checks.ebuilds.isebuild import IsEbuild
-from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors
-from repoman.checks.ebuilds.manifests import Manifests
-from repoman.check_missingslot import check_missingslot
-from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid
-from repoman.checks.ebuilds.pkgmetadata import PkgMetadata
-from repoman.checks.ebuilds.use_flags import USEFlagChecks
-from repoman.checks.ebuilds.variables.description import DescriptionChecks
-from repoman.checks.ebuilds.variables.eapi import EAPIChecks
-from repoman.checks.ebuilds.variables.license import LicenseChecks
-from repoman.checks.ebuilds.variables.restrict import RestrictChecks
-from repoman.ebuild import Ebuild
+from repoman.checks.ebuilds.checks import checks_init
from repoman.errors import err
from repoman.gpg import gpgsign, need_signature
-from repoman.modules.commit import repochecks
-from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
from repoman.qa_data import (
format_qa_output, format_qa_output_column, qahelp,
- qawarnings, qacats, missingvars,
- suspect_virtual, suspect_rdepend)
-from repoman.qa_tracker import QATracker
-from repoman.repos import RepoSettings, repo_metadata
-from repoman.scan import Changes, scan
+ qawarnings, qacats)
+from repoman.repos import RepoSettings
+from repoman.scanner import Scanner
from repoman._subprocess import repoman_popen, repoman_getstatusoutput
from repoman import utilities
from repoman.vcs.vcs import (
git_supports_gpg_sign, vcs_files_to_cps, VCSSettings)
-from repoman.vcs.vcsstatus import VCSStatus
if sys.hexversion >= 0x3000000:
@@ -90,21 +64,11 @@ util.initialize_logger()
bad = create_color_func("BAD")
-live_eclasses = portage.const.LIVE_ECLASSES
-non_ascii_re = re.compile(r'[^\x00-\x7f]')
-
# A sane umask is needed for files that portage creates.
os.umask(0o22)
-def sort_key(item):
- return item[2].sub_path
-
def repoman_main(argv):
- # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
- # behave incrementally.
- repoman_incrementals = tuple(
- x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
config_root = os.environ.get("PORTAGE_CONFIGROOT")
repoman_settings = portage.config(config_root=config_root,
local_config=False)
@@ -142,30 +106,9 @@ def repoman_main(argv):
repo_settings = RepoSettings(
config_root, portdir, portdir_overlay,
repoman_settings, vcs_settings, options, qawarnings)
-
repoman_settings = repo_settings.repoman_settings
-
portdb = repo_settings.portdb
- if options.echangelog is None and
repo_settings.repo_config.update_changelog:
- options.echangelog = 'y'
-
- if vcs_settings.vcs is None:
- options.echangelog = 'n'
-
- # The --echangelog option causes automatic ChangeLog generation,
- # which invalidates changelog.ebuildadded and changelog.missing
- # checks.
- # Note: Some don't use ChangeLogs in distributed SCMs.
- # It will be generated on server side from scm log,
- # before package moves to the rsync server.
- # This is needed because they try to avoid merge collisions.
- # Gentoo's Council decided to always use the ChangeLog file.
- # TODO: shouldn't this just be switched on the repo, iso the VCS?
- is_echangelog_enabled = options.echangelog in ('y', 'force')
- vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
- check_changelog = not is_echangelog_enabled and
vcs_settings.vcs_is_cvs_or_svn
-
if 'digest' in repoman_settings.features and options.digest != 'n':
options.digest = 'y'
@@ -178,663 +121,16 @@ def repoman_main(argv):
env = os.environ.copy()
env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
- categories = []
- for path in repo_settings.repo_config.eclass_db.porttrees:
- categories.extend(portage.util.grabfile(
- os.path.join(path, 'profiles', 'categories')))
- repoman_settings.categories = frozenset(
- portage.util.stack_lists([categories], incremental=1))
- categories = repoman_settings.categories
-
- portdb.settings = repoman_settings
- # We really only need to cache the metadata that's necessary for
visibility
- # filtering. Anything else can be discarded to reduce memory
consumption.
- portdb._aux_cache_keys.clear()
- portdb._aux_cache_keys.update(
- ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
-
- reposplit = myreporoot.split(os.path.sep)
- repolevel = len(reposplit)
-
- ###################
- commitmessage = None
- if options.mode == 'commit':
- repochecks.commit_check(repolevel, reposplit)
- repochecks.conflict_check(vcs_settings, options)
-
- ###################
-
- # Make startdir relative to the canonical repodir, so that we can pass
- # it to digestgen and it won't have to be canonicalized again.
- if repolevel == 1:
- startdir = repo_settings.repodir
- else:
- startdir = normalize_path(mydir)
- startdir = os.path.join(
- repo_settings.repodir, *startdir.split(os.sep)[-2 -
repolevel + 3:])
- ###################
-
-
- # get lists of valid keywords, licenses, and use
- new_data = repo_metadata(repo_settings.portdb, repoman_settings)
- kwlist, liclist, uselist, profile_list, \
- global_pmaskdict, liclist_deprecated = new_data
-
- repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
- repoman_settings.backup_changes('PORTAGE_ARCHLIST')
-
- ####################
-
- profiles = setup_profile(profile_list)
-
- ####################
-
- check_profiles(profiles, repoman_settings.archlist())
-
- ####################
-
- scanlist = scan(repolevel, reposplit, startdir, categories,
repo_settings)
-
- ####################
-
- dev_keywords = dev_profile_keywords(profiles)
-
- qatracker = QATracker()
-
-
- if options.mode == "manifest":
- pass
- elif options.pretend:
- print(green("\nRepoMan does a once-over of the
neighborhood..."))
- else:
- print(green("\nRepoMan scours the neighborhood..."))
-
- #####################
-
- changed = Changes(options)
- changed.scan(vcs_settings)
-
- ######################
-
- have_pmasked = False
- have_dev_keywords = False
- dofail = 0
-
- # NOTE: match-all caches are not shared due to potential
- # differences between profiles in _get_implicit_iuse.
- arch_caches = {}
- arch_xmatch_caches = {}
- shared_xmatch_caches = {"cp-list": {}}
-
- include_arches = None
- if options.include_arches:
- include_arches = set()
- include_arches.update(*[x.split() for x in
options.include_arches])
-
- # Disable the "ebuild.notadded" check when not in commit mode and
- # running `svn status` in every package dir will be too expensive.
-
- check_ebuild_notadded = not \
- (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode
!= "commit")
-
- effective_scanlist = scanlist
- if options.if_modified == "y":
- effective_scanlist = sorted(vcs_files_to_cps(
- chain(changed.changed, changed.new, changed.removed),
- repolevel, reposplit, categories))
-
- ######################
- # initialize our checks classes here before the big xpkg loop
- manifester = Manifests(options, qatracker, repoman_settings)
- is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
- filescheck = FileChecks(
- qatracker, repoman_settings, repo_settings, portdb,
vcs_settings)
- status_check = VCSStatus(vcs_settings, qatracker)
- fetchcheck = FetchChecks(
- qatracker, repoman_settings, repo_settings, portdb,
vcs_settings)
- pkgmeta = PkgMetadata(options, qatracker, repoman_settings)
- thirdparty = ThirdPartyMirrors(repoman_settings, qatracker)
- use_flag_checks = USEFlagChecks(qatracker, uselist)
- keywordcheck = KeywordChecks(qatracker, options)
- liveeclasscheck = LiveEclassChecks(qatracker)
- rubyeclasscheck = RubyEclassChecks(qatracker)
- eapicheck = EAPIChecks(qatracker, repo_settings)
- descriptioncheck = DescriptionChecks(qatracker)
- licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated)
- restrictcheck = RestrictChecks(qatracker)
- ######################
-
- for xpkg in effective_scanlist:
- # ebuilds and digests added to cvs respectively.
- logging.info("checking package %s" % xpkg)
- # save memory by discarding xmatch caches from previous
package(s)
- arch_xmatch_caches.clear()
- eadded = []
- catdir, pkgdir = xpkg.split("/")
- checkdir = repo_settings.repodir + "/" + xpkg
- checkdir_relative = ""
- if repolevel < 3:
- checkdir_relative = os.path.join(pkgdir,
checkdir_relative)
- if repolevel < 2:
- checkdir_relative = os.path.join(catdir,
checkdir_relative)
- checkdir_relative = os.path.join(".", checkdir_relative)
-
- #####################
- if manifester.run(checkdir, portdb):
- continue
- if not manifester.generated_manifest:
- manifester.digest_check(xpkg, checkdir)
- ######################
-
- if options.mode == 'manifest-check':
- continue
-
- checkdirlist = os.listdir(checkdir)
-
- ######################
- pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg)
- if is_ebuild.continue_:
- # If we can't access all the metadata then it's totally
unsafe to
- # commit since there's no way to generate a correct
Manifest.
- # Do not try to do any more QA checks on this package
since missing
- # metadata leads to false positives for several checks,
and false
- # positives confuse users.
- can_force = False
- continue
- ######################
-
- keywordcheck.prepare()
-
- # Sort ebuilds in ascending order for the KEYWORDS.dropped
check.
- ebuildlist = sorted(pkgs.values())
- ebuildlist = [pkg.pf for pkg in ebuildlist]
- #######################
- filescheck.check(
- checkdir, checkdirlist, checkdir_relative,
changed.changed, changed.new)
- #######################
- status_check.check(check_ebuild_notadded, checkdir,
checkdir_relative, xpkg)
- eadded.extend(status_check.eadded)
-
- #################
- fetchcheck.check(
- xpkg, checkdir, checkdir_relative, changed.changed,
changed.new)
- #################
-
- if check_changelog and "ChangeLog" not in checkdirlist:
- qatracker.add_error("changelog.missing", xpkg +
"/ChangeLog")
- #################
- pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel)
- muselist = frozenset(pkgmeta.musedict)
- #################
-
- changelog_path = os.path.join(checkdir_relative, "ChangeLog")
- changelog_modified = changelog_path in changed.changelogs
-
- # detect unused local USE-descriptions
- used_useflags = set()
-
- for y_ebuild in ebuildlist:
- ##################
- ebuild = Ebuild(
- repo_settings, repolevel, pkgdir, catdir,
vcs_settings,
- xpkg, y_ebuild)
- ##################
-
- if check_changelog and not changelog_modified \
- and ebuild.ebuild_path in changed.new_ebuilds:
- qatracker.add_error('changelog.ebuildadded',
ebuild.relative_path)
-
- if ebuild.untracked(check_ebuild_notadded, y_ebuild,
eadded):
- # ebuild not added to vcs
- qatracker.add_error(
- "ebuild.notadded", xpkg + "/" +
y_ebuild + ".ebuild")
-
- ##################
- if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker):
- continue
- ###################
- pkg = pkgs[y_ebuild]
- if pkg_invalid(pkg, qatracker, ebuild):
- allvalid = False
- continue
-
- myaux = pkg._metadata
- eapi = myaux["EAPI"]
- inherited = pkg.inherited
- live_ebuild = live_eclasses.intersection(inherited)
-
- #######################
- eapicheck.check(pkg, ebuild)
- #######################
-
- for k, v in myaux.items():
- if not isinstance(v, basestring):
- continue
- m = non_ascii_re.search(v)
- if m is not None:
- qatracker.add_error(
- "variable.invalidchar",
- "%s: %s variable contains
non-ASCII "
- "character at position %s" %
- (ebuild.relative_path, k,
m.start() + 1))
-
- if not fetchcheck.src_uri_error:
- #######################
- thirdparty.check(myaux, ebuild.relative_path)
- #######################
- if myaux.get("PROVIDE"):
- qatracker.add_error("virtual.oldstyle",
ebuild.relative_path)
-
- for pos, missing_var in enumerate(missingvars):
- if not myaux.get(missing_var):
- if catdir == "virtual" and \
- missing_var in ("HOMEPAGE",
"LICENSE"):
- continue
- if live_ebuild and missing_var ==
"KEYWORDS":
- continue
- myqakey = missingvars[pos] + ".missing"
- qatracker.add_error(myqakey, xpkg + "/"
+ y_ebuild + ".ebuild")
-
- if catdir == "virtual":
- for var in ("HOMEPAGE", "LICENSE"):
- if myaux.get(var):
- myqakey = var + ".virtual"
- qatracker.add_error(myqakey,
ebuild.relative_path)
-
- #######################
- descriptioncheck.check(pkg, ebuild)
- #######################
-
- keywords = myaux["KEYWORDS"].split()
-
- ebuild_archs = set(
- kw.lstrip("~") for kw in keywords if not
kw.startswith("-"))
-
- #######################
- keywordcheck.check(
- pkg, xpkg, ebuild, y_ebuild, keywords,
ebuild_archs, changed,
- live_ebuild, kwlist, profiles)
- #######################
-
- if live_ebuild and repo_settings.repo_config.name ==
"gentoo":
- #######################
- liveeclasscheck.check(
- pkg, xpkg, ebuild, y_ebuild, keywords,
global_pmaskdict)
- #######################
-
- if options.ignore_arches:
- arches = [[
- repoman_settings["ARCH"],
repoman_settings["ARCH"],
-
repoman_settings["ACCEPT_KEYWORDS"].split()]]
- else:
- arches = set()
- for keyword in keywords:
- if keyword[0] == "-":
- continue
- elif keyword[0] == "~":
- arch = keyword[1:]
- if arch == "*":
- for expanded_arch in
profiles:
- if
expanded_arch == "**":
- continue
- arches.add(
-
(keyword, expanded_arch, (
-
expanded_arch, "~" + expanded_arch)))
- else:
- arches.add((keyword,
arch, (arch, keyword)))
- else:
- if keyword == "*":
- for expanded_arch in
profiles:
- if
expanded_arch == "**":
- continue
- arches.add(
-
(keyword, expanded_arch, (expanded_arch,)))
- else:
- arches.add((keyword,
keyword, (keyword,)))
- if not arches:
- # Use an empty profile for checking
dependencies of
- # packages that have empty KEYWORDS.
- arches.add(('**', '**', ('**',)))
-
- unknown_pkgs = set()
- baddepsyntax = False
- badlicsyntax = False
- badprovsyntax = False
- # catpkg = catdir + "/" + y_ebuild
-
- inherited_java_eclass = "java-pkg-2" in inherited or \
- "java-pkg-opt-2" in inherited
- inherited_wxwidgets_eclass = "wxwidgets" in inherited
- # operator_tokens = set(["||", "(", ")"])
- type_list, badsyntax = [], []
- for mytype in Package._dep_keys + ("LICENSE",
"PROPERTIES", "PROVIDE"):
- mydepstr = myaux[mytype]
-
- buildtime = mytype in Package._buildtime_keys
- runtime = mytype in Package._runtime_keys
- token_class = None
- if mytype.endswith("DEPEND"):
- token_class = portage.dep.Atom
+ # Perform the main checks
+ scanner = Scanner(repo_settings, myreporoot, config_root, options,
+ vcs_settings, mydir, env)
+ qatracker, can_force = scanner.scan_pkgs(can_force)
- try:
- atoms = portage.dep.use_reduce(
- mydepstr, matchall=1, flat=True,
-
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
- except portage.exception.InvalidDependString as
e:
- atoms = None
- badsyntax.append(str(e))
-
- if atoms and mytype.endswith("DEPEND"):
- if runtime and \
- "test?" in mydepstr.split():
- qatracker.add_error(
- mytype + '.suspect',
- "%s: 'test?' USE
conditional in %s" %
- (ebuild.relative_path,
mytype))
-
- for atom in atoms:
- if atom == "||":
- continue
-
- is_blocker = atom.blocker
-
- # Skip dependency.unknown for
blockers, so that we
- # don't encourage people to
remove necessary blockers,
- # as discussed in bug 382407.
We use atom.without_use
- # due to bug 525376.
- if not is_blocker and \
- not
portdb.xmatch("match-all", atom.without_use) and \
- not
atom.cp.startswith("virtual/"):
-
unknown_pkgs.add((mytype, atom.unevaluated_atom))
-
- if catdir != "virtual":
- if not is_blocker and \
- atom.cp in
suspect_virtual:
-
qatracker.add_error(
-
'virtual.suspect', ebuild.relative_path +
- ": %s:
consider using '%s' instead of '%s'" %
-
(mytype, suspect_virtual[atom.cp], atom))
- if not is_blocker and \
-
atom.cp.startswith("perl-core/"):
-
qatracker.add_error('dependency.perlcore',
-
ebuild.relative_path +
- ": %s:
please use '%s' instead of '%s'" %
- (mytype,
-
atom.replace("perl-core/","virtual/perl-"),
- atom))
-
- if buildtime and \
- not is_blocker and \
- not
inherited_java_eclass and \
- atom.cp ==
"virtual/jdk":
- qatracker.add_error(
-
'java.eclassesnotused', ebuild.relative_path)
- elif buildtime and \
- not is_blocker and \
- not
inherited_wxwidgets_eclass and \
- atom.cp ==
"x11-libs/wxGTK":
- qatracker.add_error(
-
'wxwidgets.eclassnotused',
- "%s: %ss on
x11-libs/wxGTK without inheriting"
- "
wxwidgets.eclass" % (ebuild.relative_path, mytype))
- elif runtime:
- if not is_blocker and \
- atom.cp in
suspect_rdepend:
-
qatracker.add_error(
- mytype
+ '.suspect',
-
ebuild.relative_path + ": '%s'" % atom)
-
- if atom.operator == "~" and \
-
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
- qacat =
'dependency.badtilde'
- qatracker.add_error(
- qacat, "%s: %s
uses the ~ operator"
- " with a
non-zero revision: '%s'" %
-
(ebuild.relative_path, mytype, atom))
-
- check_missingslot(atom, mytype,
eapi, portdb, qatracker,
- ebuild.relative_path,
myaux)
-
- type_list.extend([mytype] * (len(badsyntax) -
len(type_list)))
-
- for m, b in zip(type_list, badsyntax):
- if m.endswith("DEPEND"):
- qacat = "dependency.syntax"
- else:
- qacat = m + ".syntax"
- qatracker.add_error(
- qacat, "%s: %s: %s" %
(ebuild.relative_path, m, b))
-
- badlicsyntax = len([z for z in type_list if z ==
"LICENSE"])
- badprovsyntax = len([z for z in type_list if z ==
"PROVIDE"])
- baddepsyntax = len(type_list) != badlicsyntax +
badprovsyntax
- badlicsyntax = badlicsyntax > 0
- badprovsyntax = badprovsyntax > 0
-
- #################
- use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild,
muselist)
-
- ebuild_used_useflags = use_flag_checks.getUsedUseFlags()
- used_useflags =
used_useflags.union(ebuild_used_useflags)
- #################
- rubyeclasscheck.check(pkg, ebuild)
- #################
-
- # license checks
- if not badlicsyntax:
- #################
- licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
- #################
-
- #################
- restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
- #################
-
- # Syntax Checks
-
- if not vcs_settings.vcs_preserves_mtime:
- if ebuild.ebuild_path not in
changed.new_ebuilds and \
- ebuild.ebuild_path not in
changed.ebuilds:
- pkg.mtime = None
- try:
- # All ebuilds should have utf_8 encoding.
- f = io.open(
- _unicode_encode(
- ebuild.full_path,
encoding=_encodings['fs'], errors='strict'),
- mode='r',
encoding=_encodings['repo.content'])
- try:
- for check_name, e in run_checks(f, pkg):
- qatracker.add_error(
- check_name,
ebuild.relative_path + ': %s' % e)
- finally:
- f.close()
- except UnicodeDecodeError:
- # A file.UTF8 failure will have already been
recorded above.
- pass
-
- if options.force:
- # The dep_check() calls are the most expensive
QA test. If --force
- # is enabled, there's no point in wasting time
on these since the
- # user is intent on forcing the commit anyway.
- continue
-
- relevant_profiles = []
- for keyword, arch, groups in arches:
- if arch not in profiles:
- # A missing profile will create an
error further down
- # during the KEYWORDS verification.
- continue
-
- if include_arches is not None:
- if arch not in include_arches:
- continue
-
- relevant_profiles.extend(
- (keyword, groups, prof) for prof in
profiles[arch])
-
- relevant_profiles.sort(key=sort_key)
-
- for keyword, groups, prof in relevant_profiles:
-
- is_stable_profile = prof.status == "stable"
- is_dev_profile = prof.status == "dev" and \
- options.include_dev
- is_exp_profile = prof.status == "exp" and \
- options.include_exp_profiles == 'y'
- if not (is_stable_profile or is_dev_profile or
is_exp_profile):
- continue
+ commitmessage = None
- dep_settings = arch_caches.get(prof.sub_path)
- if dep_settings is None:
- dep_settings = portage.config(
-
config_profile_path=prof.abs_path,
-
config_incrementals=repoman_incrementals,
- config_root=config_root,
- local_config=False,
-
_unmatched_removal=options.unmatched_removal,
- env=env,
repositories=repoman_settings.repositories)
- dep_settings.categories =
repoman_settings.categories
- if options.without_mask:
- dep_settings._mask_manager_obj
= \
-
copy.deepcopy(dep_settings._mask_manager)
-
dep_settings._mask_manager._pmaskdict.clear()
- arch_caches[prof.sub_path] =
dep_settings
-
- xmatch_cache_key = (prof.sub_path,
tuple(groups))
- xcache =
arch_xmatch_caches.get(xmatch_cache_key)
- if xcache is None:
- portdb.melt()
- portdb.freeze()
- xcache = portdb.xcache
- xcache.update(shared_xmatch_caches)
- arch_xmatch_caches[xmatch_cache_key] =
xcache
-
-
repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings
- portdb.settings = dep_settings
- portdb.xcache = xcache
-
- dep_settings["ACCEPT_KEYWORDS"] = "
".join(groups)
- # just in case, prevent config.reset() from
nuking these.
- dep_settings.backup_changes("ACCEPT_KEYWORDS")
-
- # This attribute is used in dbapi._match_use()
to apply
- # use.stable.{mask,force} settings based on the
stable
- # status of the parent package. This is
required in order
- # for USE deps of unstable packages to be
resolved correctly,
- # since otherwise use.stable.{mask,force}
settings of
- # dependencies may conflict (see bug #456342).
- dep_settings._parent_stable =
dep_settings._isStable(pkg)
-
- # Handle package.use*.{force,mask) calculation,
for use
- # in dep_check.
- dep_settings.useforce =
dep_settings._use_manager.getUseForce(
- pkg, stable=dep_settings._parent_stable)
- dep_settings.usemask =
dep_settings._use_manager.getUseMask(
- pkg, stable=dep_settings._parent_stable)
-
- if not baddepsyntax:
- ismasked = not ebuild_archs or \
- pkg.cpv not in
portdb.xmatch("match-visible",
- Atom("%s::%s" % (pkg.cp,
repo_settings.repo_config.name)))
- if ismasked:
- if not have_pmasked:
- have_pmasked =
bool(dep_settings._getMaskAtom(
- pkg.cpv,
pkg._metadata))
- if options.ignore_masked:
- continue
- # we are testing deps for a
masked package; give it some lee-way
- suffix = "masked"
- matchmode = "minimum-all"
- else:
- suffix = ""
- matchmode = "minimum-visible"
-
- if not have_dev_keywords:
- have_dev_keywords = \
-
bool(dev_keywords.intersection(keywords))
-
- if prof.status == "dev":
- suffix = suffix + "indev"
-
- for mytype in Package._dep_keys:
-
- mykey = "dependency.bad" +
suffix
- myvalue = myaux[mytype]
- if not myvalue:
- continue
-
- success, atoms =
portage.dep_check(
- myvalue, portdb,
dep_settings,
- use="all",
mode=matchmode, trees=repo_settings.trees)
-
- if success:
- if atoms:
-
- # Don't bother
with dependency.unknown for
- # cases in
which *DEPEND.bad is triggered.
- for atom in
atoms:
- #
dep_check returns all blockers and they
- #
aren't counted for *DEPEND.bad, so we
- #
ignore them here.
- if not
atom.blocker:
-
unknown_pkgs.discard(
-
(mytype, atom.unevaluated_atom))
-
- if not
prof.sub_path:
- #
old-style virtuals currently aren't
- #
resolvable with empty profile, since
- #
'virtuals' mappings are unavailable
- # (it
would be expensive to search
- # for
PROVIDE in all ebuilds)
- atoms =
[
-
atom for atom in atoms if not (
-
atom.cp.startswith('virtual/')
-
and not portdb.cp_list(atom.cp))]
-
- # we have some
unsolvable deps
- # remove !
deps, which always show up as unsatisfiable
- atoms = [
-
str(atom.unevaluated_atom)
- for
atom in atoms if not atom.blocker]
-
- # if we emptied
out our list, continue:
- if not atoms:
- continue
-
qatracker.add_error(mykey,
- "%s:
%s: %s(%s)\n%s"
- %
(ebuild.relative_path, mytype, keyword, prof,
-
pformat(atoms, indent=6)))
- else:
-
qatracker.add_error(mykey,
- "%s: %s:
%s(%s)\n%s"
- %
(ebuild.relative_path, mytype, keyword, prof,
-
pformat(atoms, indent=6)))
-
- if not baddepsyntax and unknown_pkgs:
- type_map = {}
- for mytype, atom in unknown_pkgs:
- type_map.setdefault(mytype,
set()).add(atom)
- for mytype, atoms in type_map.items():
- qatracker.add_error(
- "dependency.unknown", "%s: %s:
%s"
- % (ebuild.relative_path,
mytype, ", ".join(sorted(atoms))))
-
- # check if there are unused local USE-descriptions in
metadata.xml
- # (unless there are any invalids, to avoid noise)
- if allvalid:
- for myflag in muselist.difference(used_useflags):
- qatracker.add_error(
- "metadata.warning",
- "%s/metadata.xml: unused local
USE-description: '%s'"
- % (xpkg, myflag))
-
-
- if options.if_modified == "y" and len(effective_scanlist) < 1:
+ if options.if_modified == "y" and len(scanner.effective_scanlist) < 1:
logging.warning("--if-modified is enabled, but no modified
packages were found!")
- if options.mode == "manifest":
- sys.exit(dofail)
-
# dofail will be true if we have failed in at least one non-warning
category
dofail = 0
# dowarn will be true if we tripped any warnings
@@ -842,6 +138,10 @@ def repoman_main(argv):
# dofull will be true if we should print a "repoman full" informational
message
dofull = options.mode != 'full'
+ # early out for manifest generation
+ if options.mode == "manifest":
+ sys.exit(dofail)
+
for x in qacats:
if x not in qatracker.fails:
continue
@@ -884,9 +184,9 @@ def repoman_main(argv):
suggest_ignore_masked = False
suggest_include_dev = False
- if have_pmasked and not (options.without_mask or options.ignore_masked):
+ if scanner.have['pmasked'] and not (options.without_mask or
options.ignore_masked):
suggest_ignore_masked = True
- if have_dev_keywords and not options.include_dev:
+ if scanner.have['dev_keywords'] and not options.include_dev:
suggest_include_dev = True
if suggest_ignore_masked or suggest_include_dev:
@@ -1164,8 +464,8 @@ def repoman_main(argv):
commitmessagefile = None
if not commitmessage or not commitmessage.strip():
msg_prefix = ""
- if repolevel > 1:
- msg_prefix = "/".join(reposplit[1:]) + ": "
+ if scanner.repolevel > 1:
+ msg_prefix = "/".join(scanner.reposplit[1:]) +
": "
try:
editor = os.environ.get("EDITOR")
@@ -1196,10 +496,10 @@ def repoman_main(argv):
report_options.append("--force")
if options.ignore_arches:
report_options.append("--ignore-arches")
- if include_arches is not None:
+ if scanner.include_arches is not None:
report_options.append(
"--include-arches=\"%s\"" %
- " ".join(sorted(include_arches)))
+ " ".join(sorted(scanner.include_arches)))
if vcs_settings.vcs == "git":
# Use new footer only for git (see bug #438364).
@@ -1238,18 +538,18 @@ def repoman_main(argv):
committer_name =
utilities.get_committer_name(env=repoman_settings)
for x in sorted(vcs_files_to_cps(
chain(myupdates, mymanifests, myremoved),
- repolevel, reposplit, categories)):
+ scanner.repolevel, scanner.reposplit,
scanner.categories)):
catdir, pkgdir = x.split("/")
checkdir = repo_settings.repodir + "/" + x
checkdir_relative = ""
- if repolevel < 3:
+ if scanner.repolevel < 3:
checkdir_relative =
os.path.join(pkgdir, checkdir_relative)
- if repolevel < 2:
+ if scanner.repolevel < 2:
checkdir_relative =
os.path.join(catdir, checkdir_relative)
checkdir_relative = os.path.join(".",
checkdir_relative)
changelog_path =
os.path.join(checkdir_relative, "ChangeLog")
- changelog_modified = changelog_path in
changed.changelogs
+ changelog_modified = changelog_path in
scanner.changed.changelogs
if changelog_modified and options.echangelog !=
'force':
continue
@@ -1459,7 +759,7 @@ def repoman_main(argv):
if modified:
portage.util.write_atomic(x, b''.join(mylines),
mode='wb')
- if repolevel == 1:
+ if scanner.repolevel == 1:
utilities.repoman_sez(
"\"You're rather crazy... "
"doing the entire repository.\"\n")
@@ -1467,7 +767,7 @@ def repoman_main(argv):
if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or
myremoved):
for x in sorted(vcs_files_to_cps(
chain(myupdates, myremoved, mymanifests),
- repolevel, reposplit, categories)):
+ scanner.repolevel, scanner.reposplit,
scanner.categories)):
repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
digestgen(mysettings=repoman_settings,
myportdb=portdb)
@@ -1480,7 +780,7 @@ def repoman_main(argv):
try:
for x in sorted(vcs_files_to_cps(
chain(myupdates, myremoved,
mymanifests),
- repolevel, reposplit, categories)):
+ scanner.repolevel, scanner.reposplit,
scanner.categories)):
repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
manifest_path =
os.path.join(repoman_settings["O"], "Manifest")
if not need_signature(manifest_path):
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
new file mode 100644
index 0000000..44ff33b
--- /dev/null
+++ b/pym/repoman/scanner.py
@@ -0,0 +1,715 @@
+
+import copy
+import io
+import logging
+import re
+import sys
+from itertools import chain
+from pprint import pformat
+
+from _emerge.Package import Package
+
+import portage
+from portage import normalize_path
+from portage import os
+from portage import _encodings
+from portage import _unicode_encode
+from portage.dep import Atom
+from portage.output import green
+from repoman.checks.directories.files import FileChecks
+from repoman.checks.ebuilds.checks import run_checks
+from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks
+from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks
+from repoman.checks.ebuilds.fetches import FetchChecks
+from repoman.checks.ebuilds.keywords import KeywordChecks
+from repoman.checks.ebuilds.isebuild import IsEbuild
+from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors
+from repoman.checks.ebuilds.manifests import Manifests
+from repoman.check_missingslot import check_missingslot
+from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid
+from repoman.checks.ebuilds.pkgmetadata import PkgMetadata
+from repoman.checks.ebuilds.use_flags import USEFlagChecks
+from repoman.checks.ebuilds.variables.description import DescriptionChecks
+from repoman.checks.ebuilds.variables.eapi import EAPIChecks
+from repoman.checks.ebuilds.variables.license import LicenseChecks
+from repoman.checks.ebuilds.variables.restrict import RestrictChecks
+from repoman.ebuild import Ebuild
+from repoman.modules.commit import repochecks
+from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
+from repoman.qa_data import missingvars, suspect_virtual, suspect_rdepend
+from repoman.qa_tracker import QATracker
+from repoman.repos import repo_metadata
+from repoman.scan import Changes, scan
+from repoman.vcs.vcsstatus import VCSStatus
+from repoman.vcs.vcs import vcs_files_to_cps
+
+if sys.hexversion >= 0x3000000:
+ basestring = str
+
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]')
+
+
+def sort_key(item):
+ return item[2].sub_path
+
+
+
+class Scanner(object):
+ '''Primary scan class. Operates all the small Q/A tests and checks'''
+
+ def __init__(self, repo_settings, myreporoot, config_root, options,
+ vcs_settings, mydir, env):
+ '''Class __init__'''
+ self.repo_settings = repo_settings
+ self.config_root = config_root
+ self.options = options
+ self.vcs_settings = vcs_settings
+ self.env = env
+
+ # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
+ # behave incrementally.
+ self.repoman_incrementals = tuple(
+ x for x in portage.const.INCREMENTALS if x !=
'ACCEPT_KEYWORDS')
+
+ self.categories = []
+ for path in self.repo_settings.repo_config.eclass_db.porttrees:
+ self.categories.extend(portage.util.grabfile(
+ os.path.join(path, 'profiles', 'categories')))
+ self.repo_settings.repoman_settings.categories = frozenset(
+ portage.util.stack_lists([self.categories],
incremental=1))
+ self.categories = self.repo_settings.repoman_settings.categories
+
+ self.portdb = repo_settings.portdb
+ self.portdb.settings = self.repo_settings.repoman_settings
+ # We really only need to cache the metadata that's necessary
for visibility
+ # filtering. Anything else can be discarded to reduce memory
consumption.
+ self.portdb._aux_cache_keys.clear()
+ self.portdb._aux_cache_keys.update(
+ ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
+
+ self.reposplit = myreporoot.split(os.path.sep)
+ self.repolevel = len(self.reposplit)
+
+ if self.options.mode == 'commit':
+ repochecks.commit_check(self.repolevel, self.reposplit)
+ repochecks.conflict_check(self.vcs_settings,
self.options)
+
+ # Make startdir relative to the canonical repodir, so that we
can pass
+ # it to digestgen and it won't have to be canonicalized again.
+ if self.repolevel == 1:
+ startdir = self.repo_settings.repodir
+ else:
+ startdir = normalize_path(mydir)
+ startdir = os.path.join(
+ self.repo_settings.repodir,
*startdir.split(os.sep)[-2 - self.repolevel + 3:])
+
+ # get lists of valid keywords, licenses, and use
+ new_data = repo_metadata(self.portdb,
self.repo_settings.repoman_settings)
+ kwlist, liclist, uselist, profile_list, \
+ global_pmaskdict, liclist_deprecated = new_data
+ self.repo_metadata = {
+ 'kwlist': kwlist,
+ 'liclist': liclist,
+ 'uselist': uselist,
+ 'profile_list': profile_list,
+ 'pmaskdict': global_pmaskdict,
+ 'lic_deprecated': liclist_deprecated,
+ }
+
+ self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = '
'.join(sorted(kwlist))
+
self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')
+
+ self.profiles = setup_profile(profile_list)
+
+ check_profiles(self.profiles,
self.repo_settings.repoman_settings.archlist())
+
+ scanlist = scan(self.repolevel, self.reposplit, startdir,
self.categories, self.repo_settings)
+
+ self.dev_keywords = dev_profile_keywords(self.profiles)
+
+ self.qatracker = QATracker()
+
+ if self.options.echangelog is None and
self.repo_settings.repo_config.update_changelog:
+ self.options.echangelog = 'y'
+
+ if self.vcs_settings.vcs is None:
+ self.options.echangelog = 'n'
+
+ self.check = {}
+ # The --echangelog option causes automatic ChangeLog generation,
+ # which invalidates changelog.ebuildadded and changelog.missing
+ # checks.
+ # Note: Some don't use ChangeLogs in distributed SCMs.
+ # It will be generated on server side from scm log,
+ # before package moves to the rsync server.
+ # This is needed because they try to avoid merge collisions.
+ # Gentoo's Council decided to always use the ChangeLog file.
+ # TODO: shouldn't this just be switched on the repo, iso the
VCS?
+ is_echangelog_enabled = self.options.echangelog in ('y',
'force')
+ self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in
('cvs', 'svn')
+ self.check['changelog'] = not is_echangelog_enabled and
self.vcs_settings.vcs_is_cvs_or_svn
+
+ if self.options.mode == "manifest":
+ pass
+ elif self.options.pretend:
+ print(green("\nRepoMan does a once-over of the
neighborhood..."))
+ else:
+ print(green("\nRepoMan scours the neighborhood..."))
+
+ self.changed = Changes(self.options)
+ self.changed.scan(self.vcs_settings)
+
+ self.have = {
+ 'pmasked': False,
+ 'dev_keywords': False,
+ }
+
+ # NOTE: match-all caches are not shared due to potential
+ # differences between profiles in _get_implicit_iuse.
+ self.caches = {
+ 'arch': {},
+ 'arch_xmatch': {},
+ 'shared_xmatch': {"cp-list": {}},
+ }
+
+ self.include_arches = None
+ if self.options.include_arches:
+ self.include_arches = set()
+ self.include_arches.update(*[x.split() for x in
self.options.include_arches])
+
+ # Disable the "ebuild.notadded" check when not in commit mode
and
+ # running `svn status` in every package dir will be too
expensive.
+ self.check['ebuild_notadded'] = not \
+ (self.vcs_settings.vcs == "svn" and self.repolevel < 3
and self.options.mode != "commit")
+
+ self.effective_scanlist = scanlist
+ if self.options.if_modified == "y":
+ self.effective_scanlist = sorted(vcs_files_to_cps(
+ chain(self.changed.changed, self.changed.new,
self.changed.removed),
+ self.repolevel, self.reposplit,
self.categories))
+
+ self.live_eclasses = portage.const.LIVE_ECLASSES
+
+ # initialize our checks classes here before the big xpkg loop
+ self.manifester = Manifests(self.options, self.qatracker,
self.repo_settings.repoman_settings)
+ self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings,
self.repo_settings, self.portdb, self.qatracker)
+ self.filescheck = FileChecks(
+ self.qatracker, self.repo_settings.repoman_settings,
self.repo_settings, self.portdb, self.vcs_settings)
+ self.status_check = VCSStatus(self.vcs_settings, self.qatracker)
+ self.fetchcheck = FetchChecks(
+ self.qatracker, self.repo_settings.repoman_settings,
self.repo_settings, self.portdb, self.vcs_settings)
+ self.pkgmeta = PkgMetadata(self.options, self.qatracker,
self.repo_settings.repoman_settings)
+ self.thirdparty =
ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker)
+ self.use_flag_checks = USEFlagChecks(self.qatracker, uselist)
+ self.keywordcheck = KeywordChecks(self.qatracker, self.options)
+ self.liveeclasscheck = LiveEclassChecks(self.qatracker)
+ self.rubyeclasscheck = RubyEclassChecks(self.qatracker)
+ self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings)
+ self.descriptioncheck = DescriptionChecks(self.qatracker)
+ self.licensecheck = LicenseChecks(self.qatracker, liclist,
liclist_deprecated)
+ self.restrictcheck = RestrictChecks(self.qatracker)
+
+
+ def scan_pkgs(self, can_force):
+ for xpkg in self.effective_scanlist:
+ # ebuilds and digests added to cvs respectively.
+ logging.info("checking package %s" % xpkg)
+ # save memory by discarding xmatch caches from previous
package(s)
+ self.caches['arch_xmatch'].clear()
+ self.eadded = []
+ catdir, pkgdir = xpkg.split("/")
+ checkdir = self.repo_settings.repodir + "/" + xpkg
+ checkdir_relative = ""
+ if self.repolevel < 3:
+ checkdir_relative = os.path.join(pkgdir,
checkdir_relative)
+ if self.repolevel < 2:
+ checkdir_relative = os.path.join(catdir,
checkdir_relative)
+ checkdir_relative = os.path.join(".", checkdir_relative)
+
+ if self.manifester.run(checkdir, self.portdb):
+ continue
+ if not self.manifester.generated_manifest:
+ self.manifester.digest_check(xpkg, checkdir)
+ if self.options.mode == 'manifest-check':
+ continue
+
+ checkdirlist = os.listdir(checkdir)
+
+ self.pkgs, self.allvalid =
self.is_ebuild.check(checkdirlist, checkdir, xpkg)
+ if self.is_ebuild.continue_:
+ # If we can't access all the metadata then it's
totally unsafe to
+ # commit since there's no way to generate a
correct Manifest.
+ # Do not try to do any more QA checks on this
package since missing
+ # metadata leads to false positives for several
checks, and false
+ # positives confuse users.
+ can_force = False
+ continue
+
+ self.keywordcheck.prepare()
+
+ # Sort ebuilds in ascending order for the
KEYWORDS.dropped check.
+ ebuildlist = sorted(self.pkgs.values())
+ ebuildlist = [pkg.pf for pkg in ebuildlist]
+
+ self.filescheck.check(
+ checkdir, checkdirlist, checkdir_relative,
self.changed.changed, self.changed.new)
+
+ self.status_check.check(self.check['ebuild_notadded'],
checkdir, checkdir_relative, xpkg)
+ self.eadded.extend(self.status_check.eadded)
+
+ self.fetchcheck.check(
+ xpkg, checkdir, checkdir_relative,
self.changed.changed, self.changed.new)
+
+ if self.check['changelog'] and "ChangeLog" not in
checkdirlist:
+ self.qatracker.add_error("changelog.missing",
xpkg + "/ChangeLog")
+
+ self.pkgmeta.check(xpkg, checkdir, checkdirlist,
self.repolevel)
+ self.muselist = frozenset(self.pkgmeta.musedict)
+
+ changelog_path = os.path.join(checkdir_relative,
"ChangeLog")
+ self.changelog_modified = changelog_path in
self.changed.changelogs
+
+ self._scan_ebuilds(ebuildlist, xpkg, catdir, pkgdir)
+ return self.qatracker, can_force
+
+
+ def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
+ # detect unused local USE-descriptions
+ used_useflags = set()
+
+ for y_ebuild in ebuildlist:
+
+ ebuild = Ebuild(
+ self.repo_settings, self.repolevel, pkgdir,
catdir, self.vcs_settings,
+ xpkg, y_ebuild)
+
+ if self.check['changelog'] and not
self.changelog_modified \
+ and ebuild.ebuild_path in
self.changed.new_ebuilds:
+
self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)
+
+ if ebuild.untracked(self.check['ebuild_notadded'],
y_ebuild, self.eadded):
+ # ebuild not added to vcs
+ self.qatracker.add_error(
+ "ebuild.notadded", xpkg + "/" +
y_ebuild + ".ebuild")
+
+ if bad_split_check(xpkg, y_ebuild, pkgdir,
self.qatracker):
+ continue
+
+ pkg = self.pkgs[y_ebuild]
+ if pkg_invalid(pkg, self.qatracker, ebuild):
+ self.allvalid = False
+ continue
+
+ myaux = pkg._metadata
+ eapi = myaux["EAPI"]
+ inherited = pkg.inherited
+ live_ebuild = self.live_eclasses.intersection(inherited)
+
+ self.eapicheck.check(pkg, ebuild)
+
+ for k, v in myaux.items():
+ if not isinstance(v, basestring):
+ continue
+ m = NON_ASCII_RE.search(v)
+ if m is not None:
+ self.qatracker.add_error(
+ "variable.invalidchar",
+ "%s: %s variable contains
non-ASCII "
+ "character at position %s" %
+ (ebuild.relative_path, k,
m.start() + 1))
+
+ if not self.fetchcheck.src_uri_error:
+ self.thirdparty.check(myaux,
ebuild.relative_path)
+
+ if myaux.get("PROVIDE"):
+ self.qatracker.add_error("virtual.oldstyle",
ebuild.relative_path)
+
+ for pos, missing_var in enumerate(missingvars):
+ if not myaux.get(missing_var):
+ if catdir == "virtual" and \
+ missing_var in ("HOMEPAGE",
"LICENSE"):
+ continue
+ if live_ebuild and missing_var ==
"KEYWORDS":
+ continue
+ myqakey = missingvars[pos] + ".missing"
+ self.qatracker.add_error(myqakey, xpkg
+ "/" + y_ebuild + ".ebuild")
+
+ if catdir == "virtual":
+ for var in ("HOMEPAGE", "LICENSE"):
+ if myaux.get(var):
+ myqakey = var + ".virtual"
+
self.qatracker.add_error(myqakey, ebuild.relative_path)
+
+ self.descriptioncheck.check(pkg, ebuild)
+
+ keywords = myaux["KEYWORDS"].split()
+
+ ebuild_archs = set(
+ kw.lstrip("~") for kw in keywords if not
kw.startswith("-"))
+
+ self.keywordcheck.check(
+ pkg, xpkg, ebuild, y_ebuild, keywords,
ebuild_archs, self.changed,
+ live_ebuild, self.repo_metadata['kwlist'],
self.profiles)
+
+ if live_ebuild and self.repo_settings.repo_config.name
== "gentoo":
+ self.liveeclasscheck.check(
+ pkg, xpkg, ebuild, y_ebuild, keywords,
self.repo_metadata['pmaskdict'])
+
+ if self.options.ignore_arches:
+ arches = [[
+
self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ARCH"],
+
self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
+ else:
+ arches = set()
+ for keyword in keywords:
+ if keyword[0] == "-":
+ continue
+ elif keyword[0] == "~":
+ arch = keyword[1:]
+ if arch == "*":
+ for expanded_arch in
self.profiles:
+ if
expanded_arch == "**":
+ continue
+ arches.add(
+
(keyword, expanded_arch, (
+
expanded_arch, "~" + expanded_arch)))
+ else:
+ arches.add((keyword,
arch, (arch, keyword)))
+ else:
+ if keyword == "*":
+ for expanded_arch in
self.profiles:
+ if
expanded_arch == "**":
+ continue
+ arches.add(
+
(keyword, expanded_arch, (expanded_arch,)))
+ else:
+ arches.add((keyword,
keyword, (keyword,)))
+ if not arches:
+ # Use an empty profile for checking
dependencies of
+ # packages that have empty KEYWORDS.
+ arches.add(('**', '**', ('**',)))
+
+ unknown_pkgs = set()
+ baddepsyntax = False
+ badlicsyntax = False
+ badprovsyntax = False
+ # catpkg = catdir + "/" + y_ebuild
+
+ inherited_java_eclass = "java-pkg-2" in inherited or \
+ "java-pkg-opt-2" in inherited
+ inherited_wxwidgets_eclass = "wxwidgets" in inherited
+ # operator_tokens = set(["||", "(", ")"])
+ type_list, badsyntax = [], []
+ for mytype in Package._dep_keys + ("LICENSE",
"PROPERTIES", "PROVIDE"):
+ mydepstr = myaux[mytype]
+
+ buildtime = mytype in Package._buildtime_keys
+ runtime = mytype in Package._runtime_keys
+ token_class = None
+ if mytype.endswith("DEPEND"):
+ token_class = portage.dep.Atom
+
+ try:
+ atoms = portage.dep.use_reduce(
+ mydepstr, matchall=1, flat=True,
+
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
+ except portage.exception.InvalidDependString as
e:
+ atoms = None
+ badsyntax.append(str(e))
+
+ if atoms and mytype.endswith("DEPEND"):
+ if runtime and \
+ "test?" in mydepstr.split():
+ self.qatracker.add_error(
+ mytype + '.suspect',
+ "%s: 'test?' USE
conditional in %s" %
+ (ebuild.relative_path,
mytype))
+
+ for atom in atoms:
+ if atom == "||":
+ continue
+
+ is_blocker = atom.blocker
+
+ # Skip dependency.unknown for
blockers, so that we
+ # don't encourage people to
remove necessary blockers,
+ # as discussed in bug 382407.
We use atom.without_use
+ # due to bug 525376.
+ if not is_blocker and \
+ not
self.portdb.xmatch("match-all", atom.without_use) and \
+ not
atom.cp.startswith("virtual/"):
+
unknown_pkgs.add((mytype, atom.unevaluated_atom))
+
+ if catdir != "virtual":
+ if not is_blocker and \
+ atom.cp in
suspect_virtual:
+
self.qatracker.add_error(
+
'virtual.suspect', ebuild.relative_path +
+ ": %s:
consider using '%s' instead of '%s'" %
+
(mytype, suspect_virtual[atom.cp], atom))
+ if not is_blocker and \
+
atom.cp.startswith("perl-core/"):
+
self.qatracker.add_error('dependency.perlcore',
+
ebuild.relative_path +
+ ": %s:
please use '%s' instead of '%s'" %
+ (mytype,
+
atom.replace("perl-core/","virtual/perl-"),
+ atom))
+
+ if buildtime and \
+ not is_blocker and \
+ not
inherited_java_eclass and \
+ atom.cp ==
"virtual/jdk":
+
self.qatracker.add_error(
+
'java.eclassesnotused', ebuild.relative_path)
+ elif buildtime and \
+ not is_blocker and \
+ not
inherited_wxwidgets_eclass and \
+ atom.cp ==
"x11-libs/wxGTK":
+
self.qatracker.add_error(
+
'wxwidgets.eclassnotused',
+ "%s: %ss on
x11-libs/wxGTK without inheriting"
+ "
wxwidgets.eclass" % (ebuild.relative_path, mytype))
+ elif runtime:
+ if not is_blocker and \
+ atom.cp in
suspect_rdepend:
+
self.qatracker.add_error(
+ mytype
+ '.suspect',
+
ebuild.relative_path + ": '%s'" % atom)
+
+ if atom.operator == "~" and \
+
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
+ qacat =
'dependency.badtilde'
+
self.qatracker.add_error(
+ qacat, "%s: %s
uses the ~ operator"
+ " with a
non-zero revision: '%s'" %
+
(ebuild.relative_path, mytype, atom))
+
+ check_missingslot(atom, mytype,
eapi, self.portdb, self.qatracker,
+ ebuild.relative_path,
myaux)
+
+ type_list.extend([mytype] * (len(badsyntax) -
len(type_list)))
+
+ for m, b in zip(type_list, badsyntax):
+ if m.endswith("DEPEND"):
+ qacat = "dependency.syntax"
+ else:
+ qacat = m + ".syntax"
+ self.qatracker.add_error(
+ qacat, "%s: %s: %s" %
(ebuild.relative_path, m, b))
+
+ badlicsyntax = len([z for z in type_list if z ==
"LICENSE"])
+ badprovsyntax = len([z for z in type_list if z ==
"PROVIDE"])
+ baddepsyntax = len(type_list) != badlicsyntax +
badprovsyntax
+ badlicsyntax = badlicsyntax > 0
+ badprovsyntax = badprovsyntax > 0
+
+ self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild,
self.muselist)
+
+ ebuild_used_useflags =
self.use_flag_checks.getUsedUseFlags()
+ used_useflags =
used_useflags.union(ebuild_used_useflags)
+
+ self.rubyeclasscheck.check(pkg, ebuild)
+
+ # license checks
+ if not badlicsyntax:
+ self.licensecheck.check(pkg, xpkg, ebuild,
y_ebuild)
+
+ self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
+
+ # Syntax Checks
+ if not self.vcs_settings.vcs_preserves_mtime:
+ if ebuild.ebuild_path not in
self.changed.new_ebuilds and \
+ ebuild.ebuild_path not in
self.changed.ebuilds:
+ pkg.mtime = None
+ try:
+ # All ebuilds should have utf_8 encoding.
+ f = io.open(
+ _unicode_encode(
+ ebuild.full_path,
encoding=_encodings['fs'], errors='strict'),
+ mode='r',
encoding=_encodings['repo.content'])
+ try:
+ for check_name, e in run_checks(f, pkg):
+ self.qatracker.add_error(
+ check_name,
ebuild.relative_path + ': %s' % e)
+ finally:
+ f.close()
+ except UnicodeDecodeError:
+ # A file.UTF8 failure will have already been
recorded above.
+ pass
+
+ if self.options.force:
+ # The dep_check() calls are the most expensive
QA test. If --force
+ # is enabled, there's no point in wasting time
on these since the
+ # user is intent on forcing the commit anyway.
+ continue
+
+ relevant_profiles = []
+ for keyword, arch, groups in arches:
+ if arch not in self.profiles:
+ # A missing profile will create an
error further down
+ # during the KEYWORDS verification.
+ continue
+
+ if self.include_arches is not None:
+ if arch not in self.include_arches:
+ continue
+
+ relevant_profiles.extend(
+ (keyword, groups, prof) for prof in
self.profiles[arch])
+
+ relevant_profiles.sort(key=sort_key)
+
+ for keyword, groups, prof in relevant_profiles:
+
+ is_stable_profile = prof.status == "stable"
+ is_dev_profile = prof.status == "dev" and \
+ self.options.include_dev
+ is_exp_profile = prof.status == "exp" and \
+ self.options.include_exp_profiles == 'y'
+ if not (is_stable_profile or is_dev_profile or
is_exp_profile):
+ continue
+
+ dep_settings =
self.caches['arch'].get(prof.sub_path)
+ if dep_settings is None:
+ dep_settings = portage.config(
+
config_profile_path=prof.abs_path,
+
config_incrementals=self.repoman_incrementals,
+ config_root=self.config_root,
+ local_config=False,
+
_unmatched_removal=self.options.unmatched_removal,
+ env=self.env,
repositories=self.repo_settings.repoman_settings.repositories)
+ dep_settings.categories =
self.repo_settings.repoman_settings.categories
+ if self.options.without_mask:
+ dep_settings._mask_manager_obj
= \
+
copy.deepcopy(dep_settings._mask_manager)
+
dep_settings._mask_manager._pmaskdict.clear()
+ self.caches['arch'][prof.sub_path] =
dep_settings
+
+ xmatch_cache_key = (prof.sub_path,
tuple(groups))
+ xcache =
self.caches['arch_xmatch'].get(xmatch_cache_key)
+ if xcache is None:
+ self.portdb.melt()
+ self.portdb.freeze()
+ xcache = self.portdb.xcache
+
xcache.update(self.caches['shared_xmatch'])
+
self.caches['arch_xmatch'][xmatch_cache_key] = xcache
+
+
self.repo_settings.trees[self.repo_settings.root]["porttree"].settings =
dep_settings
+ self.portdb.settings = dep_settings
+ self.portdb.xcache = xcache
+
+ dep_settings["ACCEPT_KEYWORDS"] = "
".join(groups)
+ # just in case, prevent config.reset() from
nuking these.
+ dep_settings.backup_changes("ACCEPT_KEYWORDS")
+
+ # This attribute is used in dbapi._match_use()
to apply
+ # use.stable.{mask,force} settings based on the
stable
+ # status of the parent package. This is
required in order
+ # for USE deps of unstable packages to be
resolved correctly,
+ # since otherwise use.stable.{mask,force}
settings of
+ # dependencies may conflict (see bug #456342).
+ dep_settings._parent_stable =
dep_settings._isStable(pkg)
+
+ # Handle package.use*.{force,mask) calculation,
for use
+ # in dep_check.
+ dep_settings.useforce =
dep_settings._use_manager.getUseForce(
+ pkg, stable=dep_settings._parent_stable)
+ dep_settings.usemask =
dep_settings._use_manager.getUseMask(
+ pkg, stable=dep_settings._parent_stable)
+
+ if not baddepsyntax:
+ ismasked = not ebuild_archs or \
+ pkg.cpv not in
self.portdb.xmatch("match-visible",
+ Atom("%s::%s" % (pkg.cp,
self.repo_settings.repo_config.name)))
+ if ismasked:
+ if not self.have['pmasked']:
+ self.have['pmasked'] =
bool(dep_settings._getMaskAtom(
+ pkg.cpv,
pkg._metadata))
+ if self.options.ignore_masked:
+ continue
+ # we are testing deps for a
masked package; give it some lee-way
+ suffix = "masked"
+ matchmode = "minimum-all"
+ else:
+ suffix = ""
+ matchmode = "minimum-visible"
+
+ if not self.have['dev_keywords']:
+ self.have['dev_keywords'] = \
+
bool(self.dev_keywords.intersection(keywords))
+
+ if prof.status == "dev":
+ suffix = suffix + "indev"
+
+ for mytype in Package._dep_keys:
+
+ mykey = "dependency.bad" +
suffix
+ myvalue = myaux[mytype]
+ if not myvalue:
+ continue
+
+ success, atoms =
portage.dep_check(
+ myvalue, self.portdb,
dep_settings,
+ use="all",
mode=matchmode, trees=self.repo_settings.trees)
+
+ if success:
+ if atoms:
+
+ # Don't bother
with dependency.unknown for
+ # cases in
which *DEPEND.bad is triggered.
+ for atom in
atoms:
+ #
dep_check returns all blockers and they
+ #
aren't counted for *DEPEND.bad, so we
+ #
ignore them here.
+ if not
atom.blocker:
+
unknown_pkgs.discard(
+
(mytype, atom.unevaluated_atom))
+
+ if not
prof.sub_path:
+ #
old-style virtuals currently aren't
+ #
resolvable with empty profile, since
+ #
'virtuals' mappings are unavailable
+ # (it
would be expensive to search
+ # for
PROVIDE in all ebuilds)
+ atoms =
[
+
atom for atom in atoms if not (
+
atom.cp.startswith('virtual/')
+
and not self.portdb.cp_list(atom.cp))]
+
+ # we have some
unsolvable deps
+ # remove !
deps, which always show up as unsatisfiable
+ atoms = [
+
str(atom.unevaluated_atom)
+ for
atom in atoms if not atom.blocker]
+
+ # if we emptied
out our list, continue:
+ if not atoms:
+ continue
+
self.qatracker.add_error(mykey,
+ "%s:
%s: %s(%s)\n%s"
+ %
(ebuild.relative_path, mytype, keyword, prof,
+
pformat(atoms, indent=6)))
+ else:
+
self.qatracker.add_error(mykey,
+ "%s: %s:
%s(%s)\n%s"
+ %
(ebuild.relative_path, mytype, keyword, prof,
+
pformat(atoms, indent=6)))
+
+ if not baddepsyntax and unknown_pkgs:
+ type_map = {}
+ for mytype, atom in unknown_pkgs:
+ type_map.setdefault(mytype,
set()).add(atom)
+ for mytype, atoms in type_map.items():
+ self.qatracker.add_error(
+ "dependency.unknown", "%s: %s:
%s"
+ % (ebuild.relative_path,
mytype, ", ".join(sorted(atoms))))
+
+ # check if there are unused local USE-descriptions in
metadata.xml
+ # (unless there are any invalids, to avoid noise)
+ if self.allvalid:
+ for myflag in self.muselist.difference(used_useflags):
+ self.qatracker.add_error(
+ "metadata.warning",
+ "%s/metadata.xml: unused local
USE-description: '%s'"
+ % (xpkg, myflag))
