commit: 2166e17f6f6b7d6863af519f6e988bcf95ac29c6
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 10 17:01:44 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Aug 10 17:01:44 2015 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2166e17f
main from repoman
pym/tbc/repoman/main.py | 2936 ++++++++++++++++++++++++-----------------------
1 file changed, 1476 insertions(+), 1460 deletions(-)
diff --git a/pym/tbc/repoman/main.py b/pym/tbc/repoman/main.py
old mode 100644
new mode 100755
index a30704a..86ac802
--- a/pym/tbc/repoman/main.py
+++ b/pym/tbc/repoman/main.py
@@ -1,3 +1,4 @@
+#!/usr/bin/python -bO
# Copyright 1999-2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
@@ -5,7 +6,6 @@ from __future__ import print_function, unicode_literals
import copy
import errno
-import formatter
import io
import logging
import re
@@ -15,13 +15,14 @@ import sys
import tempfile
import platform
from itertools import chain
+from pprint import pformat
from os import path as osp
if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))),
".portage_not_installed")):
pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))))
#, "pym")
sys.path.insert(0, pym_path)
# import our centrally initialized portage instance
-from tbc.repoman._portage import portage
+from repoman._portage import portage
portage._internal_caller = True
portage._disable_legacy_globals()
@@ -36,47 +37,50 @@ import portage.const
import portage.repository.config
from portage import cvstree, normalize_path
from portage import util
+from portage.dep import Atom
from portage.exception import MissingParameter
from portage.process import find_binary, spawn
from portage.output import (
bold, create_color_func, green, nocolor, red)
from portage.output import ConsoleStyleFile, StyleWriter
+from portage.util import formatter
from portage.util import writemsg_level
from portage.package.ebuild.digestgen import digestgen
-from tbc.repoman.argparser import parse_args
-from tbc.repoman.checks.directories.files import FileChecks
-from tbc.repoman.checks.ebuilds.checks import run_checks, checks_init
-from tbc.repoman.checks.ebuilds.eclasses.live import LiveEclassChecks
-from tbc.repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks
-from tbc.repoman.checks.ebuilds.fetches import FetchChecks
-from tbc.repoman.checks.ebuilds.keywords import KeywordChecks
-from tbc.repoman.checks.ebuilds.isebuild import IsEbuild
-from tbc.repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors
-from tbc.repoman.checks.ebuilds.manifests import Manifests
-from tbc.repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid
-from tbc.repoman.checks.ebuilds.pkgmetadata import PkgMetadata
-from tbc.repoman.checks.ebuilds.use_flags import USEFlagChecks
-from tbc.repoman.checks.ebuilds.variables.description import DescriptionChecks
-from tbc.repoman.checks.ebuilds.variables.eapi import EAPIChecks
-from tbc.repoman.checks.ebuilds.variables.license import LicenseChecks
-from tbc.repoman.checks.ebuilds.variables.restrict import RestrictChecks
-from tbc.repoman.ebuild import Ebuild
-from tbc.repoman.errors import err
-from tbc.repoman.modules.commit import repochecks
-from tbc.repoman.profile import check_profiles, dev_keywords, setup_profile
-from tbc.repoman.qa_data import (
+from repoman.argparser import parse_args
+from repoman.checks.directories.files import FileChecks
+from repoman.checks.ebuilds.checks import run_checks, checks_init
+from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks
+from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks
+from repoman.checks.ebuilds.fetches import FetchChecks
+from repoman.checks.ebuilds.keywords import KeywordChecks
+from repoman.checks.ebuilds.isebuild import IsEbuild
+from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors
+from repoman.checks.ebuilds.manifests import Manifests
+from repoman.check_missingslot import check_missingslot
+from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid
+from repoman.checks.ebuilds.pkgmetadata import PkgMetadata
+from repoman.checks.ebuilds.use_flags import USEFlagChecks
+from repoman.checks.ebuilds.variables.description import DescriptionChecks
+from repoman.checks.ebuilds.variables.eapi import EAPIChecks
+from repoman.checks.ebuilds.variables.license import LicenseChecks
+from repoman.checks.ebuilds.variables.restrict import RestrictChecks
+from repoman.ebuild import Ebuild
+from repoman.errors import err
+from repoman.modules.commit import repochecks
+from repoman.profile import check_profiles, dev_keywords, setup_profile
+from repoman.qa_data import (
format_qa_output, format_qa_output_column, qahelp,
qawarnings, qacats, missingvars,
suspect_virtual, suspect_rdepend)
-from tbc.repoman.qa_tracker import QATracker
-from tbc.repoman.repos import RepoSettings, repo_metadata
-from tbc.repoman.scan import Changes, scan
-from tbc.repoman._subprocess import repoman_popen, repoman_getstatusoutput
-from tbc.repoman import utilities
-from tbc.repoman.vcs.vcs import (
+from repoman.qa_tracker import QATracker
+from repoman.repos import RepoSettings, repo_metadata
+from repoman.scan import Changes, scan
+from repoman._subprocess import repoman_popen, repoman_getstatusoutput
+from repoman import utilities
+from repoman.vcs.vcs import (
git_supports_gpg_sign, vcs_files_to_cps, VCSSettings)
-from tbc.repoman.vcs.vcsstatus import VCSStatus
+from repoman.vcs.vcsstatus import VCSStatus
if sys.hexversion >= 0x3000000:
@@ -93,1540 +97,1552 @@ non_ascii_re = re.compile(r'[^\x00-\x7f]')
# A sane umask is needed for files that portage creates.
os.umask(0o22)
+# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
+# behave incrementally.
+repoman_incrementals = tuple(
+ x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
+config_root = os.environ.get("PORTAGE_CONFIGROOT")
+repoman_settings = portage.config(config_root=config_root, local_config=False)
-def sort_key(item):
- return item[2].sub_path
-
-# Setup the GPG commands
-def gpgsign(filename):
- gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
- if gpgcmd is None:
- raise MissingParameter(
- "PORTAGE_GPG_SIGNING_COMMAND is unset!"
- " Is make.globals missing?")
- if "${PORTAGE_GPG_KEY}" in gpgcmd and \
- "PORTAGE_GPG_KEY" not in repoman_settings:
- raise MissingParameter("PORTAGE_GPG_KEY is unset!")
- if "${PORTAGE_GPG_DIR}" in gpgcmd:
- if "PORTAGE_GPG_DIR" not in repoman_settings:
- repoman_settings["PORTAGE_GPG_DIR"] = \
- os.path.expanduser("~/.gnupg")
- logging.info(
- "Automatically setting PORTAGE_GPG_DIR to '%s'"
%
- repoman_settings["PORTAGE_GPG_DIR"])
- else:
- repoman_settings["PORTAGE_GPG_DIR"] = \
-
os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
- if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
- raise portage.exception.InvalidLocation(
- "Unable to access directory:
PORTAGE_GPG_DIR='%s'" %
- repoman_settings["PORTAGE_GPG_DIR"])
- gpgvars = {"FILE": filename}
- for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
- v = repoman_settings.get(k)
- if v is not None:
- gpgvars[k] = v
- gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
- if options.pretend:
- print("(" + gpgcmd + ")")
- else:
- # Encode unicode manually for bug #310789.
- gpgcmd = portage.util.shlex_split(gpgcmd)
-
- if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000
and \
- not os.path.isabs(gpgcmd[0]):
- # Python 3.1 _execvp throws TypeError for non-absolute
executable
- # path passed as bytes (see
http://bugs.python.org/issue8513).
- fullname = find_binary(gpgcmd[0])
- if fullname is None:
- raise
portage.exception.CommandNotFound(gpgcmd[0])
- gpgcmd[0] = fullname
-
- gpgcmd = [
- _unicode_encode(arg, encoding=_encodings['fs'],
errors='strict')
- for arg in gpgcmd]
- rValue = subprocess.call(gpgcmd)
- if rValue == os.EX_OK:
- os.rename(filename + ".asc", filename)
- else:
- raise portage.exception.PortageException(
- "!!! gpg exited with '" + str(rValue) + "'
status")
+if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
+ repoman_settings.get('TERM') == 'dumb' or \
+ not sys.stdout.isatty():
+ nocolor()
+
+options, arguments = parse_args(
+ sys.argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
+
+if options.version:
+ print("Portage", portage.VERSION)
+ sys.exit(0)
+
+if options.experimental_inherit == 'y':
+ # This is experimental, so it's non-fatal.
+ qawarnings.add("inherit.missing")
+ checks_init(experimental_inherit=True)
+
+# Set this to False when an extraordinary issue (generally
+# something other than a QA issue) makes it impossible to
+# commit (like if Manifest generation fails).
+can_force = True
+
+portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
+if portdir is None:
+ sys.exit(1)
-def need_signature(filename):
- try:
- with open(
- _unicode_encode(
- filename, encoding=_encodings['fs'],
errors='strict'),
- 'rb') as f:
- return b"BEGIN PGP SIGNED MESSAGE" not in
f.readline()
- except IOError as e:
- if e.errno in (errno.ENOENT, errno.ESTALE):
- return False
- raise
+myreporoot = os.path.basename(portdir_overlay)
+myreporoot += mydir[len(portdir_overlay):]
+##################
-def repoman_scan(repoman_settings, repo_settings, vcs_settings, portdb,
options, myreporoot, mydir, check_changelog, config_root, env):
+vcs_settings = VCSSettings(options, repoman_settings)
- # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
- # behave incrementally.
- repoman_incrementals = tuple(
- x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
- if options.mode == "manifest":
- pass
- elif options.pretend:
- print(green("\nRepoMan does a once-over of the
neighborhood..."))
- else:
- print(green("\nRepoMan scours the neighborhood..."))
+##################
- qatracker = QATracker()
+repo_settings = RepoSettings(
+ config_root, portdir, portdir_overlay,
+ repoman_settings, vcs_settings, options, qawarnings)
- reposplit = myreporoot.split(os.path.sep)
- repolevel = len(reposplit)
+repoman_settings = repo_settings.repoman_settings
- if options.mode == 'commit':
- repochecks.commit_check(repolevel, reposplit)
- repochecks.conflict_check(vcs_settings, options)
+portdb = repo_settings.portdb
+##################
- changed = Changes(options)
- changed.scan(vcs_settings)
- categories = repoman_settings.categories
-
- # Make startdir relative to the canonical repodir, so that we can pass
- # it to digestgen and it won't have to be canonicalized again.
- if repolevel == 1:
- startdir = repo_settings.repodir
- else:
- startdir = normalize_path(mydir)
- startdir = os.path.join(
- repo_settings.repodir, *startdir.split(os.sep)[-2 -
repolevel + 3:])
-
- have_pmasked = False
- have_dev_keywords = False
-
- # get lists of valid keywords, licenses, and use
- new_data = repo_metadata(repo_settings.portdb, repoman_settings)
- kwlist, liclist, uselist, profile_list, \
- global_pmaskdict, liclist_deprecated = new_data
-
- repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
- repoman_settings.backup_changes('PORTAGE_ARCHLIST')
-
- ####################
-
- profiles = setup_profile(profile_list)
-
- ####################
-
- check_profiles(profiles, repoman_settings.archlist())
-
- ####################
-
- scanlist = scan(repolevel, reposplit, startdir, categories,
repo_settings)
- effective_scanlist = scanlist
- if options.if_modified == "y":
- effective_scanlist = sorted(vcs_files_to_cps(
- chain(changed.changed, changed.new, changed.removed),
- repolevel, reposplit, categories))
- if options.if_modified == "y" and len(effective_scanlist) < 1:
- logging.warn("--if-modified is enabled, but no modified
packages were found!")
-
- ####################
-
- dev_keyword = dev_keywords(profiles)
-
- # Disable the "ebuild.notadded" check when not in commit mode and
- # running `svn status` in every package dir will be too expensive.
- check_ebuild_notadded = not \
- (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode
!= "commit")
-
- # NOTE: match-all caches are not shared due to potential
- # differences between profiles in _get_implicit_iuse.
- arch_caches = {}
- arch_xmatch_caches = {}
- shared_xmatch_caches = {"cp-list": {}}
-
- include_arches = None
- if options.include_arches:
- include_arches = set()
- include_arches.update(*[x.split() for x in
options.include_arches])
-
- # initialize our checks classes here before the big xpkg loop
- manifester = Manifests(options, qatracker, repoman_settings)
- is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
- filescheck = FileChecks(
- qatracker, repoman_settings, repo_settings, portdb,
vcs_settings)
- status_check = VCSStatus(vcs_settings, qatracker)
- fetchcheck = FetchChecks(
- qatracker, repoman_settings, repo_settings, portdb,
vcs_settings)
- pkgmeta = PkgMetadata(options, qatracker, repoman_settings)
- thirdparty = ThirdPartyMirrors(repoman_settings, qatracker)
- use_flag_checks = USEFlagChecks(qatracker, uselist)
- keywordcheck = KeywordChecks(qatracker)
- liveeclasscheck = LiveEclassChecks(qatracker)
- rubyeclasscheck = RubyEclassChecks(qatracker)
- eapicheck = EAPIChecks(qatracker, repo_settings)
- descriptioncheck = DescriptionChecks(qatracker)
- licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated)
- restrictcheck = RestrictChecks(qatracker)
- ######################
-
- for xpkg in effective_scanlist:
- # ebuilds and digests added to cvs respectively.
- logging.info("checking package %s" % xpkg)
- # save memory by discarding xmatch caches from previous
package(s)
- arch_xmatch_caches.clear()
- eadded = []
- catdir, pkgdir = xpkg.split("/")
- checkdir = repo_settings.repodir + "/" + xpkg
- checkdir_relative = ""
- if repolevel < 3:
- checkdir_relative = os.path.join(pkgdir,
checkdir_relative)
- if repolevel < 2:
- checkdir_relative = os.path.join(catdir,
checkdir_relative)
- checkdir_relative = os.path.join(".", checkdir_relative)
-
- #####################
- if manifester.run(checkdir, portdb):
- continue
- if not manifester.generated_manifest:
- manifester.digest_check(xpkg, checkdir)
- ######################
+if options.echangelog is None and repo_settings.repo_config.update_changelog:
+ options.echangelog = 'y'
- if options.mode == 'manifest-check':
- continue
+if vcs_settings.vcs is None:
+ options.echangelog = 'n'
+
+# The --echangelog option causes automatic ChangeLog generation,
+# which invalidates changelog.ebuildadded and changelog.missing
+# checks.
+# Note: Some don't use ChangeLogs in distributed SCMs.
+# It will be generated on server side from scm log,
+# before package moves to the rsync server.
+# This is needed because they try to avoid merge collisions.
+# Gentoo's Council decided to always use the ChangeLog file.
+# TODO: shouldn't this just be switched on the repo, iso the VCS?
+is_echangelog_enabled = options.echangelog in ('y', 'force')
+vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
+check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn
+
+if 'digest' in repoman_settings.features and options.digest != 'n':
+ options.digest = 'y'
+
+logging.debug("vcs: %s" % (vcs_settings.vcs,))
+logging.debug("repo config: %s" % (repo_settings.repo_config,))
+logging.debug("options: %s" % (options,))
+
+# It's confusing if these warnings are displayed without the user
+# being told which profile they come from, so disable them.
+env = os.environ.copy()
+env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
+
+categories = []
+for path in repo_settings.repo_config.eclass_db.porttrees:
+ categories.extend(portage.util.grabfile(
+ os.path.join(path, 'profiles', 'categories')))
+repoman_settings.categories = frozenset(
+ portage.util.stack_lists([categories], incremental=1))
+categories = repoman_settings.categories
+
+portdb.settings = repoman_settings
+# We really only need to cache the metadata that's necessary for visibility
+# filtering. Anything else can be discarded to reduce memory consumption.
+portdb._aux_cache_keys.clear()
+portdb._aux_cache_keys.update(
+ ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
- checkdirlist = os.listdir(checkdir)
-
- ######################
- pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg)
- if is_ebuild.continue_:
- # If we can't access all the metadata then it's totally
unsafe to
- # commit since there's no way to generate a correct
Manifest.
- # Do not try to do any more QA checks on this package
since missing
- # metadata leads to false positives for several checks,
and false
- # positives confuse users.
- can_force = False
+reposplit = myreporoot.split(os.path.sep)
+repolevel = len(reposplit)
+
+###################
+
+if options.mode == 'commit':
+ repochecks.commit_check(repolevel, reposplit)
+ repochecks.conflict_check(vcs_settings, options)
+
+###################
+
+# Make startdir relative to the canonical repodir, so that we can pass
+# it to digestgen and it won't have to be canonicalized again.
+if repolevel == 1:
+ startdir = repo_settings.repodir
+else:
+ startdir = normalize_path(mydir)
+ startdir = os.path.join(
+ repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel +
3:])
+###################
+
+# get lists of valid keywords, licenses, and use
+new_data = repo_metadata(repo_settings.portdb, repoman_settings)
+kwlist, liclist, uselist, profile_list, \
+ global_pmaskdict, liclist_deprecated = new_data
+
+repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
+repoman_settings.backup_changes('PORTAGE_ARCHLIST')
+
+####################
+
+profiles = setup_profile(profile_list)
+
+####################
+
+check_profiles(profiles, repoman_settings.archlist())
+
+####################
+
+scanlist = scan(repolevel, reposplit, startdir, categories, repo_settings)
+
+####################
+
+dev_keywords = dev_keywords(profiles)
+
+qatracker = QATracker()
+
+
+if options.mode == "manifest":
+ pass
+elif options.pretend:
+ print(green("\nRepoMan does a once-over of the neighborhood..."))
+else:
+ print(green("\nRepoMan scours the neighborhood..."))
+
+#####################
+
+changed = Changes(options)
+changed.scan(vcs_settings)
+
+######################
+
+have_pmasked = False
+have_dev_keywords = False
+dofail = 0
+
+# NOTE: match-all caches are not shared due to potential
+# differences between profiles in _get_implicit_iuse.
+arch_caches = {}
+arch_xmatch_caches = {}
+shared_xmatch_caches = {"cp-list": {}}
+
+include_arches = None
+if options.include_arches:
+ include_arches = set()
+ include_arches.update(*[x.split() for x in options.include_arches])
+
+# Disable the "ebuild.notadded" check when not in commit mode and
+# running `svn status` in every package dir will be too expensive.
+
+check_ebuild_notadded = not \
+ (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode !=
"commit")
+
+effective_scanlist = scanlist
+if options.if_modified == "y":
+ effective_scanlist = sorted(vcs_files_to_cps(
+ chain(changed.changed, changed.new, changed.removed),
+ repolevel, reposplit, categories))
+
+######################
+# initialize our checks classes here before the big xpkg loop
+manifester = Manifests(options, qatracker, repoman_settings)
+is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
+filescheck = FileChecks(
+ qatracker, repoman_settings, repo_settings, portdb, vcs_settings)
+status_check = VCSStatus(vcs_settings, qatracker)
+fetchcheck = FetchChecks(
+ qatracker, repoman_settings, repo_settings, portdb, vcs_settings)
+pkgmeta = PkgMetadata(options, qatracker, repoman_settings)
+thirdparty = ThirdPartyMirrors(repoman_settings, qatracker)
+use_flag_checks = USEFlagChecks(qatracker, uselist)
+keywordcheck = KeywordChecks(qatracker, options)
+liveeclasscheck = LiveEclassChecks(qatracker)
+rubyeclasscheck = RubyEclassChecks(qatracker)
+eapicheck = EAPIChecks(qatracker, repo_settings)
+descriptioncheck = DescriptionChecks(qatracker)
+licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated)
+restrictcheck = RestrictChecks(qatracker)
+######################
+
+for xpkg in effective_scanlist:
+ # ebuilds and digests added to cvs respectively.
+ logging.info("checking package %s" % xpkg)
+ # save memory by discarding xmatch caches from previous package(s)
+ arch_xmatch_caches.clear()
+ eadded = []
+ catdir, pkgdir = xpkg.split("/")
+ checkdir = repo_settings.repodir + "/" + xpkg
+ checkdir_relative = ""
+ if repolevel < 3:
+ checkdir_relative = os.path.join(pkgdir, checkdir_relative)
+ if repolevel < 2:
+ checkdir_relative = os.path.join(catdir, checkdir_relative)
+ checkdir_relative = os.path.join(".", checkdir_relative)
+
+#####################
+ if manifester.run(checkdir, portdb):
+ continue
+ if not manifester.generated_manifest:
+ manifester.digest_check(xpkg, checkdir)
+######################
+
+ if options.mode == 'manifest-check':
+ continue
+
+ checkdirlist = os.listdir(checkdir)
+
+######################
+ pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg)
+ if is_ebuild.continue_:
+ # If we can't access all the metadata then it's totally unsafe
to
+ # commit since there's no way to generate a correct Manifest.
+ # Do not try to do any more QA checks on this package since
missing
+ # metadata leads to false positives for several checks, and
false
+ # positives confuse users.
+ can_force = False
+ continue
+######################
+
+ keywordcheck.prepare()
+
+ # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
+ ebuildlist = sorted(pkgs.values())
+ ebuildlist = [pkg.pf for pkg in ebuildlist]
+#######################
+ filescheck.check(
+ checkdir, checkdirlist, checkdir_relative, changed.changed,
changed.new)
+#######################
+ status_check.check(check_ebuild_notadded, checkdir, checkdir_relative,
xpkg)
+ eadded.extend(status_check.eadded)
+
+#################
+ fetchcheck.check(
+ xpkg, checkdir, checkdir_relative, changed.changed, changed.new)
+#################
+
+ if check_changelog and "ChangeLog" not in checkdirlist:
+ qatracker.add_error("changelog.missing", xpkg + "/ChangeLog")
+#################
+ pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel)
+ muselist = frozenset(pkgmeta.musedict)
+#################
+
+ changelog_path = os.path.join(checkdir_relative, "ChangeLog")
+ changelog_modified = changelog_path in changed.changelogs
+
+ # detect unused local USE-descriptions
+ used_useflags = set()
+
+ for y_ebuild in ebuildlist:
+ ##################
+ ebuild = Ebuild(
+ repo_settings, repolevel, pkgdir, catdir, vcs_settings,
+ xpkg, y_ebuild)
+ ##################
+
+ if check_changelog and not changelog_modified \
+ and ebuild.ebuild_path in changed.new_ebuilds:
+ qatracker.add_error('changelog.ebuildadded',
ebuild.relative_path)
+
+ if ebuild.untracked(check_ebuild_notadded, y_ebuild, eadded):
+ # ebuild not added to vcs
+ qatracker.add_error(
+ "ebuild.notadded", xpkg + "/" + y_ebuild +
".ebuild")
+
+##################
+ if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker):
+ continue
+###################
+ pkg = pkgs[y_ebuild]
+ if pkg_invalid(pkg, qatracker, ebuild):
+ allvalid = False
continue
- ######################
-
- keywordcheck.prepare()
-
- # Sort ebuilds in ascending order for the KEYWORDS.dropped
check.
- ebuildlist = sorted(pkgs.values())
- ebuildlist = [pkg.pf for pkg in ebuildlist]
- #######################
- filescheck.check(
- checkdir, checkdirlist, checkdir_relative,
changed.changed, changed.new)
- #######################
- status_check.check(check_ebuild_notadded, checkdir,
checkdir_relative, xpkg)
- eadded.extend(status_check.eadded)
-
- #################
- fetchcheck.check(
- xpkg, checkdir, checkdir_relative, changed.changed,
changed.new)
- #################
-
- if check_changelog and "ChangeLog" not in checkdirlist:
- qatracker.add_error("changelog.missing", xpkg +
"/ChangeLog")
- #################
- pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel)
- muselist = frozenset(pkgmeta.musedict)
- #################
-
- changelog_path = os.path.join(checkdir_relative, "ChangeLog")
- changelog_modified = changelog_path in changed.changelogs
-
- # detect unused local USE-descriptions
- used_useflags = set()
-
- for y_ebuild in ebuildlist:
- ##################
- ebuild = Ebuild(
- repo_settings, repolevel, pkgdir, catdir,
vcs_settings,
- xpkg, y_ebuild)
- ##################
-
- if check_changelog and not changelog_modified \
- and ebuild.ebuild_path in changed.new_ebuilds:
- qatracker.add_error('changelog.ebuildadded',
ebuild.relative_path)
-
- if ebuild.untracked(check_ebuild_notadded, y_ebuild,
eadded):
- # ebuild not added to vcs
- qatracker.add_error(
- "ebuild.notadded", xpkg + "/" +
y_ebuild + ".ebuild")
- ##################
- if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker):
- continue
- ###################
- pkg = pkgs[y_ebuild]
- if pkg_invalid(pkg, qatracker, ebuild):
- allvalid = False
- continue
+ myaux = pkg._metadata
+ eapi = myaux["EAPI"]
+ inherited = pkg.inherited
+ live_ebuild = live_eclasses.intersection(inherited)
- myaux = pkg._metadata
- eapi = myaux["EAPI"]
- inherited = pkg.inherited
- live_ebuild = live_eclasses.intersection(inherited)
+ #######################
+ eapicheck.check(pkg, ebuild)
+ #######################
+ for k, v in myaux.items():
+ if not isinstance(v, basestring):
+ continue
+ m = non_ascii_re.search(v)
+ if m is not None:
+ qatracker.add_error(
+ "variable.invalidchar",
+ "%s: %s variable contains non-ASCII "
+ "character at position %s" %
+ (ebuild.relative_path, k, m.start() +
1))
+
+ if not fetchcheck.src_uri_error:
#######################
- eapicheck.check(pkg, ebuild)
+ thirdparty.check(myaux, ebuild.relative_path)
#######################
+ if myaux.get("PROVIDE"):
+ qatracker.add_error("virtual.oldstyle",
ebuild.relative_path)
- for k, v in myaux.items():
- if not isinstance(v, basestring):
+ for pos, missing_var in enumerate(missingvars):
+ if not myaux.get(missing_var):
+ if catdir == "virtual" and \
+ missing_var in ("HOMEPAGE", "LICENSE"):
continue
- m = non_ascii_re.search(v)
- if m is not None:
- qatracker.add_error(
- "variable.invalidchar",
- "%s: %s variable contains
non-ASCII "
- "character at position %s" %
- (ebuild.relative_path, k,
m.start() + 1))
-
- if not fetchcheck.src_uri_error:
- #######################
- thirdparty.check(myaux, ebuild.relative_path)
- #######################
- if myaux.get("PROVIDE"):
- qatracker.add_error("virtual.oldstyle",
ebuild.relative_path)
-
- for pos, missing_var in enumerate(missingvars):
- if not myaux.get(missing_var):
- if catdir == "virtual" and \
- missing_var in ("HOMEPAGE",
"LICENSE"):
- continue
- if live_ebuild and missing_var ==
"KEYWORDS":
- continue
- myqakey = missingvars[pos] + ".missing"
- qatracker.add_error(myqakey, xpkg + "/"
+ y_ebuild + ".ebuild")
+ if live_ebuild and missing_var == "KEYWORDS":
+ continue
+ myqakey = missingvars[pos] + ".missing"
+ qatracker.add_error(myqakey, xpkg + "/" +
y_ebuild + ".ebuild")
- if catdir == "virtual":
- for var in ("HOMEPAGE", "LICENSE"):
- if myaux.get(var):
- myqakey = var + ".virtual"
- qatracker.add_error(myqakey,
ebuild.relative_path)
+ if catdir == "virtual":
+ for var in ("HOMEPAGE", "LICENSE"):
+ if myaux.get(var):
+ myqakey = var + ".virtual"
+ qatracker.add_error(myqakey,
ebuild.relative_path)
- #######################
- descriptioncheck.check(pkg, ebuild)
- #######################
+ #######################
+ descriptioncheck.check(pkg, ebuild)
+ #######################
- keywords = myaux["KEYWORDS"].split()
+ keywords = myaux["KEYWORDS"].split()
- ebuild_archs = set(
- kw.lstrip("~") for kw in keywords if not
kw.startswith("-"))
+ ebuild_archs = set(
+ kw.lstrip("~") for kw in keywords if not
kw.startswith("-"))
+ #######################
+ keywordcheck.check(
+ pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs,
changed,
+ live_ebuild, kwlist, profiles)
+ #######################
+
+ if live_ebuild and repo_settings.repo_config.name == "gentoo":
#######################
- keywordcheck.check(
- pkg, xpkg, ebuild, y_ebuild, keywords,
ebuild_archs, changed,
- live_ebuild, kwlist, profiles)
+ liveeclasscheck.check(
+ pkg, xpkg, ebuild, y_ebuild, keywords,
global_pmaskdict)
#######################
- if live_ebuild and repo_settings.repo_config.name ==
"gentoo":
- #######################
- liveeclasscheck.check(
- pkg, xpkg, ebuild, y_ebuild, keywords,
global_pmaskdict)
- #######################
+ if options.ignore_arches:
+ arches = [[
+ repoman_settings["ARCH"],
repoman_settings["ARCH"],
+ repoman_settings["ACCEPT_KEYWORDS"].split()]]
+ else:
+ arches = set()
+ for keyword in keywords:
+ if keyword[0] == "-":
+ continue
+ elif keyword[0] == "~":
+ arch = keyword[1:]
+ if arch == "*":
+ for expanded_arch in profiles:
+ if expanded_arch ==
"**":
+ continue
+ arches.add(
+ (keyword,
expanded_arch, (
+
expanded_arch, "~" + expanded_arch)))
+ else:
+ arches.add((keyword, arch,
(arch, keyword)))
+ else:
+ if keyword == "*":
+ for expanded_arch in profiles:
+ if expanded_arch ==
"**":
+ continue
+ arches.add(
+ (keyword,
expanded_arch, (expanded_arch,)))
+ else:
+ arches.add((keyword, keyword,
(keyword,)))
+ if not arches:
+ # Use an empty profile for checking
dependencies of
+ # packages that have empty KEYWORDS.
+ arches.add(('**', '**', ('**',)))
+
+ unknown_pkgs = set()
+ baddepsyntax = False
+ badlicsyntax = False
+ badprovsyntax = False
+ catpkg = catdir + "/" + y_ebuild
+
+ inherited_java_eclass = "java-pkg-2" in inherited or \
+ "java-pkg-opt-2" in inherited
+ inherited_wxwidgets_eclass = "wxwidgets" in inherited
+ operator_tokens = set(["||", "(", ")"])
+ type_list, badsyntax = [], []
+ for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES",
"PROVIDE"):
+ mydepstr = myaux[mytype]
+
+ buildtime = mytype in Package._buildtime_keys
+ runtime = mytype in Package._runtime_keys
+ token_class = None
+ if mytype.endswith("DEPEND"):
+ token_class = portage.dep.Atom
- if options.ignore_arches:
- arches = [[
- repoman_settings["ARCH"],
repoman_settings["ARCH"],
-
repoman_settings["ACCEPT_KEYWORDS"].split()]]
- else:
- arches = set()
- for keyword in keywords:
- if keyword[0] == "-":
+ try:
+ atoms = portage.dep.use_reduce(
+ mydepstr, matchall=1, flat=True,
+ is_valid_flag=pkg.iuse.is_valid_flag,
token_class=token_class)
+ except portage.exception.InvalidDependString as e:
+ atoms = None
+ badsyntax.append(str(e))
+
+ if atoms and mytype.endswith("DEPEND"):
+ if runtime and \
+ "test?" in mydepstr.split():
+ qatracker.add_error(
+ mytype + '.suspect',
+ "%s: 'test?' USE conditional in
%s" %
+ (ebuild.relative_path, mytype))
+
+ for atom in atoms:
+ if atom == "||":
continue
- elif keyword[0] == "~":
- arch = keyword[1:]
- if arch == "*":
- for expanded_arch in
profiles:
- if
expanded_arch == "**":
- continue
- arches.add(
-
(keyword, expanded_arch, (
-
expanded_arch, "~" + expanded_arch)))
- else:
- arches.add((keyword,
arch, (arch, keyword)))
- else:
- if keyword == "*":
- for expanded_arch in
profiles:
- if
expanded_arch == "**":
- continue
- arches.add(
-
(keyword, expanded_arch, (expanded_arch,)))
- else:
- arches.add((keyword,
keyword, (keyword,)))
- if not arches:
- # Use an empty profile for checking
dependencies of
- # packages that have empty KEYWORDS.
- arches.add(('**', '**', ('**',)))
-
- unknown_pkgs = set()
- baddepsyntax = False
- badlicsyntax = False
- badprovsyntax = False
- catpkg = catdir + "/" + y_ebuild
-
- inherited_java_eclass = "java-pkg-2" in inherited or \
- "java-pkg-opt-2" in inherited
- inherited_wxwidgets_eclass = "wxwidgets" in inherited
- operator_tokens = set(["||", "(", ")"])
- type_list, badsyntax = [], []
- for mytype in Package._dep_keys + ("LICENSE",
"PROPERTIES", "PROVIDE"):
- mydepstr = myaux[mytype]
-
- buildtime = mytype in Package._buildtime_keys
- runtime = mytype in Package._runtime_keys
- token_class = None
- if mytype.endswith("DEPEND"):
- token_class = portage.dep.Atom
-
- try:
- atoms = portage.dep.use_reduce(
- mydepstr, matchall=1, flat=True,
-
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
- except portage.exception.InvalidDependString as
e:
- atoms = None
- badsyntax.append(str(e))
-
- if atoms and mytype.endswith("DEPEND"):
- if runtime and \
- "test?" in mydepstr.split():
- qatracker.add_error(
- mytype + '.suspect',
- "%s: 'test?' USE
conditional in %s" %
- (ebuild.relative_path,
mytype))
- for atom in atoms:
- if atom == "||":
- continue
+ is_blocker = atom.blocker
- is_blocker = atom.blocker
+ # Skip dependency.unknown for blockers,
so that we
+ # don't encourage people to remove
necessary blockers,
+ # as discussed in bug 382407. We use
atom.without_use
+ # due to bug 525376.
+ if not is_blocker and \
+ not portdb.xmatch("match-all",
atom.without_use) and \
+ not
atom.cp.startswith("virtual/"):
+ unknown_pkgs.add((mytype,
atom.unevaluated_atom))
- # Skip dependency.unknown for
blockers, so that we
- # don't encourage people to
remove necessary blockers,
- # as discussed in bug #382407.
+ if catdir != "virtual":
if not is_blocker and \
- not
portdb.xmatch("match-all", atom) and \
- not
atom.cp.startswith("virtual/"):
-
unknown_pkgs.add((mytype, atom.unevaluated_atom))
-
- if catdir != "virtual":
- if not is_blocker and \
- atom.cp in
suspect_virtual:
-
qatracker.add_error(
-
'virtual.suspect', ebuild.relative_path +
- ": %s:
consider using '%s' instead of '%s'" %
-
(mytype, suspect_virtual[atom.cp], atom))
- if not is_blocker and \
-
atom.cp.startswith("perl-core/"):
-
qatracker.add_error('dependency.perlcore',
-
ebuild.relative_path +
- ": %s:
please use '%s' instead of '%s'" %
- (mytype,
-
atom.replace("perl-core/","virtual/perl-"),
- atom))
-
- if buildtime and \
- not is_blocker and \
- not
inherited_java_eclass and \
- atom.cp ==
"virtual/jdk":
- qatracker.add_error(
-
'java.eclassesnotused', ebuild.relative_path)
- elif buildtime and \
- not is_blocker and \
- not
inherited_wxwidgets_eclass and \
- atom.cp ==
"x11-libs/wxGTK":
+ atom.cp in
suspect_virtual:
qatracker.add_error(
-
'wxwidgets.eclassnotused',
- "%s: %ss on
x11-libs/wxGTK without inheriting"
- "
wxwidgets.eclass" % (ebuild.relative_path, mytype))
- elif runtime:
- if not is_blocker and \
- atom.cp in
suspect_rdepend:
-
qatracker.add_error(
- mytype
+ '.suspect',
-
ebuild.relative_path + ": '%s'" % atom)
-
- if atom.operator == "~" and \
-
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
- qacat =
'dependency.badtilde'
+
'virtual.suspect', ebuild.relative_path +
+ ": %s: consider
using '%s' instead of '%s'" %
+ (mytype,
suspect_virtual[atom.cp], atom))
+ if not is_blocker and \
+
atom.cp.startswith("perl-core/"):
+
qatracker.add_error('dependency.perlcore',
+
ebuild.relative_path +
+ ": %s: please
use '%s' instead of '%s'" %
+ (mytype,
+
atom.replace("perl-core/","virtual/perl-"),
+ atom))
+
+ if buildtime and \
+ not is_blocker and \
+ not inherited_java_eclass and \
+ atom.cp == "virtual/jdk":
+ qatracker.add_error(
+ 'java.eclassesnotused',
ebuild.relative_path)
+ elif buildtime and \
+ not is_blocker and \
+ not inherited_wxwidgets_eclass
and \
+ atom.cp == "x11-libs/wxGTK":
+ qatracker.add_error(
+
'wxwidgets.eclassnotused',
+ "%s: %ss on
x11-libs/wxGTK without inheriting"
+ " wxwidgets.eclass" %
(ebuild.relative_path, mytype))
+ elif runtime:
+ if not is_blocker and \
+ atom.cp in
suspect_rdepend:
qatracker.add_error(
- qacat, "%s: %s
uses the ~ operator"
- " with a
non-zero revision: '%s'" %
-
(ebuild.relative_path, mytype, atom))
+ mytype +
'.suspect',
+
ebuild.relative_path + ": '%s'" % atom)
- type_list.extend([mytype] * (len(badsyntax) -
len(type_list)))
-
- for m, b in zip(type_list, badsyntax):
- if m.endswith("DEPEND"):
- qacat = "dependency.syntax"
- else:
- qacat = m + ".syntax"
- qatracker.add_error(
- qacat, "%s: %s: %s" %
(ebuild.relative_path, m, b))
+ if atom.operator == "~" and \
+
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
+ qacat = 'dependency.badtilde'
+ qatracker.add_error(
+ qacat, "%s: %s uses the
~ operator"
+ " with a non-zero
revision: '%s'" %
+ (ebuild.relative_path,
mytype, atom))
- badlicsyntax = len([z for z in type_list if z ==
"LICENSE"])
- badprovsyntax = len([z for z in type_list if z ==
"PROVIDE"])
- baddepsyntax = len(type_list) != badlicsyntax +
badprovsyntax
- badlicsyntax = badlicsyntax > 0
- badprovsyntax = badprovsyntax > 0
+ check_missingslot(atom, mytype, eapi,
portdb, qatracker,
+ ebuild.relative_path, myaux)
- #################
- use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild,
muselist)
+ type_list.extend([mytype] * (len(badsyntax) -
len(type_list)))
- ebuild_used_useflags = use_flag_checks.getUsedUseFlags()
- used_useflags =
used_useflags.union(ebuild_used_useflags)
+ for m, b in zip(type_list, badsyntax):
+ if m.endswith("DEPEND"):
+ qacat = "dependency.syntax"
+ else:
+ qacat = m + ".syntax"
+ qatracker.add_error(
+ qacat, "%s: %s: %s" % (ebuild.relative_path, m,
b))
+
+ badlicsyntax = len([z for z in type_list if z == "LICENSE"])
+ badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
+ baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
+ badlicsyntax = badlicsyntax > 0
+ badprovsyntax = badprovsyntax > 0
+
+ #################
+ use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, muselist)
+
+ ebuild_used_useflags = use_flag_checks.getUsedUseFlags()
+ used_useflags = used_useflags.union(ebuild_used_useflags)
+ #################
+ rubyeclasscheck.check(pkg, ebuild)
+ #################
+
+ # license checks
+ if not badlicsyntax:
#################
- rubyeclasscheck.check(pkg, ebuild)
+ licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
#################
- # license checks
- if not badlicsyntax:
- #################
- licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
- #################
+ #################
+ restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
+ #################
- #################
- restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
- #################
-
- # Syntax Checks
+ # Syntax Checks
- if not vcs_settings.vcs_preserves_mtime:
- if ebuild.ebuild_path not in
changed.new_ebuilds and \
- ebuild.ebuild_path not in
changed.ebuilds:
- pkg.mtime = None
+ if not vcs_settings.vcs_preserves_mtime:
+ if ebuild.ebuild_path not in changed.new_ebuilds and \
+ ebuild.ebuild_path not in changed.ebuilds:
+ pkg.mtime = None
+ try:
+ # All ebuilds should have utf_8 encoding.
+ f = io.open(
+ _unicode_encode(
+ ebuild.full_path,
encoding=_encodings['fs'], errors='strict'),
+ mode='r', encoding=_encodings['repo.content'])
try:
- # All ebuilds should have utf_8 encoding.
- f = io.open(
- _unicode_encode(
- ebuild.full_path,
encoding=_encodings['fs'], errors='strict'),
- mode='r',
encoding=_encodings['repo.content'])
- try:
- for check_name, e in run_checks(f, pkg):
- qatracker.add_error(
- check_name,
ebuild.relative_path + ': %s' % e)
- finally:
- f.close()
- except UnicodeDecodeError:
- # A file.UTF8 failure will have already been
recorded above.
- pass
+ for check_name, e in run_checks(f, pkg):
+ qatracker.add_error(
+ check_name,
ebuild.relative_path + ': %s' % e)
+ finally:
+ f.close()
+ except UnicodeDecodeError:
+ # A file.UTF8 failure will have already been recorded
above.
+ pass
+
+ if options.force:
+ # The dep_check() calls are the most expensive QA test.
If --force
+ # is enabled, there's no point in wasting time on these
since the
+ # user is intent on forcing the commit anyway.
+ continue
- if options.force:
- # The dep_check() calls are the most expensive
QA test. If --force
- # is enabled, there's no point in wasting time
on these since the
- # user is intent on forcing the commit anyway.
+ relevant_profiles = []
+ for keyword, arch, groups in arches:
+ if arch not in profiles:
+ # A missing profile will create an error
further down
+ # during the KEYWORDS verification.
continue
- relevant_profiles = []
- for keyword, arch, groups in arches:
- if arch not in profiles:
- # A missing profile will create an
error further down
- # during the KEYWORDS verification.
+ if include_arches is not None:
+ if arch not in include_arches:
continue
- if include_arches is not None:
- if arch not in include_arches:
- continue
-
- relevant_profiles.extend(
- (keyword, groups, prof) for prof in
profiles[arch])
+ relevant_profiles.extend(
+ (keyword, groups, prof) for prof in
profiles[arch])
- relevant_profiles.sort(key=sort_key)
+ def sort_key(item):
+ return item[2].sub_path
- for keyword, groups, prof in relevant_profiles:
+ relevant_profiles.sort(key=sort_key)
- is_stable_profile = prof.status == "stable"
- is_dev_profile = prof.status == "dev" and \
- options.include_dev
- is_exp_profile = prof.status == "exp" and \
- options.include_exp_profiles == 'y'
- if not (is_stable_profile or is_dev_profile or
is_exp_profile):
- continue
+ for keyword, groups, prof in relevant_profiles:
- dep_settings = arch_caches.get(prof.sub_path)
- if dep_settings is None:
- dep_settings = portage.config(
-
config_profile_path=prof.abs_path,
-
config_incrementals=repoman_incrementals,
- config_root=config_root,
- local_config=False,
-
_unmatched_removal=options.unmatched_removal,
- env=env,
repositories=repoman_settings.repositories)
- dep_settings.categories =
repoman_settings.categories
- if options.without_mask:
- dep_settings._mask_manager_obj
= \
-
copy.deepcopy(dep_settings._mask_manager)
-
dep_settings._mask_manager._pmaskdict.clear()
- arch_caches[prof.sub_path] =
dep_settings
-
- xmatch_cache_key = (prof.sub_path,
tuple(groups))
- xcache =
arch_xmatch_caches.get(xmatch_cache_key)
- if xcache is None:
- portdb.melt()
- portdb.freeze()
- xcache = portdb.xcache
- xcache.update(shared_xmatch_caches)
- arch_xmatch_caches[xmatch_cache_key] =
xcache
-
-
repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings
- portdb.settings = dep_settings
- portdb.xcache = xcache
-
- dep_settings["ACCEPT_KEYWORDS"] = "
".join(groups)
- # just in case, prevent config.reset() from
nuking these.
- dep_settings.backup_changes("ACCEPT_KEYWORDS")
-
- # This attribute is used in dbapi._match_use()
to apply
- # use.stable.{mask,force} settings based on the
stable
- # status of the parent package. This is
required in order
- # for USE deps of unstable packages to be
resolved correctly,
- # since otherwise use.stable.{mask,force}
settings of
- # dependencies may conflict (see bug #456342).
- dep_settings._parent_stable =
dep_settings._isStable(pkg)
-
- # Handle package.use*.{force,mask) calculation,
for use
- # in dep_check.
- dep_settings.useforce =
dep_settings._use_manager.getUseForce(
- pkg, stable=dep_settings._parent_stable)
- dep_settings.usemask =
dep_settings._use_manager.getUseMask(
- pkg, stable=dep_settings._parent_stable)
-
- if not baddepsyntax:
- ismasked = not ebuild_archs or \
- pkg.cpv not in
portdb.xmatch("match-visible", pkg.cp)
- if ismasked:
- if not have_pmasked:
- have_pmasked =
bool(dep_settings._getMaskAtom(
- pkg.cpv,
pkg._metadata))
- if options.ignore_masked:
- continue
- # we are testing deps for a
masked package; give it some lee-way
- suffix = "masked"
- matchmode = "minimum-all"
- else:
- suffix = ""
- matchmode = "minimum-visible"
-
- if not have_dev_keywords:
- have_dev_keywords = \
-
bool(dev_keyword.intersection(keywords))
-
- if prof.status == "dev":
- suffix = suffix + "indev"
-
- for mytype in Package._dep_keys:
-
- mykey = "dependency.bad" +
suffix
- myvalue = myaux[mytype]
- if not myvalue:
- continue
-
- success, atoms =
portage.dep_check(
- myvalue, portdb,
dep_settings,
- use="all",
mode=matchmode, trees=repo_settings.trees)
-
- if success:
- if atoms:
-
- # Don't bother
with dependency.unknown for
- # cases in
which *DEPEND.bad is triggered.
- for atom in
atoms:
- #
dep_check returns all blockers and they
- #
aren't counted for *DEPEND.bad, so we
- #
ignore them here.
- if not
atom.blocker:
-
unknown_pkgs.discard(
-
(mytype, atom.unevaluated_atom))
-
- if not
prof.sub_path:
- #
old-style virtuals currently aren't
- #
resolvable with empty profile, since
- #
'virtuals' mappings are unavailable
- # (it
would be expensive to search
- # for
PROVIDE in all ebuilds)
- atoms =
[
-
atom for atom in atoms if not (
-
atom.cp.startswith('virtual/')
-
and not portdb.cp_list(atom.cp))]
-
- # we have some
unsolvable deps
- # remove !
deps, which always show up as unsatisfiable
- atoms = [
-
str(atom.unevaluated_atom)
- for
atom in atoms if not atom.blocker]
-
- # if we emptied
out our list, continue:
- if not atoms:
- continue
-
qatracker.add_error(
- mykey,
"%s: %s: %s(%s) %s" % (
-
ebuild.relative_path, mytype, keyword, prof,
-
repr(atoms)))
- else:
- qatracker.add_error(
- mykey, "%s: %s:
%s(%s) %s" % (
-
ebuild.relative_path, mytype, keyword, prof,
-
repr(atoms)))
-
- if not baddepsyntax and unknown_pkgs:
- type_map = {}
- for mytype, atom in unknown_pkgs:
- type_map.setdefault(mytype,
set()).add(atom)
- for mytype, atoms in type_map.items():
- qatracker.add_error(
- "dependency.unknown", "%s: %s:
%s"
- % (ebuild.relative_path,
mytype, ", ".join(sorted(atoms))))
-
- # check if there are unused local USE-descriptions in
metadata.xml
- # (unless there are any invalids, to avoid noise)
- if allvalid:
- for myflag in muselist.difference(used_useflags):
- qatracker.add_error(
- "metadata.warning",
- "%s/metadata.xml: unused local
USE-description: '%s'"
- % (xpkg, myflag))
-
- suggest_ignore_masked = False
- suggest_include_dev = False
- if have_pmasked and not (options.without_mask or options.ignore_masked):
- suggest_ignore_masked = True
- if have_dev_keywords and not options.include_dev:
- suggest_include_dev = True
-
- if suggest_ignore_masked or suggest_include_dev:
- print()
- if suggest_ignore_masked:
- print(bold(
- "Note: use --without-mask to check "
- "KEYWORDS on dependencies of masked packages"))
-
- if suggest_include_dev:
- print(bold(
- "Note: use --include-dev (-d) to check "
- "dependencies for 'dev' profiles"))
- print()
-
- return qatracker
+ is_stable_profile = prof.status == "stable"
+ is_dev_profile = prof.status == "dev" and \
+ options.include_dev
+ is_exp_profile = prof.status == "exp" and \
+ options.include_exp_profiles == 'y'
+ if not (is_stable_profile or is_dev_profile or
is_exp_profile):
+ continue
-def repoman_main(argv, config_root=None, pkgdir=None):
+ dep_settings = arch_caches.get(prof.sub_path)
+ if dep_settings is None:
+ dep_settings = portage.config(
+ config_profile_path=prof.abs_path,
+
config_incrementals=repoman_incrementals,
+ config_root=config_root,
+ local_config=False,
+
_unmatched_removal=options.unmatched_removal,
+ env=env,
repositories=repoman_settings.repositories)
+ dep_settings.categories =
repoman_settings.categories
+ if options.without_mask:
+ dep_settings._mask_manager_obj = \
+
copy.deepcopy(dep_settings._mask_manager)
+
dep_settings._mask_manager._pmaskdict.clear()
+ arch_caches[prof.sub_path] = dep_settings
+
+ xmatch_cache_key = (prof.sub_path, tuple(groups))
+ xcache = arch_xmatch_caches.get(xmatch_cache_key)
+ if xcache is None:
+ portdb.melt()
+ portdb.freeze()
+ xcache = portdb.xcache
+ xcache.update(shared_xmatch_caches)
+ arch_xmatch_caches[xmatch_cache_key] = xcache
+
+
repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings
+ portdb.settings = dep_settings
+ portdb.xcache = xcache
+
+ dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
+ # just in case, prevent config.reset() from nuking
these.
+ dep_settings.backup_changes("ACCEPT_KEYWORDS")
+
+ # This attribute is used in dbapi._match_use() to apply
+ # use.stable.{mask,force} settings based on the stable
+ # status of the parent package. This is required in
order
+ # for USE deps of unstable packages to be resolved
correctly,
+ # since otherwise use.stable.{mask,force} settings of
+ # dependencies may conflict (see bug #456342).
+ dep_settings._parent_stable =
dep_settings._isStable(pkg)
+
+ # Handle package.use*.{force,mask) calculation, for use
+ # in dep_check.
+ dep_settings.useforce =
dep_settings._use_manager.getUseForce(
+ pkg, stable=dep_settings._parent_stable)
+ dep_settings.usemask =
dep_settings._use_manager.getUseMask(
+ pkg, stable=dep_settings._parent_stable)
+
+ if not baddepsyntax:
+ ismasked = not ebuild_archs or \
+ pkg.cpv not in
portdb.xmatch("match-visible",
+ Atom("%s::%s" % (pkg.cp,
repo_settings.repo_config.name)))
+ if ismasked:
+ if not have_pmasked:
+ have_pmasked =
bool(dep_settings._getMaskAtom(
+ pkg.cpv, pkg._metadata))
+ if options.ignore_masked:
+ continue
+ # we are testing deps for a masked
package; give it some lee-way
+ suffix = "masked"
+ matchmode = "minimum-all"
+ else:
+ suffix = ""
+ matchmode = "minimum-visible"
- if config_root is None:
- config_root = os.environ.get("PORTAGE_CONFIGROOT")
- repoman_settings = portage.config(config_root=config_root,
local_config=False)
+ if not have_dev_keywords:
+ have_dev_keywords = \
+
bool(dev_keywords.intersection(keywords))
- if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
- repoman_settings.get('TERM') == 'dumb' or \
- not sys.stdout.isatty():
- nocolor()
+ if prof.status == "dev":
+ suffix = suffix + "indev"
- options, arguments = parse_args(
- argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
+ for mytype in Package._dep_keys:
- if options.version:
- print("Portage", portage.VERSION)
- sys.exit(0)
+ mykey = "dependency.bad" + suffix
+ myvalue = myaux[mytype]
+ if not myvalue:
+ continue
- if options.experimental_inherit == 'y':
- # This is experimental, so it's non-fatal.
- qawarnings.add("inherit.missing")
- checks_init(experimental_inherit=True)
+ success, atoms = portage.dep_check(
+ myvalue, portdb, dep_settings,
+ use="all", mode=matchmode,
trees=repo_settings.trees)
+
+ if success:
+ if atoms:
+
+ # Don't bother with
dependency.unknown for
+ # cases in which
*DEPEND.bad is triggered.
+ for atom in atoms:
+ # dep_check
returns all blockers and they
+ # aren't
counted for *DEPEND.bad, so we
+ # ignore them
here.
+ if not
atom.blocker:
+
unknown_pkgs.discard(
+
(mytype, atom.unevaluated_atom))
+
+ if not prof.sub_path:
+ # old-style
virtuals currently aren't
+ # resolvable
with empty profile, since
+ # 'virtuals'
mappings are unavailable
+ # (it would be
expensive to search
+ # for PROVIDE
in all ebuilds)
+ atoms = [
+ atom
for atom in atoms if not (
+
atom.cp.startswith('virtual/')
+
and not portdb.cp_list(atom.cp))]
+
+ # we have some
unsolvable deps
+ # remove ! deps, which
always show up as unsatisfiable
+ atoms = [
+
str(atom.unevaluated_atom)
+ for atom in
atoms if not atom.blocker]
+
+ # if we emptied out our
list, continue:
+ if not atoms:
+ continue
+
qatracker.add_error(mykey,
+ "%s: %s:
%s(%s)\n%s"
+ %
(ebuild.relative_path, mytype, keyword, prof,
+
pformat(atoms, indent=6)))
+ else:
+ qatracker.add_error(mykey,
+ "%s: %s: %s(%s)\n%s"
+ %
(ebuild.relative_path, mytype, keyword, prof,
+ pformat(atoms,
indent=6)))
+
+ if not baddepsyntax and unknown_pkgs:
+ type_map = {}
+ for mytype, atom in unknown_pkgs:
+ type_map.setdefault(mytype, set()).add(atom)
+ for mytype, atoms in type_map.items():
+ qatracker.add_error(
+ "dependency.unknown", "%s: %s: %s"
+ % (ebuild.relative_path, mytype, ",
".join(sorted(atoms))))
+
+ # check if there are unused local USE-descriptions in metadata.xml
+ # (unless there are any invalids, to avoid noise)
+ if allvalid:
+ for myflag in muselist.difference(used_useflags):
+ qatracker.add_error(
+ "metadata.warning",
+ "%s/metadata.xml: unused local USE-description:
'%s'"
+ % (xpkg, myflag))
+
+if options.if_modified == "y" and len(effective_scanlist) < 1:
+ logging.warning("--if-modified is enabled, but no modified packages
were found!")
+
+if options.mode == "manifest":
+ sys.exit(dofail)
+
+# dofail will be true if we have failed in at least one non-warning category
+dofail = 0
+# dowarn will be true if we tripped any warnings
+dowarn = 0
+# dofull will be true if we should print a "repoman full" informational message
+dofull = options.mode != 'full'
+
+for x in qacats:
+ if x not in qatracker.fails:
+ continue
+ dowarn = 1
+ if x not in qawarnings:
+ dofail = 1
+
+if dofail or \
+ (dowarn and not (options.quiet or options.mode == "scan")):
+ dofull = 0
+
+# Save QA output so that it can be conveniently displayed
+# in $EDITOR while the user creates a commit message.
+# Otherwise, the user would not be able to see this output
+# once the editor has taken over the screen.
+qa_output = io.StringIO()
+style_file = ConsoleStyleFile(sys.stdout)
+if options.mode == 'commit' and \
+ (not commitmessage or not commitmessage.strip()):
+ style_file.write_listener = qa_output
+console_writer = StyleWriter(file=style_file, maxcol=9999)
+console_writer.style_listener = style_file.new_styles
+
+f = formatter.AbstractFormatter(console_writer)
+
+format_outputs = {
+ 'column': format_qa_output_column,
+ 'default': format_qa_output
+}
+
+format_output = format_outputs.get(
+ options.output_style, format_outputs['default'])
+format_output(f, qatracker.fails, dofull, dofail, options, qawarnings)
+
+style_file.flush()
+del console_writer, f, style_file
+qa_output = qa_output.getvalue()
+qa_output = qa_output.splitlines(True)
+
+suggest_ignore_masked = False
+suggest_include_dev = False
+
+if have_pmasked and not (options.without_mask or options.ignore_masked):
+ suggest_ignore_masked = True
+if have_dev_keywords and not options.include_dev:
+ suggest_include_dev = True
+
+if suggest_ignore_masked or suggest_include_dev:
+ print()
+ if suggest_ignore_masked:
+ print(bold(
+ "Note: use --without-mask to check "
+ "KEYWORDS on dependencies of masked packages"))
+
+ if suggest_include_dev:
+ print(bold(
+ "Note: use --include-dev (-d) to check "
+ "dependencies for 'dev' profiles"))
+ print()
+
+if options.mode != 'commit':
+ if dofull:
+ print(bold("Note: type \"repoman full\" for a complete
listing."))
+ if dowarn and not dofail:
+ utilities.repoman_sez(
+ "\"You're only giving me a partial QA payment?\n"
+ " I'll take it this time, but I'm not
happy.\"")
+ elif not dofail:
+ utilities.repoman_sez(
+ "\"If everyone were like you, I'd be out of
business!\"")
+ elif dofail:
+ print(bad("Please fix these important QA issues first."))
+ utilities.repoman_sez(
+ "\"Make your QA payment on time"
+ " and you'll never see the likes of me.\"\n")
+ sys.exit(1)
+else:
+ if dofail and can_force and options.force and not options.pretend:
+ utilities.repoman_sez(
+ " \"You want to commit even with these QA issues?\n"
+ " I'll take it this time, but I'm not
happy.\"\n")
+ elif dofail:
+ if options.force and not can_force:
+ print(bad(
+ "The --force option has been disabled"
+ " due to extraordinary issues."))
+ print(bad("Please fix these important QA issues first."))
+ utilities.repoman_sez(
+ "\"Make your QA payment on time"
+ " and you'll never see the likes of me.\"\n")
+ sys.exit(1)
- # Set this to False when an extraordinary issue (generally
- # something other than a QA issue) makes it impossible to
- # commit (like if Manifest generation fails).
- can_force = True
+ if options.pretend:
+ utilities.repoman_sez(
+ "\"So, you want to play it safe. Good call.\"\n")
- portdir, portdir_overlay, mydir =
utilities.FindPortdir(repoman_settings, pkgdir)
- if portdir is None:
+ myunadded = []
+ if vcs_settings.vcs == "cvs":
+ try:
+ myvcstree = portage.cvstree.getentries("./",
recursive=1)
+ myunadded = portage.cvstree.findunadded(
+ myvcstree, recursive=1, basedir="./")
+ except SystemExit as e:
+ raise # TODO propagate this
+ except:
+ err("Error retrieving CVS tree; exiting.")
+ if vcs_settings.vcs == "svn":
+ try:
+ with repoman_popen("svn status --no-ignore") as f:
+ svnstatus = f.readlines()
+ myunadded = [
+ "./" + elem.rstrip().split()[1]
+ for elem in svnstatus
+ if elem.startswith("?") or elem.startswith("I")]
+ except SystemExit as e:
+ raise # TODO propagate this
+ except:
+ err("Error retrieving SVN info; exiting.")
+ if vcs_settings.vcs == "git":
+ # get list of files not under version control or missing
+ myf = repoman_popen("git ls-files --others")
+ myunadded = ["./" + elem[:-1] for elem in myf]
+ myf.close()
+ if vcs_settings.vcs == "bzr":
+ try:
+ with repoman_popen("bzr status -S .") as f:
+ bzrstatus = f.readlines()
+ myunadded = [
+ "./" +
elem.rstrip().split()[1].split('/')[-1:][0]
+ for elem in bzrstatus
+ if elem.startswith("?") or elem[0:2] == " D"]
+ except SystemExit as e:
+ raise # TODO propagate this
+ except:
+ err("Error retrieving bzr info; exiting.")
+ if vcs_settings.vcs == "hg":
+ with repoman_popen("hg status --no-status --unknown .") as f:
+ myunadded = f.readlines()
+ myunadded = ["./" + elem.rstrip() for elem in myunadded]
+
+ # Mercurial doesn't handle manually deleted files as removed
from
+ # the repository, so the user need to remove them before commit,
+ # using "hg remove [FILES]"
+ with repoman_popen("hg status --no-status --deleted .") as f:
+ mydeleted = f.readlines()
+ mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
+
+ myautoadd = []
+ if myunadded:
+ for x in range(len(myunadded) - 1, -1, -1):
+ xs = myunadded[x].split("/")
+ if xs[-1] == "files":
+ print("!!! files dir is not added! Please
correct this.")
+ sys.exit(-1)
+ elif xs[-1] == "Manifest":
+ # It's a manifest... auto add
+ myautoadd += [myunadded[x]]
+ del myunadded[x]
+
+ if myunadded:
+ print(red(
+ "!!! The following files are in your local tree"
+ " but are not added to the master"))
+ print(red(
+ "!!! tree. Please remove them from the local tree"
+ " or add them to the master tree."))
+ for x in myunadded:
+ print(" ", x)
+ print()
+ print()
sys.exit(1)
- myreporoot = os.path.basename(portdir_overlay)
- myreporoot += mydir[len(portdir_overlay):]
- ##################
-
- vcs_settings = VCSSettings(options, repoman_settings)
-
- ##################
-
- repo_settings = RepoSettings(
- config_root, portdir, portdir_overlay,
- repoman_settings, vcs_settings, options, qawarnings)
-
- repoman_settings = repo_settings.repoman_settings
-
- portdb = repo_settings.portdb
- ##################
-
- if options.echangelog is None and
repo_settings.repo_config.update_changelog:
- options.echangelog = 'y'
-
- if vcs_settings.vcs is None:
- options.echangelog = 'n'
-
- # The --echangelog option causes automatic ChangeLog generation,
- # which invalidates changelog.ebuildadded and changelog.missing
- # checks.
- # Note: Some don't use ChangeLogs in distributed SCMs.
- # It will be generated on server side from scm log,
- # before package moves to the rsync server.
- # This is needed because they try to avoid merge collisions.
- # Gentoo's Council decided to always use the ChangeLog file.
- # TODO: shouldn't this just be switched on the repo, iso the VCS?
- is_echangelog_enabled = options.echangelog in ('y', 'force')
- vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
- check_changelog = not is_echangelog_enabled and
vcs_settings.vcs_is_cvs_or_svn
-
- if 'digest' in repoman_settings.features and options.digest != 'n':
- options.digest = 'y'
-
- logging.debug("vcs: %s" % (vcs_settings.vcs,))
- logging.debug("repo config: %s" % (repo_settings.repo_config,))
- logging.debug("options: %s" % (options,))
-
- # It's confusing if these warnings are displayed without the user
- # being told which profile they come from, so disable them.
- env = os.environ.copy()
- env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
-
- categories = []
- for path in repo_settings.repo_config.eclass_db.porttrees:
- categories.extend(portage.util.grabfile(
- os.path.join(path, 'profiles', 'categories')))
- repoman_settings.categories = frozenset(
- portage.util.stack_lists([categories], incremental=1))
-
- portdb.settings = repoman_settings
- # We really only need to cache the metadata that's necessary for
visibility
- # filtering. Anything else can be discarded to reduce memory
consumption.
- portdb._aux_cache_keys.clear()
- portdb._aux_cache_keys.update(
- ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
-
- qatracker = repoman_scan(repoman_settings, repo_settings, vcs_settings,
portdb, options, myreporoot, mydir, check_changelog, config_root, env)
-
- if options.mode == "manifest":
- sys.exit(dofail)
-
- # dofail will be true if we have failed in at least one non-warning
category
- dofail = 0
- # dowarn will be true if we tripped any warnings
- dowarn = 0
- # dofull will be true if we should print a "repoman full" informational
message
- dofull = options.mode != 'full'
-
- for x in qacats:
- if x not in qatracker.fails:
- continue
- dowarn = 1
- if x not in qawarnings:
- dofail = 1
-
- if dofail or \
- (dowarn and not (options.quiet or options.mode == "scan")):
- dofull = 0
-
- # Save QA output so that it can be conveniently displayed
- # in $EDITOR while the user creates a commit message.
- # Otherwise, the user would not be able to see this output
- # once the editor has taken over the screen.
- qa_output = io.StringIO()
- style_file = ConsoleStyleFile(sys.stdout)
- if options.mode == 'commit' and \
- (not commitmessage or not commitmessage.strip()):
- style_file.write_listener = qa_output
- console_writer = StyleWriter(file=style_file, maxcol=9999)
- console_writer.style_listener = style_file.new_styles
-
- f = formatter.AbstractFormatter(console_writer)
-
- format_outputs = {
- 'column': format_qa_output_column,
- 'default': format_qa_output
- }
-
- format_output = format_outputs.get(
- options.output_style, format_outputs['default'])
- format_output(f, qatracker.fails, dofull, dofail, options, qawarnings)
-
- style_file.flush()
- del console_writer, f, style_file
- qa_output = qa_output.getvalue()
- qa_output = qa_output.splitlines(True)
-
- suggest_ignore_masked = False
- suggest_include_dev = False
-
- if options.mode != 'commit':
- if dofull:
- print(bold("Note: type \"repoman full\" for a complete
listing."))
- if dowarn and not dofail:
- utilities.repoman_sez(
- "\"You're only giving me a partial QA
payment?\n"
- " I'll take it this time, but I'm
not happy.\"")
- elif not dofail:
- utilities.repoman_sez(
- "\"If everyone were like you, I'd be out of
business!\"")
- elif dofail:
- print(bad("Please fix these important QA issues
first."))
- utilities.repoman_sez(
- "\"Make your QA payment on time"
- " and you'll never see the likes of me.\"\n")
- return qatracker, qawarnings
- else:
- if dofail and can_force and options.force and not
options.pretend:
- utilities.repoman_sez(
- " \"You want to commit even with these QA
issues?\n"
- " I'll take it this time, but I'm
not happy.\"\n")
- elif dofail:
- if options.force and not can_force:
- print(bad(
- "The --force option has been disabled"
- " due to extraordinary issues."))
- print(bad("Please fix these important QA issues
first."))
- utilities.repoman_sez(
- "\"Make your QA payment on time"
- " and you'll never see the likes of me.\"\n")
- return qatracker, qawarnings
+ if vcs_settings.vcs == "hg" and mydeleted:
+ print(red(
+ "!!! The following files are removed manually"
+ " from your local tree but are not"))
+ print(red(
+ "!!! removed from the repository."
+ " Please remove them, using \"hg remove [FILES]\"."))
+ for x in mydeleted:
+ print(" ", x)
+ print()
+ print()
+ sys.exit(1)
- if options.pretend:
+ if vcs_settings.vcs == "cvs":
+ mycvstree = cvstree.getentries("./", recursive=1)
+ mychanged = cvstree.findchanged(mycvstree, recursive=1,
basedir="./")
+ mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
+ myremoved = portage.cvstree.findremoved(mycvstree, recursive=1,
basedir="./")
+ bin_blob_pattern = re.compile("^-kb$")
+ no_expansion = set(portage.cvstree.findoption(
+ mycvstree, bin_blob_pattern, recursive=1, basedir="./"))
+
+ if vcs_settings.vcs == "svn":
+ with repoman_popen("svn status") as f:
+ svnstatus = f.readlines()
+ mychanged = [
+ "./" + elem.split()[-1:][0]
+ for elem in svnstatus
+ if (elem[:1] in "MR" or elem[1:2] in "M")]
+ mynew = [
+ "./" + elem.split()[-1:][0]
+ for elem in svnstatus
+ if elem.startswith("A")]
+ myremoved = [
+ "./" + elem.split()[-1:][0]
+ for elem in svnstatus
+ if elem.startswith("D")]
+
+ # Subversion expands keywords specified in svn:keywords
properties.
+ with repoman_popen("svn propget -R svn:keywords") as f:
+ props = f.readlines()
+ expansion = dict(
+ ("./" + prop.split(" - ")[0], prop.split(" -
")[1].split())
+ for prop in props if " - " in prop)
+
+ elif vcs_settings.vcs == "git":
+ with repoman_popen(
+ "git diff-index --name-only "
+ "--relative --diff-filter=M HEAD") as f:
+ mychanged = f.readlines()
+ mychanged = ["./" + elem[:-1] for elem in mychanged]
+
+ with repoman_popen(
+ "git diff-index --name-only "
+ "--relative --diff-filter=A HEAD") as f:
+ mynew = f.readlines()
+ mynew = ["./" + elem[:-1] for elem in mynew]
+
+ with repoman_popen(
+ "git diff-index --name-only "
+ "--relative --diff-filter=D HEAD") as f:
+ myremoved = f.readlines()
+ myremoved = ["./" + elem[:-1] for elem in myremoved]
+
+ if vcs_settings.vcs == "bzr":
+ with repoman_popen("bzr status -S .") as f:
+ bzrstatus = f.readlines()
+ mychanged = [
+ "./" + elem.split()[-1:][0].split('/')[-1:][0]
+ for elem in bzrstatus
+ if elem and elem[1:2] == "M"]
+ mynew = [
+ "./" + elem.split()[-1:][0].split('/')[-1:][0]
+ for elem in bzrstatus
+ if elem and (elem[1:2] in "NK" or elem[0:1] == "R")]
+ myremoved = [
+ "./" + elem.split()[-1:][0].split('/')[-1:][0]
+ for elem in bzrstatus
+ if elem.startswith("-")]
+ myremoved = [
+ "./" + elem.split()[-3:-2][0].split('/')[-1:][0]
+ for elem in bzrstatus
+ if elem and (elem[1:2] == "K" or elem[0:1] == "R")]
+ # Bazaar expands nothing.
+
+ if vcs_settings.vcs == "hg":
+ with repoman_popen("hg status --no-status --modified .") as f:
+ mychanged = f.readlines()
+ mychanged = ["./" + elem.rstrip() for elem in mychanged]
+
+ with repoman_popen("hg status --no-status --added .") as f:
+ mynew = f.readlines()
+ mynew = ["./" + elem.rstrip() for elem in mynew]
+
+ with repoman_popen("hg status --no-status --removed .") as f:
+ myremoved = f.readlines()
+ myremoved = ["./" + elem.rstrip() for elem in myremoved]
+
+ if vcs_settings.vcs:
+ a_file_is_changed = mychanged or mynew or myremoved
+ a_file_is_deleted_hg = vcs_settings.vcs == "hg" and mydeleted
+
+ if not (a_file_is_changed or a_file_is_deleted_hg):
utilities.repoman_sez(
- "\"So, you want to play it safe. Good
call.\"\n")
-
- myunadded = []
- if vcs_settings.vcs == "cvs":
- try:
- myvcstree = portage.cvstree.getentries("./",
recursive=1)
- myunadded = portage.cvstree.findunadded(
- myvcstree, recursive=1, basedir="./")
- except SystemExit as e:
- raise # TODO propagate this
- except:
- err("Error retrieving CVS tree; exiting.")
- if vcs_settings.vcs == "svn":
- try:
- with repoman_popen("svn status --no-ignore") as
f:
- svnstatus = f.readlines()
- myunadded = [
- "./" + elem.rstrip().split()[1]
- for elem in svnstatus
- if elem.startswith("?") or
elem.startswith("I")]
- except SystemExit as e:
- raise # TODO propagate this
- except:
- err("Error retrieving SVN info; exiting.")
- if vcs_settings.vcs == "git":
- # get list of files not under version control or missing
- myf = repoman_popen("git ls-files --others")
- myunadded = ["./" + elem[:-1] for elem in myf]
- myf.close()
- if vcs_settings.vcs == "bzr":
- try:
- with repoman_popen("bzr status -S .") as f:
- bzrstatus = f.readlines()
- myunadded = [
- "./" +
elem.rstrip().split()[1].split('/')[-1:][0]
- for elem in bzrstatus
- if elem.startswith("?") or elem[0:2] ==
" D"]
- except SystemExit as e:
- raise # TODO propagate this
- except:
- err("Error retrieving bzr info; exiting.")
- if vcs_settings.vcs == "hg":
- with repoman_popen("hg status --no-status --unknown .")
as f:
- myunadded = f.readlines()
- myunadded = ["./" + elem.rstrip() for elem in myunadded]
-
- # Mercurial doesn't handle manually deleted files as
removed from
- # the repository, so the user need to remove them
before commit,
- # using "hg remove [FILES]"
- with repoman_popen("hg status --no-status --deleted .")
as f:
- mydeleted = f.readlines()
- mydeleted = ["./" + elem.rstrip() for elem in mydeleted]
-
- myautoadd = []
- if myunadded:
- for x in range(len(myunadded) - 1, -1, -1):
- xs = myunadded[x].split("/")
- if xs[-1] == "files":
- print("!!! files dir is not added!
Please correct this.")
- sys.exit(-1)
- elif xs[-1] == "Manifest":
- # It's a manifest... auto add
- myautoadd += [myunadded[x]]
- del myunadded[x]
-
- if myunadded:
- print(red(
- "!!! The following files are in your local tree"
- " but are not added to the master"))
- print(red(
- "!!! tree. Please remove them from the local
tree"
- " or add them to the master tree."))
- for x in myunadded:
- print(" ", x)
+ "\"Doing nothing is not always good for QA.\"")
print()
+ print("(Didn't find any changed files...)")
print()
sys.exit(1)
- if vcs_settings.vcs == "hg" and mydeleted:
- print(red(
- "!!! The following files are removed manually"
- " from your local tree but are not"))
- print(red(
- "!!! removed from the repository."
- " Please remove them, using \"hg remove
[FILES]\"."))
- for x in mydeleted:
- print(" ", x)
- print()
- print()
+ # Manifests need to be regenerated after all other commits, so don't
commit
+ # them now even if they have changed.
+ mymanifests = set()
+ myupdates = set()
+ for f in mychanged + mynew:
+ if "Manifest" == os.path.basename(f):
+ mymanifests.add(f)
+ else:
+ myupdates.add(f)
+ myupdates.difference_update(myremoved)
+ myupdates = list(myupdates)
+ mymanifests = list(mymanifests)
+ myheaders = []
+ mydirty = []
+
+ commitmessage = options.commitmsg
+ if options.commitmsgfile:
+ try:
+ f = io.open(
+ _unicode_encode(
+ options.commitmsgfile,
+ encoding=_encodings['fs'],
errors='strict'),
+ mode='r', encoding=_encodings['content'],
errors='replace')
+ commitmessage = f.read()
+ f.close()
+ del f
+ except (IOError, OSError) as e:
+ if e.errno == errno.ENOENT:
+ portage.writemsg(
+ "!!! File Not Found:"
+ " --commitmsgfile='%s'\n" %
options.commitmsgfile)
+ else:
+ raise
+ # We've read the content so the file is no longer needed.
+ commitmessagefile = None
+ if not commitmessage or not commitmessage.strip():
+ msg_prefix = ""
+ if repolevel > 1:
+ msg_prefix = "/".join(reposplit[1:]) + ": "
+
+ try:
+ editor = os.environ.get("EDITOR")
+ if editor and utilities.editor_is_executable(editor):
+ commitmessage =
utilities.get_commit_message_with_editor(
+ editor, message=qa_output,
prefix=msg_prefix)
+ else:
+ commitmessage =
utilities.get_commit_message_with_stdin()
+ except KeyboardInterrupt:
+ logging.fatal("Interrupted; exiting...")
sys.exit(1)
+ if (not commitmessage or not commitmessage.strip()
+ or commitmessage.strip() == msg_prefix):
+ print("* no commit message? aborting commit.")
+ sys.exit(1)
+ commitmessage = commitmessage.rstrip()
+ changelog_msg = commitmessage
+ portage_version = getattr(portage, "VERSION", None)
+ gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
+ dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
+ if portage_version is None:
+ sys.stderr.write("Failed to insert portage version in
message!\n")
+ sys.stderr.flush()
+ portage_version = "Unknown"
+
+ report_options = []
+ if options.force:
+ report_options.append("--force")
+ if options.ignore_arches:
+ report_options.append("--ignore-arches")
+ if include_arches is not None:
+ report_options.append(
+ "--include-arches=\"%s\"" %
+ " ".join(sorted(include_arches)))
+
+ if vcs_settings.vcs == "git":
+ # Use new footer only for git (see bug #438364).
+ commit_footer = "\n\nPackage-Manager: portage-%s" %
portage_version
+ if report_options:
+ commit_footer += "\nRepoMan-Options: " + "
".join(report_options)
+ if repo_settings.sign_manifests:
+ commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, )
+ if dco_sob:
+ commit_footer += "\nSigned-off-by: %s" % (dco_sob, )
+ else:
+ unameout = platform.system() + " "
+ if platform.system() in ["Darwin", "SunOS"]:
+ unameout += platform.processor()
+ else:
+ unameout += platform.machine()
+ commit_footer = "\n\n"
+ if dco_sob:
+ commit_footer += "Signed-off-by: %s\n" % (dco_sob, )
+ commit_footer += "(Portage version: %s/%s/%s" % \
+ (portage_version, vcs_settings.vcs, unameout)
+ if report_options:
+ commit_footer += ", RepoMan options: " + "
".join(report_options)
+ if repo_settings.sign_manifests:
+ commit_footer += ", signed Manifest commit with key %s"
% \
+ (gpg_key, )
+ else:
+ commit_footer += ", unsigned Manifest commit"
+ commit_footer += ")"
+
+ commitmessage += commit_footer
+
+ broken_changelog_manifests = []
+ if options.echangelog in ('y', 'force'):
+ logging.info("checking for unmodified ChangeLog files")
+ committer_name =
utilities.get_committer_name(env=repoman_settings)
+ for x in sorted(vcs_files_to_cps(
+ chain(myupdates, mymanifests, myremoved),
+ repolevel, reposplit, categories)):
+ catdir, pkgdir = x.split("/")
+ checkdir = repo_settings.repodir + "/" + x
+ checkdir_relative = ""
+ if repolevel < 3:
+ checkdir_relative = os.path.join(pkgdir,
checkdir_relative)
+ if repolevel < 2:
+ checkdir_relative = os.path.join(catdir,
checkdir_relative)
+ checkdir_relative = os.path.join(".", checkdir_relative)
+
+ changelog_path = os.path.join(checkdir_relative,
"ChangeLog")
+ changelog_modified = changelog_path in
changed.changelogs
+ if changelog_modified and options.echangelog != 'force':
+ continue
- if vcs_settings.vcs == "cvs":
- mycvstree = cvstree.getentries("./", recursive=1)
- mychanged = cvstree.findchanged(mycvstree, recursive=1,
basedir="./")
- mynew = cvstree.findnew(mycvstree, recursive=1,
basedir="./")
- myremoved = portage.cvstree.findremoved(mycvstree,
recursive=1, basedir="./")
- bin_blob_pattern = re.compile("^-kb$")
- no_expansion = set(portage.cvstree.findoption(
- mycvstree, bin_blob_pattern, recursive=1,
basedir="./"))
-
- if vcs_settings.vcs == "svn":
- with repoman_popen("svn status") as f:
- svnstatus = f.readlines()
- mychanged = [
- "./" + elem.split()[-1:][0]
- for elem in svnstatus
- if (elem[:1] in "MR" or elem[1:2] in "M")]
- mynew = [
- "./" + elem.split()[-1:][0]
- for elem in svnstatus
- if elem.startswith("A")]
- myremoved = [
- "./" + elem.split()[-1:][0]
- for elem in svnstatus
- if elem.startswith("D")]
-
- # Subversion expands keywords specified in svn:keywords
properties.
- with repoman_popen("svn propget -R svn:keywords") as f:
- props = f.readlines()
- expansion = dict(
- ("./" + prop.split(" - ")[0], prop.split(" -
")[1].split())
- for prop in props if " - " in prop)
-
- elif vcs_settings.vcs == "git":
- with repoman_popen(
- "git diff-index --name-only "
- "--relative --diff-filter=M HEAD") as f:
- mychanged = f.readlines()
- mychanged = ["./" + elem[:-1] for elem in mychanged]
-
- with repoman_popen(
- "git diff-index --name-only "
- "--relative --diff-filter=A HEAD") as f:
- mynew = f.readlines()
- mynew = ["./" + elem[:-1] for elem in mynew]
-
- with repoman_popen(
- "git diff-index --name-only "
- "--relative --diff-filter=D HEAD") as f:
- myremoved = f.readlines()
- myremoved = ["./" + elem[:-1] for elem in myremoved]
-
- if vcs_settings.vcs == "bzr":
- with repoman_popen("bzr status -S .") as f:
- bzrstatus = f.readlines()
- mychanged = [
- "./" + elem.split()[-1:][0].split('/')[-1:][0]
- for elem in bzrstatus
- if elem and elem[1:2] == "M"]
- mynew = [
- "./" + elem.split()[-1:][0].split('/')[-1:][0]
- for elem in bzrstatus
- if elem and (elem[1:2] in "NK" or elem[0:1] ==
"R")]
- myremoved = [
- "./" + elem.split()[-1:][0].split('/')[-1:][0]
- for elem in bzrstatus
- if elem.startswith("-")]
- myremoved = [
- "./" + elem.split()[-3:-2][0].split('/')[-1:][0]
- for elem in bzrstatus
- if elem and (elem[1:2] == "K" or elem[0:1] ==
"R")]
- # Bazaar expands nothing.
+ # get changes for this package
+ cdrlen = len(checkdir_relative)
+ check_relative = lambda e:
e.startswith(checkdir_relative)
+ split_relative = lambda e: e[cdrlen:]
+ clnew = list(map(split_relative, filter(check_relative,
mynew)))
+ clremoved = list(map(split_relative,
filter(check_relative, myremoved)))
+ clchanged = list(map(split_relative,
filter(check_relative, mychanged)))
+
+ # Skip ChangeLog generation if only the Manifest was
modified,
+ # as discussed in bug #398009.
+ nontrivial_cl_files = set()
+ nontrivial_cl_files.update(clnew, clremoved, clchanged)
+ nontrivial_cl_files.difference_update(['Manifest'])
+ if not nontrivial_cl_files and options.echangelog !=
'force':
+ continue
- if vcs_settings.vcs == "hg":
- with repoman_popen("hg status --no-status --modified
.") as f:
- mychanged = f.readlines()
- mychanged = ["./" + elem.rstrip() for elem in mychanged]
-
- with repoman_popen("hg status --no-status --added .")
as f:
- mynew = f.readlines()
- mynew = ["./" + elem.rstrip() for elem in mynew]
-
- with repoman_popen("hg status --no-status --removed .")
as f:
- myremoved = f.readlines()
- myremoved = ["./" + elem.rstrip() for elem in myremoved]
-
- if vcs_settings.vcs:
- a_file_is_changed = mychanged or mynew or myremoved
- a_file_is_deleted_hg = vcs_settings.vcs == "hg" and
mydeleted
-
- if not (a_file_is_changed or a_file_is_deleted_hg):
- utilities.repoman_sez(
- "\"Doing nothing is not always good for
QA.\"")
- print()
- print("(Didn't find any changed files...)")
- print()
+ new_changelog = utilities.UpdateChangeLog(
+ checkdir_relative, committer_name,
changelog_msg,
+ os.path.join(repo_settings.repodir,
'skel.ChangeLog'),
+ catdir, pkgdir,
+ new=clnew, removed=clremoved, changed=clchanged,
+ pretend=options.pretend)
+ if new_changelog is None:
+ writemsg_level(
+ "!!! Updating the ChangeLog failed\n",
+ level=logging.ERROR, noiselevel=-1)
sys.exit(1)
- # Manifests need to be regenerated after all other commits, so don't
commit
- # them now even if they have changed.
- mymanifests = set()
- myupdates = set()
- for f in mychanged + mynew:
- if "Manifest" == os.path.basename(f):
- mymanifests.add(f)
+ # if the ChangeLog was just created, add it to vcs
+ if new_changelog:
+ myautoadd.append(changelog_path)
+ # myautoadd is appended to myupdates below
else:
- myupdates.add(f)
- myupdates.difference_update(myremoved)
- myupdates = list(myupdates)
- mymanifests = list(mymanifests)
- myheaders = []
- mydirty = []
-
- commitmessage = options.commitmsg
- if options.commitmsgfile:
- try:
- f = io.open(
- _unicode_encode(
- options.commitmsgfile,
- encoding=_encodings['fs'],
errors='strict'),
- mode='r',
encoding=_encodings['content'], errors='replace')
- commitmessage = f.read()
- f.close()
- del f
- except (IOError, OSError) as e:
- if e.errno == errno.ENOENT:
- portage.writemsg(
- "!!! File Not Found:"
- " --commitmsgfile='%s'\n" %
options.commitmsgfile)
- else:
- raise
- # We've read the content so the file is no longer
needed.
- commitmessagefile = None
- if not commitmessage or not commitmessage.strip():
- try:
- editor = os.environ.get("EDITOR")
- if editor and
utilities.editor_is_executable(editor):
- commitmessage =
utilities.get_commit_message_with_editor(
- editor, message=qa_output)
- else:
- commitmessage =
utilities.get_commit_message_with_stdin()
- except KeyboardInterrupt:
- logging.fatal("Interrupted; exiting...")
- sys.exit(1)
- if not commitmessage or not commitmessage.strip():
- print("* no commit message? aborting commit.")
- sys.exit(1)
- commitmessage = commitmessage.rstrip()
- changelog_msg = commitmessage
- portage_version = getattr(portage, "VERSION", None)
- gpg_key = repoman_settings.get("PORTAGE_GPG_KEY", "")
- dco_sob = repoman_settings.get("DCO_SIGNED_OFF_BY", "")
- if portage_version is None:
- sys.stderr.write("Failed to insert portage version in
message!\n")
- sys.stderr.flush()
- portage_version = "Unknown"
-
- report_options = []
- if options.force:
- report_options.append("--force")
- if options.ignore_arches:
- report_options.append("--ignore-arches")
- if include_arches is not None:
- report_options.append(
- "--include-arches=\"%s\"" %
- " ".join(sorted(include_arches)))
-
- if vcs_settings.vcs == "git":
- # Use new footer only for git (see bug #438364).
- commit_footer = "\n\nPackage-Manager: portage-%s" %
portage_version
- if report_options:
- commit_footer += "\nRepoMan-Options: " + "
".join(report_options)
- if repo_settings.sign_manifests:
- commit_footer += "\nManifest-Sign-Key: %s" %
(gpg_key, )
- if dco_sob:
- commit_footer += "\nSigned-off-by: %s" %
(dco_sob, )
- else:
- unameout = platform.system() + " "
- if platform.system() in ["Darwin", "SunOS"]:
- unameout += platform.processor()
- else:
- unameout += platform.machine()
- commit_footer = "\n\n"
- if dco_sob:
- commit_footer += "Signed-off-by: %s\n" %
(dco_sob, )
- commit_footer += "(Portage version: %s/%s/%s" % \
- (portage_version, vcs_settings.vcs, unameout)
- if report_options:
- commit_footer += ", RepoMan options: " + "
".join(report_options)
- if repo_settings.sign_manifests:
- commit_footer += ", signed Manifest commit with
key %s" % \
- (gpg_key, )
+ myupdates.append(changelog_path)
+
+ if options.ask and not options.pretend:
+ # regenerate Manifest for modified ChangeLog
(bug #420735)
+ repoman_settings["O"] = checkdir
+ digestgen(mysettings=repoman_settings,
myportdb=portdb)
else:
- commit_footer += ", unsigned Manifest commit"
- commit_footer += ")"
+ broken_changelog_manifests.append(x)
- commitmessage += commit_footer
+ if myautoadd:
+ print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
+ add_cmd = [vcs_settings.vcs, "add"]
+ add_cmd += myautoadd
+ if options.pretend:
+ portage.writemsg_stdout(
+ "(%s)\n" % " ".join(add_cmd),
+ noiselevel=-1)
+ else:
- broken_changelog_manifests = []
- if options.echangelog in ('y', 'force'):
- logging.info("checking for unmodified ChangeLog files")
- committer_name =
utilities.get_committer_name(env=repoman_settings)
- for x in sorted(vcs_files_to_cps(
- chain(myupdates, mymanifests, myremoved),
- repolevel, reposplit, categories)):
- catdir, pkgdir = x.split("/")
- checkdir = repo_settings.repodir + "/" + x
- checkdir_relative = ""
- if repolevel < 3:
- checkdir_relative =
os.path.join(pkgdir, checkdir_relative)
- if repolevel < 2:
- checkdir_relative =
os.path.join(catdir, checkdir_relative)
- checkdir_relative = os.path.join(".",
checkdir_relative)
-
- changelog_path =
os.path.join(checkdir_relative, "ChangeLog")
- changelog_modified = changelog_path in
changed.changelogs
- if changelog_modified and options.echangelog !=
'force':
+ if sys.hexversion < 0x3020000 and sys.hexversion >=
0x3000000 and \
+ not os.path.isabs(add_cmd[0]):
+ # Python 3.1 _execvp throws TypeError for
non-absolute executable
+ # path passed as bytes (see
http://bugs.python.org/issue8513).
+ fullname = find_binary(add_cmd[0])
+ if fullname is None:
+ raise
portage.exception.CommandNotFound(add_cmd[0])
+ add_cmd[0] = fullname
+
+ add_cmd = [_unicode_encode(arg) for arg in add_cmd]
+ retcode = subprocess.call(add_cmd)
+ if retcode != os.EX_OK:
+ logging.error(
+ "Exiting on %s error code: %s\n" %
(vcs_settings.vcs, retcode))
+ sys.exit(retcode)
+
+ myupdates += myautoadd
+
+ print("* %s files being committed..." % green(str(len(myupdates))),
end=' ')
+
+ if vcs_settings.vcs not in ('cvs', 'svn'):
+ # With git, bzr and hg, there's never any keyword expansion, so
+ # there's no need to regenerate manifests and all files will be
+ # committed in one big commit at the end.
+ print()
+ elif not repo_settings.repo_config.thin_manifest:
+ if vcs_settings.vcs == 'cvs':
+ headerstring = "'\$(Header|Id).*\$'"
+ elif vcs_settings.vcs == "svn":
+ svn_keywords = dict((k.lower(), k) for k in [
+ "Rev",
+ "Revision",
+ "LastChangedRevision",
+ "Date",
+ "LastChangedDate",
+ "Author",
+ "LastChangedBy",
+ "URL",
+ "HeadURL",
+ "Id",
+ "Header",
+ ])
+
+ for myfile in myupdates:
+
+ # for CVS, no_expansion contains files that are
excluded from expansion
+ if vcs_settings.vcs == "cvs":
+ if myfile in no_expansion:
continue
- # get changes for this package
- cdrlen = len(checkdir_relative)
- check_relative = lambda e:
e.startswith(checkdir_relative)
- split_relative = lambda e: e[cdrlen:]
- clnew = list(map(split_relative,
filter(check_relative, mynew)))
- clremoved = list(map(split_relative,
filter(check_relative, myremoved)))
- clchanged = list(map(split_relative,
filter(check_relative, mychanged)))
-
- # Skip ChangeLog generation if only the
Manifest was modified,
- # as discussed in bug #398009.
- nontrivial_cl_files = set()
- nontrivial_cl_files.update(clnew, clremoved,
clchanged)
-
nontrivial_cl_files.difference_update(['Manifest'])
- if not nontrivial_cl_files and
options.echangelog != 'force':
+ # for SVN, expansion contains files that are included
in expansion
+ elif vcs_settings.vcs == "svn":
+ if myfile not in expansion:
continue
- new_changelog = utilities.UpdateChangeLog(
- checkdir_relative, committer_name,
changelog_msg,
- os.path.join(repo_settings.repodir,
'skel.ChangeLog'),
- catdir, pkgdir,
- new=clnew, removed=clremoved,
changed=clchanged,
- pretend=options.pretend)
- if new_changelog is None:
- writemsg_level(
- "!!! Updating the ChangeLog
failed\n",
- level=logging.ERROR,
noiselevel=-1)
- sys.exit(1)
+ # Subversion keywords are case-insensitive
+ # in svn:keywords properties,
+ # but case-sensitive in contents of files.
+ enabled_keywords = []
+ for k in expansion[myfile]:
+ keyword = svn_keywords.get(k.lower())
+ if keyword is not None:
+ enabled_keywords.append(keyword)
+
+ headerstring = "'\$(%s).*\$'" %
"|".join(enabled_keywords)
+
+ myout = repoman_getstatusoutput(
+ "egrep -q %s %s" % (headerstring,
portage._shell_quote(myfile)))
+ if myout[0] == 0:
+ myheaders.append(myfile)
+
+ print("%s have headers that will change." %
green(str(len(myheaders))))
+ print(
+ "* Files with headers will"
+ " cause the manifests to be changed and committed
separately.")
+
+ logging.info("myupdates: %s", myupdates)
+ logging.info("myheaders: %s", myheaders)
+
+ uq = UserQuery(options)
+ if options.ask and uq.query('Commit changes?', True) != 'Yes':
+ print("* aborting commit.")
+ sys.exit(128 + signal.SIGINT)
+
+ # Handle the case where committed files have keywords which
+ # will change and need a priming commit before the Manifest
+ # can be committed.
+ if (myupdates or myremoved) and myheaders:
+ myfiles = myupdates + myremoved
+ fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
+ mymsg = os.fdopen(fd, "wb")
+ mymsg.write(_unicode_encode(commitmessage))
+ mymsg.close()
+
+ separator = '-' * 78
- # if the ChangeLog was just created, add it to
vcs
- if new_changelog:
- myautoadd.append(changelog_path)
- # myautoadd is appended to myupdates
below
- else:
- myupdates.append(changelog_path)
+ print()
+ print(green("Using commit message:"))
+ print(green(separator))
+ print(commitmessage)
+ print(green(separator))
+ print()
- if options.ask and not options.pretend:
- # regenerate Manifest for modified
ChangeLog (bug #420735)
- repoman_settings["O"] = checkdir
- digestgen(mysettings=repoman_settings,
myportdb=portdb)
- else:
- broken_changelog_manifests.append(x)
+ # Having a leading ./ prefix on file paths can trigger a bug in
+ # the cvs server when committing files to multiple directories,
+ # so strip the prefix.
+ myfiles = [f.lstrip("./") for f in myfiles]
+
+ commit_cmd = [vcs_settings.vcs]
+ commit_cmd.extend(vcs_settings.vcs_global_opts)
+ commit_cmd.append("commit")
+ commit_cmd.extend(vcs_settings.vcs_local_opts)
+ commit_cmd.extend(["-F", commitmessagefile])
+ commit_cmd.extend(myfiles)
- if myautoadd:
- print(">>> Auto-Adding missing Manifest/ChangeLog
file(s)...")
- add_cmd = [vcs_settings.vcs, "add"]
- add_cmd += myautoadd
+ try:
if options.pretend:
- portage.writemsg_stdout(
- "(%s)\n" % " ".join(add_cmd),
- noiselevel=-1)
+ print("(%s)" % (" ".join(commit_cmd),))
else:
+ retval = spawn(commit_cmd,
env=repo_settings.commit_env)
+ if retval != os.EX_OK:
+ writemsg_level(
+ "!!! Exiting on %s (shell) "
+ "error code: %s\n" %
(vcs_settings.vcs, retval),
+ level=logging.ERROR,
noiselevel=-1)
+ sys.exit(retval)
+ finally:
+ try:
+ os.unlink(commitmessagefile)
+ except OSError:
+ pass
- if sys.hexversion < 0x3020000 and
sys.hexversion >= 0x3000000 and \
- not os.path.isabs(add_cmd[0]):
- # Python 3.1 _execvp throws TypeError
for non-absolute executable
- # path passed as bytes (see
http://bugs.python.org/issue8513).
- fullname = find_binary(add_cmd[0])
- if fullname is None:
- raise
portage.exception.CommandNotFound(add_cmd[0])
- add_cmd[0] = fullname
-
- add_cmd = [_unicode_encode(arg) for arg in
add_cmd]
- retcode = subprocess.call(add_cmd)
- if retcode != os.EX_OK:
- logging.error(
- "Exiting on %s error code:
%s\n" % (vcs_settings.vcs, retcode))
- sys.exit(retcode)
-
- myupdates += myautoadd
-
- print("* %s files being committed..." %
green(str(len(myupdates))), end=' ')
-
- if vcs_settings.vcs not in ('cvs', 'svn'):
- # With git, bzr and hg, there's never any keyword
expansion, so
- # there's no need to regenerate manifests and all files
will be
- # committed in one big commit at the end.
- print()
- elif not repo_settings.repo_config.thin_manifest:
- if vcs_settings.vcs == 'cvs':
- headerstring = "'\$(Header|Id).*\$'"
- elif vcs_settings.vcs == "svn":
- svn_keywords = dict((k.lower(), k) for k in [
- "Rev",
- "Revision",
- "LastChangedRevision",
- "Date",
- "LastChangedDate",
- "Author",
- "LastChangedBy",
- "URL",
- "HeadURL",
- "Id",
- "Header",
- ])
-
- for myfile in myupdates:
-
- # for CVS, no_expansion contains files that are
excluded from expansion
- if vcs_settings.vcs == "cvs":
- if myfile in no_expansion:
- continue
-
- # for SVN, expansion contains files that are
included in expansion
- elif vcs_settings.vcs == "svn":
- if myfile not in expansion:
- continue
-
- # Subversion keywords are
case-insensitive
- # in svn:keywords properties,
- # but case-sensitive in contents of
files.
- enabled_keywords = []
- for k in expansion[myfile]:
- keyword =
svn_keywords.get(k.lower())
- if keyword is not None:
-
enabled_keywords.append(keyword)
-
- headerstring = "'\$(%s).*\$'" %
"|".join(enabled_keywords)
-
- myout = repoman_getstatusoutput(
- "egrep -q %s %s" % (headerstring,
portage._shell_quote(myfile)))
- if myout[0] == 0:
- myheaders.append(myfile)
-
- print("%s have headers that will change." %
green(str(len(myheaders))))
- print(
- "* Files with headers will"
- " cause the manifests to be changed and
committed separately.")
-
- logging.info("myupdates: %s", myupdates)
- logging.info("myheaders: %s", myheaders)
-
- uq = UserQuery(options)
- if options.ask and uq.query('Commit changes?', True) != 'Yes':
- print("* aborting commit.")
- sys.exit(128 + signal.SIGINT)
-
- # Handle the case where committed files have keywords which
- # will change and need a priming commit before the Manifest
- # can be committed.
- if (myupdates or myremoved) and myheaders:
- myfiles = myupdates + myremoved
- fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
- mymsg = os.fdopen(fd, "wb")
- mymsg.write(_unicode_encode(commitmessage))
- mymsg.close()
-
- separator = '-' * 78
-
- print()
- print(green("Using commit message:"))
- print(green(separator))
- print(commitmessage)
- print(green(separator))
- print()
+ # Setup the GPG commands
+ def gpgsign(filename):
+ gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
+ if gpgcmd in [None, '']:
+ raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is
unset!"
+ " Is make.globals missing?")
+ if "${PORTAGE_GPG_KEY}" in gpgcmd and \
+ "PORTAGE_GPG_KEY" not in repoman_settings:
+ raise MissingParameter("PORTAGE_GPG_KEY is unset!")
+ if "${PORTAGE_GPG_DIR}" in gpgcmd:
+ if "PORTAGE_GPG_DIR" not in repoman_settings:
+ repoman_settings["PORTAGE_GPG_DIR"] = \
+ os.path.expanduser("~/.gnupg")
+ logging.info(
+ "Automatically setting PORTAGE_GPG_DIR
to '%s'" %
+ repoman_settings["PORTAGE_GPG_DIR"])
+ else:
+ repoman_settings["PORTAGE_GPG_DIR"] = \
+
os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"])
+ if not os.access(repoman_settings["PORTAGE_GPG_DIR"],
os.X_OK):
+ raise portage.exception.InvalidLocation(
+ "Unable to access directory:
PORTAGE_GPG_DIR='%s'" %
+ repoman_settings["PORTAGE_GPG_DIR"])
+ gpgvars = {"FILE": filename}
+ for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
+ v = repoman_settings.get(k)
+ if v is not None:
+ gpgvars[k] = v
+ gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
+ if options.pretend:
+ print("(" + gpgcmd + ")")
+ else:
+ # Encode unicode manually for bug #310789.
+ gpgcmd = portage.util.shlex_split(gpgcmd)
+
+ if sys.hexversion < 0x3020000 and sys.hexversion >=
0x3000000 and \
+ not os.path.isabs(gpgcmd[0]):
+ # Python 3.1 _execvp throws TypeError for
non-absolute executable
+ # path passed as bytes (see
http://bugs.python.org/issue8513).
+ fullname = find_binary(gpgcmd[0])
+ if fullname is None:
+ raise
portage.exception.CommandNotFound(gpgcmd[0])
+ gpgcmd[0] = fullname
+
+ gpgcmd = [
+ _unicode_encode(arg, encoding=_encodings['fs'],
errors='strict')
+ for arg in gpgcmd]
+ rValue = subprocess.call(gpgcmd)
+ if rValue == os.EX_OK:
+ os.rename(filename + ".asc", filename)
+ else:
+ raise portage.exception.PortageException(
+ "!!! gpg exited with '" + str(rValue) +
"' status")
- # Having a leading ./ prefix on file paths can trigger
a bug in
- # the cvs server when committing files to multiple
directories,
- # so strip the prefix.
- myfiles = [f.lstrip("./") for f in myfiles]
+ def need_signature(filename):
+ try:
+ with open(
+ _unicode_encode(
+ filename, encoding=_encodings['fs'],
errors='strict'),
+ 'rb') as f:
+ return b"BEGIN PGP SIGNED MESSAGE" not in
f.readline()
+ except IOError as e:
+ if e.errno in (errno.ENOENT, errno.ESTALE):
+ return False
+ raise
- commit_cmd = [vcs_settings.vcs]
- commit_cmd.extend(vcs_settings.vcs_global_opts)
- commit_cmd.append("commit")
- commit_cmd.extend(vcs_settings.vcs_local_opts)
- commit_cmd.extend(["-F", commitmessagefile])
- commit_cmd.extend(myfiles)
+ # When files are removed and re-added, the cvs server will put /Attic/
+ # inside the $Header path. This code detects the problem and corrects it
+ # so that the Manifest will generate correctly. See bug #169500.
+ # Use binary mode in order to avoid potential character encoding issues.
+ cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
+ attic_str = b'/Attic/'
+ attic_replace = b'/'
+ for x in myheaders:
+ f = open(
+ _unicode_encode(x, encoding=_encodings['fs'],
errors='strict'),
+ mode='rb')
+ mylines = f.readlines()
+ f.close()
+ modified = False
+ for i, line in enumerate(mylines):
+ if cvs_header_re.match(line) is not None and \
+ attic_str in line:
+ mylines[i] = line.replace(attic_str,
attic_replace)
+ modified = True
+ if modified:
+ portage.util.write_atomic(x, b''.join(mylines),
mode='wb')
- try:
- if options.pretend:
- print("(%s)" % (" ".join(commit_cmd),))
- else:
- retval = spawn(commit_cmd,
env=repo_settings.commit_env)
- if retval != os.EX_OK:
- writemsg_level(
- "!!! Exiting on %s
(shell) "
- "error code: %s\n" %
(vcs_settings.vcs, retval),
- level=logging.ERROR,
noiselevel=-1)
- sys.exit(retval)
- finally:
- try:
- os.unlink(commitmessagefile)
- except OSError:
- pass
-
- # When files are removed and re-added, the cvs server will put
/Attic/
- # inside the $Header path. This code detects the problem and
corrects it
- # so that the Manifest will generate correctly. See bug #169500.
- # Use binary mode in order to avoid potential character
encoding issues.
- cvs_header_re = re.compile(br'^#\s*\$Header.*\$$')
- attic_str = b'/Attic/'
- attic_replace = b'/'
- for x in myheaders:
- f = open(
- _unicode_encode(x, encoding=_encodings['fs'],
errors='strict'),
- mode='rb')
- mylines = f.readlines()
- f.close()
- modified = False
- for i, line in enumerate(mylines):
- if cvs_header_re.match(line) is not None and \
- attic_str in line:
- mylines[i] = line.replace(attic_str,
attic_replace)
- modified = True
- if modified:
- portage.util.write_atomic(x, b''.join(mylines),
mode='wb')
-
- if repolevel == 1:
- utilities.repoman_sez(
- "\"You're rather crazy... "
- "doing the entire repository.\"\n")
-
- if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or
myremoved):
+ if repolevel == 1:
+ utilities.repoman_sez(
+ "\"You're rather crazy... "
+ "doing the entire repository.\"\n")
+
+ if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or myremoved):
+ for x in sorted(vcs_files_to_cps(
+ chain(myupdates, myremoved, mymanifests),
+ repolevel, reposplit, categories)):
+ repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
+ digestgen(mysettings=repoman_settings, myportdb=portdb)
+
+ elif broken_changelog_manifests:
+ for x in broken_changelog_manifests:
+ repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
+ digestgen(mysettings=repoman_settings, myportdb=portdb)
+
+ signed = False
+ if repo_settings.sign_manifests:
+ signed = True
+ try:
for x in sorted(vcs_files_to_cps(
chain(myupdates, myremoved, mymanifests),
repolevel, reposplit, categories)):
repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
- digestgen(mysettings=repoman_settings,
myportdb=portdb)
-
- elif broken_changelog_manifests:
- for x in broken_changelog_manifests:
- repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
- digestgen(mysettings=repoman_settings,
myportdb=portdb)
+ manifest_path =
os.path.join(repoman_settings["O"], "Manifest")
+ if not need_signature(manifest_path):
+ continue
+ gpgsign(manifest_path)
+ except portage.exception.PortageException as e:
+ portage.writemsg("!!! %s\n" % str(e))
+ portage.writemsg("!!! Disabled FEATURES='sign'\n")
+ signed = False
+
+ if vcs_settings.vcs == 'git':
+ # It's not safe to use the git commit -a option since there
might
+ # be some modified files elsewhere in the working tree that the
+ # user doesn't want to commit. Therefore, call git update-index
+ # in order to ensure that the index is updated with the latest
+ # versions of all new and modified files in the relevant portion
+ # of the working tree.
+ myfiles = mymanifests + myupdates
+ myfiles.sort()
+ update_index_cmd = ["git", "update-index"]
+ update_index_cmd.extend(f.lstrip("./") for f in myfiles)
+ if options.pretend:
+ print("(%s)" % (" ".join(update_index_cmd),))
+ else:
+ retval = spawn(update_index_cmd, env=os.environ)
+ if retval != os.EX_OK:
+ writemsg_level(
+ "!!! Exiting on %s (shell) "
+ "error code: %s\n" % (vcs_settings.vcs,
retval),
+ level=logging.ERROR, noiselevel=-1)
+ sys.exit(retval)
+
+ if True:
+ myfiles = mymanifests[:]
+ # If there are no header (SVN/CVS keywords) changes in
+ # the files, this Manifest commit must include the
+ # other (yet uncommitted) files.
+ if not myheaders:
+ myfiles += myupdates
+ myfiles += myremoved
+ myfiles.sort()
+
+ fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
+ mymsg = os.fdopen(fd, "wb")
+ mymsg.write(_unicode_encode(commitmessage))
+ mymsg.close()
+
+ commit_cmd = []
+ if options.pretend and vcs_settings.vcs is None:
+ # substitute a bogus value for pretend output
+ commit_cmd.append("cvs")
+ else:
+ commit_cmd.append(vcs_settings.vcs)
+ commit_cmd.extend(vcs_settings.vcs_global_opts)
+ commit_cmd.append("commit")
+ commit_cmd.extend(vcs_settings.vcs_local_opts)
+ if vcs_settings.vcs == "hg":
+ commit_cmd.extend(["--logfile", commitmessagefile])
+ commit_cmd.extend(myfiles)
+ else:
+ commit_cmd.extend(["-F", commitmessagefile])
+ commit_cmd.extend(f.lstrip("./") for f in myfiles)
- signed = False
- if repo_settings.sign_manifests:
- signed = True
- try:
- for x in sorted(vcs_files_to_cps(
- chain(myupdates, myremoved,
mymanifests),
- repolevel, reposplit, categories)):
- repoman_settings["O"] =
os.path.join(repo_settings.repodir, x)
- manifest_path =
os.path.join(repoman_settings["O"], "Manifest")
- if not need_signature(manifest_path):
- continue
- gpgsign(manifest_path)
- except portage.exception.PortageException as e:
- portage.writemsg("!!! %s\n" % str(e))
- portage.writemsg("!!! Disabled
FEATURES='sign'\n")
- signed = False
-
- if vcs_settings.vcs == 'git':
- # It's not safe to use the git commit -a option since
there might
- # be some modified files elsewhere in the working tree
that the
- # user doesn't want to commit. Therefore, call git
update-index
- # in order to ensure that the index is updated with the
latest
- # versions of all new and modified files in the
relevant portion
- # of the working tree.
- myfiles = mymanifests + myupdates
- myfiles.sort()
- update_index_cmd = ["git", "update-index"]
- update_index_cmd.extend(f.lstrip("./") for f in myfiles)
+ try:
if options.pretend:
- print("(%s)" % (" ".join(update_index_cmd),))
+ print("(%s)" % (" ".join(commit_cmd),))
else:
- retval = spawn(update_index_cmd, env=os.environ)
+ retval = spawn(commit_cmd,
env=repo_settings.commit_env)
if retval != os.EX_OK:
+ if
repo_settings.repo_config.sign_commit and vcs_settings.vcs == 'git' and \
+ not git_supports_gpg_sign():
+ # Inform user that newer git is
needed (bug #403323).
+ logging.error(
+ "Git >=1.7.9 is
required for signed commits!")
+
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" %
(vcs_settings.vcs, retval),
level=logging.ERROR,
noiselevel=-1)
sys.exit(retval)
-
- if True:
- myfiles = mymanifests[:]
- # If there are no header (SVN/CVS keywords) changes in
- # the files, this Manifest commit must include the
- # other (yet uncommitted) files.
- if not myheaders:
- myfiles += myupdates
- myfiles += myremoved
- myfiles.sort()
-
- fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
- mymsg = os.fdopen(fd, "wb")
- mymsg.write(_unicode_encode(commitmessage))
- mymsg.close()
-
- commit_cmd = []
- if options.pretend and vcs_settings.vcs is None:
- # substitute a bogus value for pretend output
- commit_cmd.append("cvs")
- else:
- commit_cmd.append(vcs_settings.vcs)
- commit_cmd.extend(vcs_settings.vcs_global_opts)
- commit_cmd.append("commit")
- commit_cmd.extend(vcs_settings.vcs_local_opts)
- if vcs_settings.vcs == "hg":
- commit_cmd.extend(["--logfile",
commitmessagefile])
- commit_cmd.extend(myfiles)
- else:
- commit_cmd.extend(["-F", commitmessagefile])
- commit_cmd.extend(f.lstrip("./") for f in
myfiles)
-
+ finally:
try:
- if options.pretend:
- print("(%s)" % (" ".join(commit_cmd),))
- else:
- retval = spawn(commit_cmd,
env=repo_settings.commit_env)
- if retval != os.EX_OK:
- if
repo_settings.repo_config.sign_commit and vcs_settings.vcs == 'git' and \
- not
git_supports_gpg_sign():
- # Inform user that
newer git is needed (bug #403323).
- logging.error(
- "Git >=1.7.9 is
required for signed commits!")
-
- writemsg_level(
- "!!! Exiting on %s
(shell) "
- "error code: %s\n" %
(vcs_settings.vcs, retval),
- level=logging.ERROR,
noiselevel=-1)
- sys.exit(retval)
- finally:
- try:
- os.unlink(commitmessagefile)
- except OSError:
- pass
+ os.unlink(commitmessagefile)
+ except OSError:
+ pass
- print()
- if vcs_settings.vcs:
- print("Commit complete.")
- else:
- print(
- "repoman was too scared"
- " by not seeing any familiar version control
file"
- " that he forgot to commit anything")
- utilities.repoman_sez(
- "\"If everyone were like you, I'd be out of
business!\"\n")
- return qatracker, qawarnings
+ print()
+ if vcs_settings.vcs:
+ print("Commit complete.")
+ else:
+ print(
+ "repoman was too scared"
+ " by not seeing any familiar version control file"
+ " that he forgot to commit anything")
+ utilities.repoman_sez(
+ "\"If everyone were like you, I'd be out of business!\"\n")
+sys.exit(0)
