commit: 801894358cc5569b03844bcd043931d441fb8508
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 30 11:36:57 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jul 30 11:36:57 2015 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=80189435
add repoman full scan and split repoman/main to more functions
tbc/pym/package.py | 9 +-
tbc/pym/qachecks.py | 79 ++++++------
tbc/pym/repoman/main.py | 278 +++++++++++++++++++++----------------------
tbc/pym/repoman/utilities.py | 4 +-
4 files changed, 191 insertions(+), 179 deletions(-)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index ba58e2e..0a19f15 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -8,7 +8,7 @@ from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman
+from tbc.qachecks import digestcheck, check_repoman, repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list,
add_old_ebuild, \
get_package_metadata_sql, update_package_metadata, update_manifest_sql,
\
@@ -282,6 +282,10 @@ class tbc_package(object):
manifest_checksum_tree =
self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
if manifest_checksum_tree is None:
return
+
+ # Check cp with repoman repoman full
+ repoman_full(self._session, pkgdir, self._config_id)
+
package_id = add_new_package_sql(self._session, cp, repo)
package_metadataDict = self.get_package_metadataDict(pkgdir,
package_id)
@@ -336,6 +340,9 @@ class tbc_package(object):
log_msg = "U %s:%s" % (cp, repo)
add_logs(self._session, log_msg, "info",
self._config_id)
+ # Check cp with repoman repoman full
+ repoman_full(self._session, pkgdir, self._config_id)
+
# Get the ebuild list for cp
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp,
use_cache=1, mytree=mytree)
diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index 8c522c6..2e25d86 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -15,6 +15,8 @@ from portage.exception import DigestException, FileNotFound,
ParseError, Permiss
from _emerge.Package import Package
from _emerge.RootConfig import RootConfig
from tbc.repoman.checks.ebuilds.checks import run_checks
+from tbc.repoman.main import repoman_main
+from tbc.sqlquerys import get_configmetadata_info, get_config_info,
get_setup_info
import portage
# Copy of portage.digestcheck() but without the writemsg() stuff
@@ -142,38 +144,47 @@ def check_file_in_manifest(pkgdir, mysettings, portdb,
cpv, build_use_flags_list
return
def check_repoman(mysettings, myportdb, cpv, repo):
- # We run repoman run_checks on the ebuild
- ebuild_version_tree = portage.versions.cpv_getversion(cpv)
- element = portage.versions.cpv_getkey(cpv).split('/')
- categories = element[0]
- package = element[1]
- pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories +
"/" + package
- full_path = pkgdir + "/" + package + "-" + ebuild_version_tree
+ ".ebuild"
- root = '/'
- trees = {
- root : {'porttree' : portage.portagetree(root,
settings=mysettings)}
- }
- root_config = RootConfig(mysettings, trees[root], None)
- allvars = set(x for x in portage.auxdbkeys if not
x.startswith("UNUSED_"))
- allvars.update(Package.metadata_keys)
- allvars = sorted(allvars)
- myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars,
myrepo=repo)))
- pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config,
type_name='ebuild')
- fails = []
+ # We run repoman run_checks on the ebuild
+ ebuild_version_tree = portage.versions.cpv_getversion(cpv)
+ element = portage.versions.cpv_getkey(cpv).split('/')
+ categories = element[0]
+ package = element[1]
+ pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" +
package
+ full_path = pkgdir + "/" + package + "-" + ebuild_version_tree +
".ebuild"
+ root = '/'
+ trees = {
+ root : {'porttree' : portage.portagetree(root, settings=mysettings)}
+ }
+ root_config = RootConfig(mysettings, trees[root], None)
+ allvars = set(x for x in portage.auxdbkeys if not
x.startswith("UNUSED_"))
+ allvars.update(Package.metadata_keys)
+ allvars = sorted(allvars)
+ myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars, myrepo=repo)))
+ pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config,
type_name='ebuild')
+ fails = []
+ try:
+ # All ebuilds should have utf_8 encoding.
+ f = codecs.open(_unicode_encode(full_path,
+ encoding = _encodings['fs'], errors = 'strict'),
+ mode = 'r', encoding = _encodings['repo.content'])
try:
- # All ebuilds should have utf_8 encoding.
- f = codecs.open(_unicode_encode(full_path,
- encoding = _encodings['fs'], errors = 'strict'),
- mode = 'r', encoding = _encodings['repo.content'])
- try:
- for check_name, e in run_checks(f, pkg):
- fails.append(check_name + ": " + e +
"\n")
- finally:
- f.close()
- except UnicodeDecodeError:
- # A file.UTF8 failure will have already been recorded
above.
- pass
- # fails will have a list with repoman errors
- if fails == []:
- return False
- return fails
\ No newline at end of file
+ for check_name, e in run_checks(f, pkg):
+ fails.append(check_name + ": " + e + "\n")
+ finally:
+ f.close()
+ except UnicodeDecodeError:
+ # A file.UTF8 failure will have already been recorded above.
+ pass
+ # fails will have a list with repoman errors
+ if fails == []:
+ return False
+ return fails
+
+def repoman_full(session, pkgdir, config_id):
+ ConfigsMetaData = get_configmetadata_info(session, config_id)
+ ConfigInfo = get_config_info(session, config_id)
+ SetupInfo = get_setup_info(session, config_id)
+ config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname +
"/" + SetupInfo.Setup
+ argscmd = []
+ argscmd.append('full')
+ repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py
index 2196009..5bdee0e 100755
--- a/tbc/pym/repoman/main.py
+++ b/tbc/pym/repoman/main.py
@@ -1,5 +1,4 @@
-#!/usr/bin/python -bO
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function, unicode_literals
@@ -165,113 +164,16 @@ def need_signature(filename):
return False
raise
-def repoman_main(argv):
- # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
- # behave incrementally.
- repoman_incrementals = tuple(
- x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
- config_root = os.environ.get("PORTAGE_CONFIGROOT")
- repoman_settings = portage.config(config_root=config_root,
local_config=False)
-
- if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
- repoman_settings.get('TERM') == 'dumb' or \
- not sys.stdout.isatty():
- nocolor()
-
- options, arguments = parse_args(
- argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
-
- if options.version:
- print("Portage", portage.VERSION)
- sys.exit(0)
-
- if options.experimental_inherit == 'y':
- # This is experimental, so it's non-fatal.
- qawarnings.add("inherit.missing")
- checks_init(experimental_inherit=True)
-
- # Set this to False when an extraordinary issue (generally
- # something other than a QA issue) makes it impossible to
- # commit (like if Manifest generation fails).
- can_force = True
-
- portdir, portdir_overlay, mydir =
utilities.FindPortdir(repoman_settings)
- if portdir is None:
- sys.exit(1)
-
- myreporoot = os.path.basename(portdir_overlay)
- myreporoot += mydir[len(portdir_overlay):]
- ##################
-
- vcs_settings = VCSSettings(options, repoman_settings)
-
- ##################
-
- repo_settings = RepoSettings(
- config_root, portdir, portdir_overlay,
- repoman_settings, vcs_settings, options, qawarnings)
-
- repoman_settings = repo_settings.repoman_settings
-
- portdb = repo_settings.portdb
- ##################
-
- if options.echangelog is None and
repo_settings.repo_config.update_changelog:
- options.echangelog = 'y'
-
- if vcs_settings.vcs is None:
- options.echangelog = 'n'
-
- # The --echangelog option causes automatic ChangeLog generation,
- # which invalidates changelog.ebuildadded and changelog.missing
- # checks.
- # Note: Some don't use ChangeLogs in distributed SCMs.
- # It will be generated on server side from scm log,
- # before package moves to the rsync server.
- # This is needed because they try to avoid merge collisions.
- # Gentoo's Council decided to always use the ChangeLog file.
- # TODO: shouldn't this just be switched on the repo, iso the VCS?
- is_echangelog_enabled = options.echangelog in ('y', 'force')
- vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
- check_changelog = not is_echangelog_enabled and
vcs_settings.vcs_is_cvs_or_svn
+def repoman_scan(repoman_settings, repo_settings, vcs_settings, options,
repolevel):
- if 'digest' in repoman_settings.features and options.digest != 'n':
- options.digest = 'y'
-
- logging.debug("vcs: %s" % (vcs_settings.vcs,))
- logging.debug("repo config: %s" % (repo_settings.repo_config,))
- logging.debug("options: %s" % (options,))
-
- # It's confusing if these warnings are displayed without the user
- # being told which profile they come from, so disable them.
- env = os.environ.copy()
- env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
-
- categories = []
- for path in repo_settings.repo_config.eclass_db.porttrees:
- categories.extend(portage.util.grabfile(
- os.path.join(path, 'profiles', 'categories')))
- repoman_settings.categories = frozenset(
- portage.util.stack_lists([categories], incremental=1))
- categories = repoman_settings.categories
-
- portdb.settings = repoman_settings
- # We really only need to cache the metadata that's necessary for
visibility
- # filtering. Anything else can be discarded to reduce memory
consumption.
- portdb._aux_cache_keys.clear()
- portdb._aux_cache_keys.update(
- ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
-
- reposplit = myreporoot.split(os.path.sep)
- repolevel = len(reposplit)
-
- ###################
-
- if options.mode == 'commit':
- repochecks.commit_check(repolevel, reposplit)
- repochecks.conflict_check(vcs_settings, options)
+ if options.mode == "manifest":
+ pass
+ elif options.pretend:
+ print(green("\nRepoMan does a once-over of the
neighborhood..."))
+ else:
+ print(green("\nRepoMan scours the neighborhood..."))
- ###################
+ qatracker = QATracker()
# Make startdir relative to the canonical repodir, so that we can pass
# it to digestgen and it won't have to be canonicalized again.
@@ -281,7 +183,9 @@ def repoman_main(argv):
startdir = normalize_path(mydir)
startdir = os.path.join(
repo_settings.repodir, *startdir.split(os.sep)[-2 -
repolevel + 3:])
- ###################
+
+ have_pmasked = False
+ have_dev_keywords = False
# get lists of valid keywords, licenses, and use
new_data = repo_metadata(repo_settings.portdb, repoman_settings)
@@ -305,28 +209,12 @@ def repoman_main(argv):
####################
- dev_keywords = dev_keywords(profiles)
-
- qatracker = QATracker()
-
-
- if options.mode == "manifest":
- pass
- elif options.pretend:
- print(green("\nRepoMan does a once-over of the
neighborhood..."))
- else:
- print(green("\nRepoMan scours the neighborhood..."))
-
- #####################
-
- changed = Changes(options)
- changed.scan(vcs_settings)
-
- ######################
+ dev_keyword = dev_keywords(profiles)
- have_pmasked = False
- have_dev_keywords = False
- dofail = 0
+ # Disable the "ebuild.notadded" check when not in commit mode and
+ # running `svn status` in every package dir will be too expensive.
+ check_ebuild_notadded = not \
+ (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode
!= "commit")
# NOTE: match-all caches are not shared due to potential
# differences between profiles in _get_implicit_iuse.
@@ -339,19 +227,6 @@ def repoman_main(argv):
include_arches = set()
include_arches.update(*[x.split() for x in
options.include_arches])
- # Disable the "ebuild.notadded" check when not in commit mode and
- # running `svn status` in every package dir will be too expensive.
-
- check_ebuild_notadded = not \
- (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode
!= "commit")
-
- effective_scanlist = scanlist
- if options.if_modified == "y":
- effective_scanlist = sorted(vcs_files_to_cps(
- chain(changed.changed, changed.new, changed.removed),
- repolevel, reposplit, categories))
-
- ######################
# initialize our checks classes here before the big xpkg loop
manifester = Manifests(options, qatracker, repoman_settings)
is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
@@ -814,7 +689,7 @@ def repoman_main(argv):
if not have_dev_keywords:
have_dev_keywords = \
-
bool(dev_keywords.intersection(keywords))
+
bool(dev_keyword.intersection(keywords))
if prof.status == "dev":
suffix = suffix + "indev"
@@ -890,6 +765,123 @@ def repoman_main(argv):
"metadata.warning",
"%s/metadata.xml: unused local
USE-description: '%s'"
% (xpkg, myflag))
+ return qatracker
+
+def repoman_main(argv, config_root=None, pkgdir=None):
+ # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
+ # behave incrementally.
+ repoman_incrementals = tuple(
+ x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
+ if config_root is None:
+ config_root = os.environ.get("PORTAGE_CONFIGROOT")
+ repoman_settings = portage.config(config_root=config_root,
local_config=False)
+
+ if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
+ repoman_settings.get('TERM') == 'dumb' or \
+ not sys.stdout.isatty():
+ nocolor()
+
+ options, arguments = parse_args(
+ argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
+
+ if options.version:
+ print("Portage", portage.VERSION)
+ sys.exit(0)
+
+ if options.experimental_inherit == 'y':
+ # This is experimental, so it's non-fatal.
+ qawarnings.add("inherit.missing")
+ checks_init(experimental_inherit=True)
+
+ # Set this to False when an extraordinary issue (generally
+ # something other than a QA issue) makes it impossible to
+ # commit (like if Manifest generation fails).
+ can_force = True
+
+ portdir, portdir_overlay, mydir =
utilities.FindPortdir(repoman_settings, pkgdir)
+ if portdir is None:
+ sys.exit(1)
+
+ myreporoot = os.path.basename(portdir_overlay)
+ myreporoot += mydir[len(portdir_overlay):]
+ ##################
+
+ vcs_settings = VCSSettings(options, repoman_settings)
+
+ ##################
+
+ repo_settings = RepoSettings(
+ config_root, portdir, portdir_overlay,
+ repoman_settings, vcs_settings, options, qawarnings)
+
+ repoman_settings = repo_settings.repoman_settings
+
+ portdb = repo_settings.portdb
+ ##################
+
+ if options.echangelog is None and
repo_settings.repo_config.update_changelog:
+ options.echangelog = 'y'
+
+ if vcs_settings.vcs is None:
+ options.echangelog = 'n'
+
+ # The --echangelog option causes automatic ChangeLog generation,
+ # which invalidates changelog.ebuildadded and changelog.missing
+ # checks.
+ # Note: Some don't use ChangeLogs in distributed SCMs.
+ # It will be generated on server side from scm log,
+ # before package moves to the rsync server.
+ # This is needed because they try to avoid merge collisions.
+ # Gentoo's Council decided to always use the ChangeLog file.
+ # TODO: shouldn't this just be switched on the repo, iso the VCS?
+ is_echangelog_enabled = options.echangelog in ('y', 'force')
+ vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
+ check_changelog = not is_echangelog_enabled and
vcs_settings.vcs_is_cvs_or_svn
+
+ if 'digest' in repoman_settings.features and options.digest != 'n':
+ options.digest = 'y'
+
+ logging.debug("vcs: %s" % (vcs_settings.vcs,))
+ logging.debug("repo config: %s" % (repo_settings.repo_config,))
+ logging.debug("options: %s" % (options,))
+
+ # It's confusing if these warnings are displayed without the user
+ # being told which profile they come from, so disable them.
+ env = os.environ.copy()
+ env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
+
+ categories = []
+ for path in repo_settings.repo_config.eclass_db.porttrees:
+ categories.extend(portage.util.grabfile(
+ os.path.join(path, 'profiles', 'categories')))
+ repoman_settings.categories = frozenset(
+ portage.util.stack_lists([categories], incremental=1))
+ categories = repoman_settings.categories
+
+ portdb.settings = repoman_settings
+ # We really only need to cache the metadata that's necessary for
visibility
+ # filtering. Anything else can be discarded to reduce memory
consumption.
+ portdb._aux_cache_keys.clear()
+ portdb._aux_cache_keys.update(
+ ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
+
+ reposplit = myreporoot.split(os.path.sep)
+ repolevel = len(reposplit)
+
+ if options.mode == 'commit':
+ repochecks.commit_check(repolevel, reposplit)
+ repochecks.conflict_check(vcs_settings, options)
+
+ changed = Changes(options)
+ changed.scan(vcs_settings)
+
+ effective_scanlist = scanlist
+ if options.if_modified == "y":
+ effective_scanlist = sorted(vcs_files_to_cps(
+ chain(changed.changed, changed.new, changed.removed),
+ repolevel, reposplit, categories))
+
+ qatracker = repoman_scan(repoman_settings, repo_settings, vcs_settings,
options, repolevel)
if options.if_modified == "y" and len(effective_scanlist) < 1:
logging.warn("--if-modified is enabled, but no modified
packages were found!")
diff --git a/tbc/pym/repoman/utilities.py b/tbc/pym/repoman/utilities.py
index 5e43851..b1871db 100644
--- a/tbc/pym/repoman/utilities.py
+++ b/tbc/pym/repoman/utilities.py
@@ -240,7 +240,7 @@ def get_commit_message_with_stdin():
return commitmessage
-def FindPortdir(settings):
+def FindPortdir(settings, pkdir):
""" Try to figure out what repo we are in and whether we are in a
regular
tree or an overlay.
@@ -276,6 +276,8 @@ def FindPortdir(settings):
# the current working directory (from the shell).
location = pwd
+ if not pkdir is None:
+ location = pkgdir
location = normalize_path(location)
path_ids = {}
