commit: 2410346473e1d00de202eac126d995a148b9609f
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 1 20:49:08 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Aug 1 20:49:08 2015 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=24103464
add repoman full to host and guest
tbc/pym/build_log.py | 41 ++++++++++++++++++++++++++++++-----------
tbc/pym/db_mapping.py | 8 ++++++++
tbc/pym/package.py | 32 ++++++++++++++------------------
tbc/pym/qachecks.py | 13 ++++++++++++-
tbc/pym/repoman/main.py | 10 ++++++----
tbc/pym/sqlquerys.py | 15 ++++++++++++++-
6 files changed, 84 insertions(+), 35 deletions(-)
diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index 9c8bbfa..71c14b5 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -20,18 +20,38 @@ portage.proxy.lazyimport.lazyimport(globals(),
'tbc.actions:action_info,load_emerge_config',
)
-from tbc.qachecks import check_repoman
+from tbc.qachecks import check_repoman, repoman_full
from tbc.text import get_log_text_dict
-from tbc.package import tbc_package
from tbc.readconf import read_config_settings
from tbc.flags import tbc_use_flags
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db,
add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info,
get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times,
del_old_build_jobs, add_old_ebuild, \
- update_buildjobs_status, update_manifest_sql, add_repoman_qa,
get_config_id_fqdn, get_setup_info
+ update_buildjobs_status, update_manifest_sql, add_repoman_qa,
get_config_id_fqdn, get_setup_info, \
+ add_repoman_log
from sqlalchemy.orm import sessionmaker
+def repoman_check_full(session, pkgdir, package_id, config_id):
+ # Check cp with repoman repoman full
+ status = repoman_full(session, pkgdir, config_id)
+ repoman_hash = hashlib.sha256()
+ repoman_log = ""
+ if status:
+ for k, v in status.items():
+ repoman_line = k + '/n'
+ repoman_hash.update(repoman_line.encode('utf-8'))
+ repoman_log = repoman_log + repoman_line
+ for line in v:
+ repoman_line = line + '/n'
+
repoman_hash.update(repoman_line.encode('utf-8'))
+ repoman_log = repoman_log + repoman_line
+ add_repoman_log(session, package_id, repoman_log,
repoman_hash.hexdigest())
+ return repoman_log
+ else:
+ return status
+
+
def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
myportdb = portage.portdbapi(mysettings=settings)
cpvr_list = catpkgsplit(pkg.cpv, silent=1)
@@ -48,6 +68,7 @@ def get_build_dict_db(session, config_id, settings,
tbc_settings_dict, pkg):
build_dict['cpv'] = pkg.cpv
build_dict['categories'] = categories
build_dict['package'] = package
+ build_dict['repo'] = repo
build_dict['config_id'] = config_id
init_useflags = tbc_use_flags(settings, myportdb, pkg.cpv)
iuse_flags_list, final_use_list = init_useflags.get_flags_pkg(pkg,
settings)
@@ -77,12 +98,6 @@ def get_build_dict_db(session, config_id, settings,
tbc_settings_dict, pkg):
log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv,
repo,)
add_logs(session, log_msg, "info", config_id)
update_manifest_sql(session, build_dict['package_id'],
"0")
- init_package = tbc_package(session, settings, myportdb,
config_id)
- init_package.update_package_db(build_dict['package_id'])
- ebuild_id_list, status = get_ebuild_id_db(session,
build_dict['checksum'], build_dict['package_id'])
- if status and ebuild_id_list is None:
- log_msg = "%s:%s Don't have any ebuild_id!" %
(pkg.cpv, repo,)
- add_logs(session, log_msg, "error", config_id)
else:
old_ebuild_id_list = []
for ebuild_id in ebuild_id_list:
@@ -204,8 +219,12 @@ def get_buildlog_info(session, settings, pkg, build_dict):
qa_error_list.append(logfile_text_dict[i])
i = i +1
- # Run repoman check_repoman()
- repoman_error_list = check_repoman(settings, myportdb,
build_dict['cpv'], pkg.repo)
+ # Run repoman full
+ element = portage.versions.cpv_getkey(build_dict['cpv']).split('/')
+ categories = element[0]
+ package = element[1]
+ pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" +
categories + "/" + package
+ repoman_error_list = repoman_check_full(session, pkgdir,
build_dict['package_id'], config_id)
build_log_dict = {}
build_log_dict['fail'] = False
if repoman_error_list:
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index a296c2e..dbae5a8 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -230,6 +230,14 @@ class BuildLogsRepomanQa(Base):
SummeryText = Column('summery_text', Text)
__tablename__ = 'build_logs_repoman_qa'
+class PackagesRepoman(Base):
+ Id = Column('id', Integer, primary_key=True)
+ PackageId = Column('package_id', Integer,
ForeignKey('packages.package_id'))
+ RepomanText = Column('repoman_text', Text)
+ RepomanHash = Column('repoman_hash', String(100))
+ TimeStamp = Column('time_stamp', DateTime, nullable=False,
default=datetime.datetime.utcnow)
+ __tablename__ = 'packages_repoman'
+
class ErrorsInfo(Base):
ErrorId = Column('error_id', Integer, primary_key=True)
ErrorName = Column('error_name', String)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index 0a19f15..657c90d 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -3,12 +3,14 @@
from __future__ import print_function
import re
+import hashlib
import portage
from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman, repoman_full
+from tbc.qachecks import digestcheck, check_repoman
+from tbc.build_log import check_repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list,
add_old_ebuild, \
get_package_metadata_sql, update_package_metadata, update_manifest_sql,
\
@@ -283,11 +285,13 @@ class tbc_package(object):
if manifest_checksum_tree is None:
return
- # Check cp with repoman repoman full
- repoman_full(self._session, pkgdir, self._config_id)
-
package_id = add_new_package_sql(self._session, cp, repo)
+ # Check cp with repoman full
+ status = check_repoman_full(self._session, pkgdir, package_id,
self._config_id)
+ if status:
+ log_msg = "Repoman %s:%s ... Fail." %
(cpv, repo)
+ add_logs(self._session, log_msg,
"error", self._config_id)
package_metadataDict = self.get_package_metadataDict(pkgdir,
package_id)
# Get the ebuild list for cp
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1,
mytree=mytree)
@@ -308,11 +312,7 @@ class tbc_package(object):
# take package descriptions from the ebuilds
if
package_metadataDict[package_id]['metadata_xml_descriptions'] !=
packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] =
packageDict[cpv]['ebuild_version_descriptions_tree']
- if packageDict[cpv]['checksum'] != "0":
- repoman_fail = check_repoman(self._mysettings,
self._myportdb, cpv, repo)
- if repoman_fail:
- log_msg = "Repoman %s:%s ... Fail." %
(cpv, repo)
- add_logs(self._session, log_msg,
"error", self._config_id)
+
self.add_package(packageDict, package_metadataDict, package_id,
new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -340,8 +340,11 @@ class tbc_package(object):
log_msg = "U %s:%s" % (cp, repo)
add_logs(self._session, log_msg, "info",
self._config_id)
- # Check cp with repoman repoman full
- repoman_full(self._session, pkgdir, self._config_id)
+ # Check cp with repoman full
+ status = check_repoman_full(self._session, pkgdir,
package_id, self._config_id)
+ if status:
+ log_msg = "Repoman %s:%s ... Fail." %
(cpv, repo)
+ add_logs(self._session, log_msg,
"error", self._config_id)
# Get the ebuild list for cp
old_ebuild_id_list = []
@@ -399,13 +402,6 @@ class tbc_package(object):
else:
ebuild_version_manifest_checksum_db =
checksums_db
- # Check with repoman
- if (ebuild_version_manifest_checksum_db is None
or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db) and
ebuild_version_checksum_tree != "0":
- repoman_fail =
check_repoman(self._mysettings, self._myportdb, cpv, repo)
- if repoman_fail:
- log_msg = "Repoman %s:%s ...
Fail." % (cpv, repo)
- add_logs(self._session,
log_msg, "error", self._config_id)
-
# Check if the checksum have change
if ebuild_version_manifest_checksum_db is None:
# N = New ebuild
diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index 2e25d86..766b572 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -186,5 +186,16 @@ def repoman_full(session, pkgdir, config_id):
SetupInfo = get_setup_info(session, config_id)
config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname +
"/" + SetupInfo.Setup
argscmd = []
+ argscmd.append('--xmlparse')
argscmd.append('full')
- repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
+ qatracker, qawarnings = repoman_main(argscmd, config_root=config_root,
pkgdir=pkgdir)
+ adict = {}
+ for key in qatracker.fails.items():
+ alist = []
+ for foo in key[1]:
+ alist.append(foo)
+ adict[key[0]] = alist
+ if adict == {}:
+ return False
+ return adict
+
diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py
index 2353455..a30704a 100644
--- a/tbc/pym/repoman/main.py
+++ b/tbc/pym/repoman/main.py
@@ -789,7 +789,9 @@ def repoman_scan(repoman_settings, repo_settings,
vcs_settings, portdb, options,
"metadata.warning",
"%s/metadata.xml: unused local
USE-description: '%s'"
% (xpkg, myflag))
-
+
+ suggest_ignore_masked = False
+ suggest_include_dev = False
if have_pmasked and not (options.without_mask or options.ignore_masked):
suggest_ignore_masked = True
if have_dev_keywords and not options.include_dev:
@@ -973,7 +975,7 @@ def repoman_main(argv, config_root=None, pkgdir=None):
utilities.repoman_sez(
"\"Make your QA payment on time"
" and you'll never see the likes of me.\"\n")
- sys.exit(1)
+ return qatracker, qawarnings
else:
if dofail and can_force and options.force and not
options.pretend:
utilities.repoman_sez(
@@ -988,7 +990,7 @@ def repoman_main(argv, config_root=None, pkgdir=None):
utilities.repoman_sez(
"\"Make your QA payment on time"
" and you'll never see the likes of me.\"\n")
- sys.exit(1)
+ return qatracker, qawarnings
if options.pretend:
utilities.repoman_sez(
@@ -1627,4 +1629,4 @@ def repoman_main(argv, config_root=None, pkgdir=None):
" that he forgot to commit anything")
utilities.repoman_sez(
"\"If everyone were like you, I'd be out of
business!\"\n")
- sys.exit(0)
+ return qatracker, qawarnings
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index 8fdf901..4d2a5e5 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -8,7 +8,8 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData,
Jobs, BuildJobs, Pack
Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs,
BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors,
ErrorsInfo, EmergeInfo, BuildLogsUse, \
BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions,
EbuildsRestrictions, EbuildsKeywords, \
- Keywords, PackagesMetadata, Emails, PackagesEmails, Setups,
BuildLogsRepomanQa, CategoriesMetadata
+ Keywords, PackagesMetadata, Emails, PackagesEmails, Setups,
BuildLogsRepomanQa, CategoriesMetadata, \
+ PackagesRepoman
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy import and_, or_
@@ -566,3 +567,15 @@ def get_ebuild_restrictions(session, ebuild_id):
restrictions.append(session.query(Restrictions).filter_by(RestrictionId =
EbuildsRestrictionsInfo.RestrictionId).one())
return restrictions
return
restrictions.append(session.query(Restrictions).filter_by(RestrictionId =
EbuildsRestrictionsInfo.RestrictionId).one())
+
+def add_repoman_log(session, package_id, repoman_log, repoman_hash):
+ try:
+ PackagesRepomanInfo =
session.query(PackagesRepoman).filter_by(PackageId = package_id).one()
+ except NoResultFound as e:
+ session.add(PackagesRepoman(PackageId = package_id, RepomanText
= repoman_log, RepomanHash = repoman_hash))
+ session.commit()
+ else:
+ if PackagesRepomanInfo.RepomanHash != repoman_hash:
+ PackagesRepomanInfo.RepomanHash = repoman_hash
+ PackagesRepomanInfo.RepomanText = repoman_log
+ session.commit()
