commit: c2cd0bb34537771d61dd8d7c577cecf25c446200
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 10 10:02:20 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jun 10 10:02:20 2023 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c2cd0bb3
Add support for SetupBugReportSteps and compression with xz
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/config.py | 3 +
buildbot_gentoo_ci/steps/logs.py | 141 ++++++++++++++++++++++++++++--------
gentooci.cfg | 20 +++++
3 files changed, 132 insertions(+), 32 deletions(-)
diff --git a/buildbot_gentoo_ci/config/config.py
b/buildbot_gentoo_ci/config/config.py
index e2f33be..696ab0f 100644
--- a/buildbot_gentoo_ci/config/config.py
+++ b/buildbot_gentoo_ci/config/config.py
@@ -93,6 +93,7 @@ class GentooCiConfig(util.ComparableMixin):
"db_url",
"project",
"worker_config",
+ "bug_config",
"repository_basedir"
])
@@ -152,3 +153,5 @@ class GentooCiConfig(util.ComparableMixin):
self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR
if 'worker_config' in config_dict:
self.project['worker_config'] = config_dict['worker_config']
+ if 'bug_config' in config_dict:
+ self.project['bug_config'] = config_dict['bug_config']
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index e4fc951..ee11166 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
@@ -14,6 +14,7 @@ from twisted.internet import defer
from twisted.python import log
from buildbot.process.buildstep import BuildStep
+from buildbot.process.properties import Properties
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
@@ -21,7 +22,7 @@ from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot.plugins import util
-from buildbot_gentoo_ci.steps import minio
+#from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
@@ -91,13 +92,15 @@ class SetupPropertys(BuildStep):
self.setProperty("default_project_data", default_project_data,
'default_project_data')
self.setProperty("version_data", version_data, 'version_data')
self.setProperty("status", 'completed', 'status')
- if self.getProperty('faild_cpv'):
+ if isinstance(self.getProperty('faild_cpv'), str):
log_cpv = self.getProperty('faild_cpv')
else:
log_cpv = self.getProperty('cpv')
self.setProperty("log_cpv", log_cpv, 'log_cpv')
- self.setProperty("bgo", dict( match=False), 'bgo')
+ self.setProperty("bgo", False, 'bgo')
self.descriptionDone = 'Runing log checker on ' + log_cpv
+ logsdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
+ self.setProperty("logsdir", logsdir, 'logsdir')
return SUCCESS
class SetupParserBuildLoger(BuildStep):
@@ -115,9 +118,9 @@ class SetupParserBuildLoger(BuildStep):
@defer.inlineCallbacks
def run(self):
self.aftersteps_list = []
- workdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
- mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname'])
+ build_log_file_compressed = log_cpv['full_logname'] + '.xz'
+ mastersrc_log = yield os.path.join(self.getProperty('logsdir'),
build_log_file_compressed)
log_py = 'log_parser.py'
config_log_py = 'logparser.json'
mastersrc_py = yield os.path.join(self.master.basedir, log_py)
@@ -125,7 +128,7 @@ class SetupParserBuildLoger(BuildStep):
# Upload logfile to worker
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_log,
-
workerdest=log_cpv['full_logname']
+
workerdest=build_log_file_compressed
))
# Upload log parser py code
self.aftersteps_list.append(steps.FileDownload(
@@ -137,6 +140,16 @@ class SetupParserBuildLoger(BuildStep):
mastersrc=mastersrc_config,
workerdest=config_log_py
))
+ #Untar the log
+ shell_commad_list = []
+ shell_commad_list.append('xz')
+ shell_commad_list.append('-dv')
+ shell_commad_list.append(build_log_file_compressed)
+ self.aftersteps_list.append(
+ steps.ShellCommand(
+ name = 'Decompress build log with xz',
+ command = shell_commad_list,
+ ))
# Run the log parser code
command = []
command.append('python3')
@@ -171,23 +184,6 @@ class MakeIssue(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
- @defer.inlineCallbacks
- def logIssue(self):
- separator1 = '\n'
- separator2 = ' '
- log = yield self.addLog('issue')
- self.error_dict['cpv'] = self.getProperty('log_cpv')
- yield log.addStdout('Title:' + '\n')
- yield log.addStdout(separator2.join([self.getProperty('log_cpv'), '-',
self.error_dict['title']]) + separator1)
- yield log.addStdout('Summary:' + '\n')
- for line in self.summary_log_list:
- yield log.addStdout(line + '\n')
- yield log.addStdout('Attachments:' + '\n')
- yield log.addStdout('emerge_info.log' + '\n')
- log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
- yield log.addStdout(log_cpv['full_logname'] + '\n')
- yield log.addStdout('world.log' + '\n')
-
def getNiceErrorLine(self, line):
# strip away hex addresses, loong path names, line and time numbers
and other stuff
# https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467
@@ -211,12 +207,16 @@ class MakeIssue(BuildStep):
if s['type'] == self.error_dict['phase'] and s['status'] ==
'error':
text_issue_list.append(v['text'])
# add the issue error
+ self.error_dict['cpv'] = self.getProperty('log_cpv')
if text_issue_list != []:
- self.error_dict['title_issue'] = text_issue_list[0].replace('*',
'').strip()
- self.error_dict['title_issue_nice'] =
self.getNiceErrorLine(text_issue_list[0].replace('*', '').strip())
+ title_line = text_issue_list[0].replace('*', '').strip()
+ self.error_dict['title_issue'] = title_line
+ self.error_dict['title_issue_nice'] =
self.getNiceErrorLine(title_line)
+ self.error_dict['title_found'] = True
else:
self.error_dict['title_issue'] = 'title_issue : None'
self.error_dict['title_nice'] = 'title_issue : None'
+ self.error_dict['title_found'] = False
self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase']
#set the error title
self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' +
self.error_dict['title_issue']
@@ -243,13 +243,14 @@ class MakeIssue(BuildStep):
phase_error = v['text'].split(' (')[1].split(' phase')[0]
self.error_dict['phase'] = phase_error
error = True
- # add build log
+ #FIXME: write summary_log_list to a file
# add issue/bug/pr report
if error:
yield self.ClassifyIssue()
print(self.error_dict)
- yield self.logIssue()
self.setProperty("status", 'failed', 'status')
+ #FIXME: nice description for post bug
+ self.error_dict['summary_log_nice'] = ''
self.setProperty("error_dict", self.error_dict, 'error_dict')
self.aftersteps_list.append(bugs.GetBugs())
if warning:
@@ -295,14 +296,13 @@ class ReadEmergeInfoLog(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
- @defer.inlineCallbacks
+ #@defer.inlineCallbacks
def run(self):
emerge_info_output = {}
emerge_info_list = []
emerge_package_info = []
# Read the file and add it to a property
- workdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
- with open(os.path.join(workdir, 'emerge_info.txt'), encoding='utf-8')
as source:
+ with open(os.path.join(self.getProperty('logsdir'),
'emerge_info.txt'), encoding='utf-8') as source:
emerge_info = source.read()
# set emerge_info_output Property
for line in emerge_info.split('\n'):
@@ -379,7 +379,7 @@ class Upload(BuildStep):
def run(self):
log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
- file_path = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id'])
,log_cpv['full_logname'])
+ file_path = yield os.path.join(self.getProperty('logsdir')
,log_cpv['full_logname'])
aftersteps_list = []
aftersteps_list.append(minio.putFileToMinio(file_path,
log_cpv['full_logname'], bucket))
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
@@ -435,6 +435,83 @@ class ParserPkgCheckLog(BuildStep):
returnstatus = WARNINGS
return returnstatus
+class SetupBugReportSteps(BuildStep):
+ name = 'SetupBugReportSteps'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnFailure = True
+ warnOnWarnings = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ bug_config = self.gentooci.config.project['bug_config']
+ # add bug enable profile and repo
+ if bug_config['enable'] and self.getProperty('status') == 'failed' and
self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
+ aftersteps_list = []
+ #Post
+ p = Properties()
+ p.master = self.master
+ separator = ' '
+ separator2 = '\n'
+ bug_args = {}
+ bug_args['user'] = bug_config['user']
+ bug_args['passwd'] = yield p.render(util.Secret("bugs_password"))
+ bug_params = {}
+ title = separator.join([bug_config['extra_summery'],
self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title']])
+ # bug title max 170
+ if len(title) >= 170:
+ title = title[:167] + '...'
+ bug_params['summary'] = title
+ # max 16384
+ description_list = self.getProperty('summary_log_list')
+ del description_list[-6:]
+ description_text = separator2.join(description_list)
+ bug_params['description'] = description_text + separator
+ bug_params['assigned_to'] = bug_config['user']
+ #params['cc'] = cc
+ bug_params['url'] =
f"https://ci.dev.gentoo.org:8010/#/builders/7/builds/{self.getProperty('buildnumber')}"
+ print(bug_params)
+ # post
+ aftersteps_list.append(bugs.Post(bug_args, bug_params))
+ # add buildbot info
+ bug_params2 = {}
+ bug_params2['comment'] =
separator2.join(bug_config['buildbot_comment'])
+ aftersteps_list.append(bugs.Modify(bug_args, bug_params2))
+ # add build info
+ comment_list = []
+ comment_list.append(f"Was trying to build
{self.getProperty('cpv')} Commit: {self.getProperty('revision')} Repository:
{self.getProperty('repository')}")
+ if self.getProperty('faild_cpv') != self.getProperty('cpv'):
+ comment_list.append(f"Depend: {self.getProperty('faild_cpv')}
fail to build")
+ comment_list.append(f"Package info:")
+ for line in
self.getProperty('emerge_info_output')['emerge_package_info']:
+ comment_list.append(f"{line}")
+ comment_list.append(f"Emerge info:")
+ for line in self.getProperty('emerge_info_output')['emerge_info']:
+ comment_list.append(f"{line}")
+ bug_params3 = {}
+ bug_params3['comment'] = separator2.join(comment_list)
+ aftersteps_list.append(bugs.Modify(bug_args, bug_params3))
+ # Attach build log
+ file_list = []
+ with os.scandir(self.getProperty('logsdir')) as entries:
+ for entry in entries:
+ file_list.append(entry.name)
+ print(file_list)
+ for filename in file_list:
+ bug_params4 = {}
+ bug_params4['filename'] = yield
os.path.join(self.getProperty('logsdir'), filename)
+ #bug_params4['comment'] = filename
+ aftersteps_list.append(bugs.Attach(bug_args, bug_params4))
+ yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ return SUCCESS
+ return SKIPPED
+
class setBuildStatus(BuildStep):
name = 'setBuildStatus'
diff --git a/gentooci.cfg b/gentooci.cfg
index cd4bfda..474da76 100644
--- a/gentooci.cfg
+++ b/gentooci.cfg
@@ -26,6 +26,9 @@ makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"')
# use ansifilter on the logs
makeconf_list.append('PORTAGE_LOG_FILTER_FILE_CMD="bash -c \'ansifilter
--ignore-clear; exec cat\'"')
+# use xz for Portage compression of both installed files and binary packages:
+makeconf_list.append('PORTAGE_COMPRESS="xz"')
+makeconf_list.append('BINPKG_COMPRESS="xz"')
# This specifies what project buildbot uses for Gentoo Ci as default
c['project'] = {
@@ -43,3 +46,20 @@ c['worker_config'] = {
'repository_basedir' : 'repositorys',
'portage_repos_path' : '/var/db/repos',
}
+# This specifies bug configs
+c['bug_config'] = {
+ 'enable' : True,
+ 'aout_assignee' : False,
+ 'user' : '[email protected]',
+ 'extra_summery' : '',
+ 'buildbot_comment' : [
+ 'This was build on a tinderbox that use Buildbot API[1].',
+ 'We use docker images to build on.',
+ 'Web: https://ci.dev.gentoo.org:8010/#/',
+ 'Code: https://gitweb.gentoo.org/proj/tinderbox-cluster.git',
+ 'Wiki: https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster',
+ 'How to get more info:
https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster/something',
+ 'IRC: #gentoo-ci on libera',
+ '[1] https://buildbot.net/',
+ ],
+}
