commit: b720c6bbcb79211a276d06a1eaf1543b89644767
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 24 23:14:47 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Mar 24 23:14:47 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b720c6bb
Add the log parser BuildFactory steps
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 8 +++
buildbot_gentoo_ci/config/buildfactorys.py | 22 +++++++-
buildbot_gentoo_ci/config/schedulers.py | 3 +
buildbot_gentoo_ci/steps/builders.py | 90 ++++++++++++++++++++----------
buildbot_gentoo_ci/steps/portage.py | 10 +++-
5 files changed, 101 insertions(+), 32 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py
b/buildbot_gentoo_ci/config/builders.py
index c7b4469..0235753 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -49,4 +49,12 @@ def gentoo_builders(b=[]):
factory=buildfactorys.run_build_request()
)
)
+ # FIXME: get workers from db
+ # Use multiplay workers
+ b.append(util.BuilderConfig(
+ name='parse_build_log',
+ workername='updatedb_1',
+ factory=buildfactorys.parse_build_log()
+ )
+ )
return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index db48130..df7baef 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -10,6 +10,7 @@ from buildbot_gentoo_ci.steps import package
from buildbot_gentoo_ci.steps import version
from buildbot_gentoo_ci.steps import builders
from buildbot_gentoo_ci.steps import portage
+from buildbot_gentoo_ci.steps import logs
def update_db_check():
f = util.BuildFactory()
@@ -159,5 +160,24 @@ def run_build_request():
f.addStep(builders.RunEmerge(step='depclean'))
f.addStep(builders.RunEmerge(step='preserved-libs'))
f.addStep(builders.RunEmerge(step='depclean'))
- f.addStep(builders.setBuildStatus())
+ return f
+
+def parse_build_log():
+ f = util.BuildFactory()
+ # FIXME: 6
+ # set needed Propertys
+ f.addStep(logs.SetupPropertys())
+ # pers the build log for info qa errors
+ #f.addStep(logs.ParserBuildLog())
+ # pers the log from pkg check
+ #f.addStep(logs.ParserPkgCheckLog())
+ # Upload the log to the cloud and remove the log
+ #f.addStep(logs.Upload())
+ # check the sum log if we need to make a issue/bug/pr report
+ # set it SUCCESS/FAILURE/WARNINGS
+ #f.addStep(logs.MakeIssue())
+ # set BuildStatus
+ #f.addStep(logs.setBuildStatus())
+ # setup things for the irc bot
+ #f.addStep(logs.SetIrcInfo())
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index 0cbec96..efdac75 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -74,6 +74,8 @@ def gentoo_schedulers():
builderNames=["build_request_data"])
run_build_request = schedulers.Triggerable(name="run_build_request",
builderNames=["run_build_request"])
+ pers_build_log = schedulers.Triggerable(name="parse_build_log",
+ builderNames=["parse_build_log"])
s = []
s.append(test_updatedb)
s.append(scheduler_update_db)
@@ -81,4 +83,5 @@ def gentoo_schedulers():
s.append(update_v_data)
s.append(build_request_data)
s.append(run_build_request)
+ s.append(parse_build_log)
return s
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 22608e3..ff6dd94 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -20,22 +20,26 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['preserved_libs'] = False
emerge_output['depclean'] = False
package_dict = {}
+ log_path_list = []
print(stderr)
emerge_output['stderr'] = stderr
# split the lines
for line in stdout.split('\n'):
# package list
subdict = {}
- if line.startswith('[ebuild') or line.startswith('[binary'):
+ subdict2 = {}
+ if line.startswith('[ebuild') or line.startswith('[binary') or
line.startswith('[nomerge'):
# if binaries
- if line.startswith('[ebuild'):
+ if line.startswith('[ebuild') or line.startswith('[nomerge'):
subdict['binary'] = False
else:
subdict['binary'] = True
# action [ N ] stuff
subdict['action'] = line[8:15].replace(' ', '')
# cpv
+ #FIXME: We my have more then one spece betvine ] and cpv
cpv_split = re.search('] (.+?) ', line).group(1).split(':')
+ print(cpv_split)
cpv = cpv_split[0]
# repository
# slot
@@ -69,8 +73,12 @@ def PersOutputOfEmerge(rc, stdout, stderr):
#FIXME: Handling of !!! output
if line.startswith('!!! existing preserved libs'):
pass
+ if line.startswith(' * '):
+ if line.endswith('.log.gz'):
+ log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed
or saved
emerge_output['package'] = package_dict
+ emerge_output['log_paths'] = log_path_list
# split the lines
#FIXME: Handling of stderr output
for line in stderr.split('\n'):
@@ -192,10 +200,14 @@ class SetupPropertys(BuildStep):
projectrepository_data = self.getProperty('projectrepository_data')
print(projectrepository_data)
project_data = yield
self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
self.setProperty('project_data', project_data, 'project_data')
+ self.setProperty('repository_data', repository_data, 'repository_data')
self.setProperty('preserved_libs', False, 'preserved-libs')
self.setProperty('depclean', False, 'depclean')
self.setProperty('cpv_build', False, 'cpv_build')
+ self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
+ self.setProperty('faild_version_data', None, 'faild_version_data')
print(self.getProperty("buildnumber"))
if self.getProperty('project_build_data') is None:
project_build_data = {}
@@ -472,6 +484,49 @@ class CheckEmergeLogs(BuildStep):
if self.step == 'pre-build':
print(emerge_output)
+ #FIXME:
+ # Look for FAILURE and logname and download needed logfile and
+ # trigger a logparser
+ # local_log_path dir set in config
+ # format
/var/cache/portage/logs/build/gui-libs/egl-wayland-1.1.6:20210321-173525.log.gz
+ if self.step == 'build':
+ print(emerge_output)
+ log_dict = {}
+ # get cpv, logname and log path
+ for log_path in emerge_output['log_paths']:
+ c = log_path.split('/')[6]
+ full_logname = log_path.split('/')[7]
+ print(full_logname)
+ pv = full_logname.split(':')[0]
+ cpv = c + '/' + pv
+ log_dict[cpv] = dict(
+ log_path = log_path,
+ full_logname = full_logname
+ )
+ print(log_dict)
+ # Find log for cpv that was requested or did faild
+ if not log_dict == {}:
+ # requested cpv
+ if self.getProperty('cpv') in log_dict:
+ log_data = log_dict[self.getProperty('cpv')]
+ masterdest = yield os.path.join(self.master.basedir,
'cpv_logs', log_data['full_logname'])
+ aftersteps_list.append(steps.FileUpload(
+ workersrc=log_data['log_path'],
+ masterdest=masterdest
+ ))
+ aftersteps_list.append(steps.Trigger(
+ schedulerNames=['parse_build_log'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : self.getProperty("cpv"),
+ 'faild_version_data' :
self.getProperty('faild_version_data'),
+ 'project_build_data' :
self.getProperty('project_build_data'),
+ 'log_build_data' : log_data,
+ 'pkg_check_log_data' :
self.getProperty("pkg_check_log_data"),
+ 'repository_data' :
self.getProperty('repository_data')
+ }
+ ))
if not self.step is None and aftersteps_list != []:
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
@@ -496,8 +551,7 @@ class RunPkgCheck(BuildStep):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
portage_repos_path = self.getProperty('portage_repos_path')
- repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
- repository_path = yield os.path.join(portage_repos_path,
repository_data['name'])
+ repository_path = yield os.path.join(portage_repos_path,
self.getProperty('repository_data')['name'])
cpv = self.getProperty("cpv")
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
@@ -544,7 +598,7 @@ class CheckPkgCheckLogs(BuildStep):
print(pkgcheck_output)
#FIXME:
# Perse the logs
- # tripp irc request with pkgcheck info
+ self.setProperty('pkg_check_log_data', None, 'pkg_check_log_data')
return SUCCESS
class RunBuild(BuildStep):
@@ -562,6 +616,8 @@ class RunBuild(BuildStep):
@defer.inlineCallbacks
def run(self):
if not self.getProperty('cpv_build'):
+ #FIXME:
+ # trigger pars_build_log if we have any logs to check
return SUCCESS
aftersteps_list = []
aftersteps_list.append(RunEmerge(step='pre-build'))
@@ -570,27 +626,3 @@ class RunBuild(BuildStep):
self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
-
-class setBuildStatus(BuildStep):
-
- name = 'setBuildStatus'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- @defer.inlineCallbacks
- def run(self):
- self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- project_build_data = self.getProperty('project_build_data')
- if project_build_data['status'] == 'in-progress':
- yield self.gentooci.db.builds.setSatusBuilds(
-
project_build_data['build_id'],
-
project_build_data['project_uuid'],
- 'completed')
- self.setProperty('project_build_data', project_build_data,
'project_build_data')
- return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/portage.py
b/buildbot_gentoo_ci/steps/portage.py
index 6d8388a..e2315cb 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -190,6 +190,8 @@ class SetMakeConf(BuildStep):
makeconf_variable_list.append('cgroup')
makeconf_variable_list.append('-news')
makeconf_variable_list.append('-collision-protect')
+ makeconf_variable_list.append('split-log')
+ makeconf_variable_list.append('compress-build-logs')
# EMERGE_DEFAULT_OPTS
if k['variable'] == 'EMERGE_DEFAULT_OPTS':
makeconf_variable_list.append('--buildpkg=y')
@@ -201,6 +203,8 @@ class SetMakeConf(BuildStep):
makeconf_variable_list.append('--nospinner')
makeconf_variable_list.append('--color=n')
makeconf_variable_list.append('--ask=n')
+ makeconf_variable_list.append('--quiet-build=y')
+ makeconf_variable_list.append('--quiet-fail=y')
# CFLAGS
if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
makeconf_variable_list.append('-O2')
@@ -236,8 +240,10 @@ class SetMakeConf(BuildStep):
makeconf_list.append('NOCOLOR=true')
makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
- makeconf_list.append('PORTAGE_ELOG_CLASSES="qa"')
- makeconf_list.append('PORTAGE_ELOG_SYSTEM="save"')
+ makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
+ makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
+ # We need echo:info to get the logfile name
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"')
# add ACCEPT_KEYWORDS from the project_data info
keyword_data = yield
self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
if project_data['status'] == 'unstable':
