commit: b3edf64aaff42659b3593dfe24a7e0a68e1881a0
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Dec 24 00:33:19 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Dec 24 00:33:19 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b3edf64a
put emerge and package info to a file insted
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 4 ++
buildbot_gentoo_ci/steps/builders.py | 74 +++++++++++++++++++++++------
buildbot_gentoo_ci/steps/logs.py | 75 +++++++++++++++++++++++++++++-
3 files changed, 137 insertions(+), 16 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index 27f0076..6203a1f 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,8 +172,12 @@ def parse_build_log():
f.addStep(logs.MakeIssue())
# add sum log to buildbot log
f.addStep(logs.setBuildbotLog())
+ # pers the emerge info
+ f.addStep(logs.SetupParserEmergeInfoLog())
# add emerge info to log and db
f.addStep(logs.setEmergeInfoLog())
+ # add package info to log and db
+ f.addStep(logs.setPackageInfoLog())
# set BuildStatus
f.addStep(logs.setBuildStatus())
# setup things for the irc bot
diff --git a/buildbot_gentoo_ci/steps/builders.py
b/buildbot_gentoo_ci/steps/builders.py
index 7247abd..e76fe3d 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -590,14 +590,35 @@ class CheckEmergeLogs(BuildStep):
workdir=workdir
))
- @defer.inlineCallbacks
- def getLogFile(self, cpv, log_dict):
- masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber"))
,log_dict[cpv]['full_logname'])
+ def addFileUploade(self, sourcefile, destfile):
self.aftersteps_list.append(steps.FileUpload(
- workersrc=log_dict[cpv]['log_path'],
- masterdest=masterdest
+ workersrc=sourcefile,
+ masterdest=destfile
))
+ @defer.inlineCallbacks
+ def getLogFile(self, cpv, log_dict):
+ destfile = yield os.path.join(self.masterdest,
log_dict[cpv]['full_logname'])
+ sourcefile = log_dict[cpv]['log_path']
+ self.addFileUploade(sourcefile, destfile)
+
+ @defer.inlineCallbacks
+ def getEmergeFiles(self, faild_version_data):
+ # get emerge info
+ destfile = yield os.path.join(self.masterdest, 'emerge_info.txt')
+ sourcefile = yield os.path.join('/', 'tmp', 'emerge_info.txt')
+ self.addFileUploade(sourcefile, destfile)
+ #FIXME:
+ # if faild_version_data:
+ # get emerge.log
+ # get elogs
+ # world file
+
+ def getBuildWorkdirFiles(self):
+ #FIXME:
+ # get files from the build workdir
+ pass
+
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
@@ -610,6 +631,8 @@ class CheckEmergeLogs(BuildStep):
]
package_dict = emerge_output['packages']
+ self.masterdest = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('workername'), str(self.getProperty("buildnumber")))
+
#FIXME: Prosees the logs and do stuff
# preserved-libs
if emerge_output['preserved_libs'] and
projects_emerge_options['preserved_libs']:
@@ -757,12 +780,12 @@ class CheckEmergeLogs(BuildStep):
print(log_dict)
cpv = self.getProperty('cpv')
faild_cpv = emerge_output['failed']
+ faild_version_data = False
if cpv in log_dict or faild_cpv in log_dict:
+ yield self.createDistDir()
if cpv in log_dict:
self.log_data[cpv] = log_dict[cpv]
- yield self.createDistDir()
yield self.getLogFile(cpv, log_dict)
- faild_version_data = False
if faild_cpv:
# failed and build requested cpv
if cpv == faild_cpv:
@@ -772,6 +795,8 @@ class CheckEmergeLogs(BuildStep):
self.log_data[faild_cpv] = log_dict[faild_cpv]
yield self.getLogFile(faild_cpv, log_dict)
faild_version_data = yield
self.getVersionData(faild_cpv)
+ self.getBuildWorkdirFiles()
+ self.getEmergeFiles(faild_version_data)
self.aftersteps_list.append(steps.Trigger(
schedulerNames=['parse_build_log'],
waitForFinish=False,
@@ -785,7 +810,6 @@ class CheckEmergeLogs(BuildStep):
'repository_data' :
self.getProperty('repository_data'),
'faild_cpv' : faild_cpv,
'step' : self.step,
- 'emerge_info' :
self.getProperty('emerge_info_output')['emerge_info'],
'build_workername' : self.getProperty('workername')
}
))
@@ -920,17 +944,37 @@ class RunEmergeInfo(BuildStep):
@defer.inlineCallbacks
def run(self):
aftersteps_list = []
+ # add emerge info
shell_commad_list = [
'emerge',
]
shell_commad_list.append('--info')
+ shell_commad_list.append('>')
+ shell_commad_list.append('/tmp/emerge_info.txt')
aftersteps_list.append(
- steps.SetPropertyFromCommand(
- command=shell_commad_list,
- strip=True,
- extract_fn=PersOutputOfEmergeInfo,
- workdir='/',
- timeout=None
+ steps.ShellCommand(
+ # the list need to be joined to pipe to a file
+ command=' '.join(shell_commad_list),
+ workdir='/'
+ ))
+ # add package info
+ cpv = self.getProperty("cpv")
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ shell_commad_list = [
+ 'emerge',
+ ]
+ shell_commad_list.append('-qpvO')
+ shell_commad_list.append('=' + self.getProperty('cpv'))
+ shell_commad_list.append('--usepkg-exclude')
+ shell_commad_list.append(c + '/' + p)
+ shell_commad_list.append('>>')
+ shell_commad_list.append('/tmp/emerge_info.txt')
+ aftersteps_list.append(
+ steps.ShellCommand(
+ # the list need to be joined to pipe to a file
+ command=' '.join(shell_commad_list),
+ workdir='/'
))
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
@@ -954,8 +998,8 @@ class RunBuild(BuildStep):
# trigger pars_build_log if we have any logs to check
return SKIPPED
aftersteps_list = []
- aftersteps_list.append(RunEmergeInfo())
aftersteps_list.append(RunEmerge(step='pre-build'))
+ aftersteps_list.append(RunEmergeInfo())
aftersteps_list.append(RunEmerge(step='build'))
aftersteps_list.append(RunEmerge(step='pre-depclean'))
aftersteps_list.append(RunEmerge(step='preserved-libs'))
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index dc5d3a1..d6ee705 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -45,6 +45,23 @@ def PersOutputOfLogParser(rc, stdout, stderr):
'build_summery_output' : build_summery_output
}
+def PersOutputOfEmergeInfo(rc, stdout, stderr):
+ #FIXME: line for package info
+ emerge_info_output = {}
+ emerge_info_output['rc'] = rc
+ emerge_info_list = []
+ emerge_package_info = []
+ for line in stdout.split('\n'):
+ if line.startswith('['):
+ emerge_package_info.append(line)
+ else:
+ emerge_info_list.append(line)
+ emerge_info_output['emerge_info'] = emerge_info_list
+ emerge_info_output['emerge_package_info'] = emerge_package_info
+ return {
+ 'emerge_info_output' : emerge_info_output
+ }
+
class SetupPropertys(BuildStep):
name = 'SetupPropertys'
@@ -387,6 +404,38 @@ class setBuildbotLog(BuildStep):
yield log.addStdout(line + '\n')
return SUCCESS
+class SetupParserEmergeInfoLog(BuildStep):
+
+ name = 'SetupParserEmergeInfoLog'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnFailure = True
+ warnOnWarnings = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ workdir = yield os.path.join(self.master.basedir, 'workers',
self.getProperty('build_workername'),
str(self.getProperty("project_build_data")['buildbot_build_id']))
+ command = []
+ command.append('cat')
+ command.append('emerge_info.txt')
+ self.aftersteps_list = []
+ self.aftersteps_list.append(master_steps.MasterSetPropertyFromCommand(
+ name =
'RunEmergeInfoLogParser',
+ haltOnFailure =
True,
+ flunkOnFailure =
True,
+ command=command,
+ workdir=workdir,
+ strip=False,
+
extract_fn=PersOutputOfEmergeInfo
+ ))
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+ return SUCCESS
+
class setEmergeInfoLog(BuildStep):
name = 'setEmergeInfoLog'
@@ -406,10 +455,34 @@ class setEmergeInfoLog(BuildStep):
log = yield self.addLog('emerge_info')
#FIXME: add emerge info to db
# add line for line
- for line in self.getProperty('emerge_info'):
+ for line in self.getProperty('emerge_info_output')['emerge_info']:
yield log.addStdout(line + '\n')
return SUCCESS
+class setPackageInfoLog(BuildStep):
+
+ name = 'setPackageInfoLog'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnFailure = True
+ warnOnWarnings = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ #setup the log
+ log = yield self.addLog('package_info')
+ #FIXME: add package info to db
+ # add line for line
+ for line in
self.getProperty('emerge_info_output')['emerge_package_info']:
+ yield log.addStdout(line + '\n')
+ return SUCCESS
+
+
class Upload(BuildStep):
name = 'Upload'
