commit: 5dc7b0dd7863aac81b462302c2fc11635411927d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 28 08:37:27 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar 28 08:37:27 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5dc7b0dd
Add log parser steps
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 8 +-
buildbot_gentoo_ci/config/schedulers.py | 2 +-
buildbot_gentoo_ci/db/model.py | 14 ++
buildbot_gentoo_ci/db/projects.py | 39 +++++
buildbot_gentoo_ci/steps/logs.py | 229 +++++++++++++++++++++++++++++
5 files changed, 288 insertions(+), 4 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py
b/buildbot_gentoo_ci/config/buildfactorys.py
index df7baef..d8db8f8 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -168,16 +168,18 @@ def parse_build_log():
# set needed Propertys
f.addStep(logs.SetupPropertys())
# pers the build log for info qa errors
- #f.addStep(logs.ParserBuildLog())
+ f.addStep(logs.ParserBuildLog())
# pers the log from pkg check
#f.addStep(logs.ParserPkgCheckLog())
# Upload the log to the cloud and remove the log
#f.addStep(logs.Upload())
# check the sum log if we need to make a issue/bug/pr report
# set it SUCCESS/FAILURE/WARNINGS
- #f.addStep(logs.MakeIssue())
+ f.addStep(logs.MakeIssue())
+ # add sum log to buildbot log
+ #f.addStep(logs.setBuildbotLog)
# set BuildStatus
- #f.addStep(logs.setBuildStatus())
+ f.addStep(logs.setBuildStatus())
# setup things for the irc bot
#f.addStep(logs.SetIrcInfo())
return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py
b/buildbot_gentoo_ci/config/schedulers.py
index efdac75..1941a82 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -74,7 +74,7 @@ def gentoo_schedulers():
builderNames=["build_request_data"])
run_build_request = schedulers.Triggerable(name="run_build_request",
builderNames=["run_build_request"])
- pers_build_log = schedulers.Triggerable(name="parse_build_log",
+ parse_build_log = schedulers.Triggerable(name="parse_build_log",
builderNames=["parse_build_log"])
s = []
s.append(test_updatedb)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 939059c..3abc686 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -221,6 +221,20 @@ class Model(base.DBConnectorComponent):
sa.Column('deleted_at', sa.DateTime, nullable=True),
)
+ projects_pattern = sautils.Table(
+ "projects_pattern", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('search', sa.String(50), nullable=False),
+ sa.Column('search_end', sa.String(50), nullable=True),
+ sa.Column('start', sa.Integer, default=0),
+ sa.Column('end', sa.Integer, default=0),
+ sa.Column('status', sa.Enum('info', 'warning', 'ignore', 'error'),
default='info'),
+ sa.Column('type', sa.Enum('info', 'qa', 'compile', 'configure',
'install', 'postinst', 'prepare', 'setup', 'test', 'unpack', 'ignore'),
default='info'),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/projects.py
b/buildbot_gentoo_ci/db/projects.py
index 37ae2f1..2edc14a 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -166,6 +166,29 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectLogSearchPatternByUuid(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_pattern
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ return [self._row2dict_projects_pattern(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def getProjectLogSearchPatternByUuidAndIgnore(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_pattern
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.status == 'ignore')
+ return [self._row2dict_projects_pattern(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -242,3 +265,19 @@ class
ProjectsConnectorComponent(base.DBConnectorComponent):
depclean=row.depclean,
preserved_libs=row.preserved_libs
)
+
+ def _row2dict_projects_pattern(self, conn, row):
+ if row.search_end == '':
+ search_end = None
+ else:
+ search_end = row.search_end
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ search=row.search,
+ search_end=search_end,
+ start=row.start,
+ end=row.end,
+ status=row.status,
+ type=row.type
+ )
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
new file mode 100644
index 0000000..ae41fe5
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -0,0 +1,229 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import re
+import gzip
+import io
+import hashlib
+
+from portage.versions import catpkgsplit
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import WARNINGS
+from buildbot.plugins import steps
+
+class SetupPropertys(BuildStep):
+
+ name = 'SetupPropertys'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ # set this in config
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_data = yield
self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
+ default_project_data = yield
self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project'])
+ version_data = yield
self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
+ self.setProperty("project_data", project_data, 'project_data')
+ self.setProperty("default_project_data", default_project_data,
'default_project_data')
+ self.setProperty("version_data", version_data, 'version_data')
+ self.setProperty("status", 'completed', 'status')
+ return SUCCESS
+
+class ParserBuildLog(BuildStep):
+
+ name = 'ParserBuildLog'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ self.logfile_text_dict = {}
+ self.summery_dict = {}
+ self.index = 1
+ self.log_search_pattern_list = []
+ self.max_text_lines = self.index -1
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def get_log_search_pattern(self):
+ # get pattern from the profile
+ # add that to log_search_pattern_list
+ for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
+ self.log_search_pattern_list.append(project_pattern)
+ # get the default profile pattern
+ # add if not pattern is in project ignore
+ for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
+ match = True
+ for project_pattern_ignore in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])):
+ if project_pattern['search'] ==
project_pattern_ignore['search']:
+ match = False
+ if match:
+ self.log_search_pattern_list.append(project_pattern)
+ print(self.log_search_pattern_list)
+
+ def search_buildlog(self, tmp_index):
+ # get text line to search
+ text_line = self.logfile_text_dict[tmp_index]
+ # loop true the pattern list for match
+ for search_pattern in self.log_search_pattern_list:
+ if re.search(search_pattern['search'], text_line) and not
search_pattern['status'] == 'ignore':
+ self.summery_dict[tmp_index] = {}
+ self.summery_dict[tmp_index]['text'] = text_line
+ self.summery_dict[tmp_index]['status'] =
search_pattern['status']
+ # add upper text lines if requested
+ # max 10
+ if search_pattern['start'] is not 0:
+ i = tmp_index
+ i_start = i - search_pattern['start']
+ match = True
+ while match:
+ i = i - 1
+ if i < 0 or i < i_start:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+ # add lower text lines if requested
+ # max 10
+ if search_pattern['end'] is not 0:
+ i = tmp_index
+ i_end = i + search_pattern['end']
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines or i > i_end:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+ # add text lines if requested that we need to search for the
end
+ # max 10
+ if search_pattern['search_end'] is not None:
+ i = tmp_index
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines:
+ match = False
+ if re.search(search_pattern['search_end'],
self.logfile_text_dict[i]):
+ if not i + 1 > self.max_text_lines or not
re.search(search_pattern['search_end'], self.logfile_text_dict[i + 1]):
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+ else:
+ match = False
+ else:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
+ self.summery_dict[i]['status'] = 'info'
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ #FIXME:
+ # get the log parser pattern from db
+ yield self.get_log_search_pattern()
+ # open the log file
+ # read it to a buffer
+ # make a dict of the buffer
+ # maby use mulitiprocces to speed up the search
+ file_path = yield os.path.join(self.master.basedir, 'cpv_logs',
self.getProperty('log_build_data')['full_logname'])
+ with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb')))
as f:
+ for text_line in f:
+ self.logfile_text_dict[self.index] = text_line
+ # run the parse patten on the line
+ # have a buffer on 5 before we run pattern check
+ if self.index >= 10:
+ yield self.search_buildlog(self.index - 9)
+ # remove text line that we don't need any more
+ if self.index >= 20:
+ del self.logfile_text_dict[self.index - 19]
+ self.index = self.index + 1
+ f.close()
+ print(self.summery_dict)
+ # check last 5 lines in logfile_text_dict
+ # setProperty summery_dict
+ self.setProperty("summery_log_dict", self.summery_dict,
'summery_log_dict')
+ return SUCCESS
+
+class MakeIssue(BuildStep):
+
+ name = 'MakeIssue'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ #@defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ summery_log_dict = self.getProperty('summery_log_dict')
+ error = False
+ warning = False
+ self.summery_log_list = []
+ log_hash = hashlib.sha256()
+ for k, v in sorted(summery_log_dict.items()):
+ if v['status'] == 'error':
+ error = True
+ if v['status'] == 'warning':
+ warning = True
+ self.summery_log_list.append(v['text'])
+ log_hash.update(v['text'].encode('utf-8'))
+ # add build log
+ # add issue/bug/pr report
+ self.setProperty("summery_log_list", self.summery_log_list,
'summery_log_list')
+ if error:
+ self.setProperty("status", 'failed', 'status')
+ if warning:
+ self.setProperty("status", 'warning', 'status')
+ return SUCCESS
+
+class setBuildStatus(BuildStep):
+
+ name = 'setBuildStatus'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = False
+ flunkOnFailure = True
+ warnOnWarnings = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
+ project_build_data = self.getProperty('project_build_data')
+ yield self.gentooci.db.builds.setSatusBuilds(
+
project_build_data['build_id'],
+
project_build_data['project_uuid'],
+ self.getProperty('status')
+ )
+ if self.getProperty('status') == 'failed':
+ return FAILURE
+ if self.getProperty('status') == 'warning':
+ return WARNINGS
+ return SUCCESS
