commit: b2f1c115420e05c27fb7a57cf45734cb7f4a7731
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 26 20:26:04 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Apr 26 20:26:04 2021 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b2f1c115
Add more checks and move * and >>> check to last
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 121 ++++++++++++++++++++++-----------------
1 file changed, 70 insertions(+), 51 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 178d71e..0960b9b 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -57,39 +57,46 @@ class ParserBuildLog(BuildStep):
self.summery_dict = {}
self.index = 1
self.log_search_pattern_list = []
- self.max_text_lines = self.index -1
+ self.max_text_lines = 0
super().__init__(**kwargs)
+ #FIXME: ansifilter
+ def ansiFilter(self, text):
+ return text
+
@defer.inlineCallbacks
def get_log_search_pattern(self):
# get pattern from the projects
# add that to log_search_pattern_list
for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('project_data')['uuid'])):
- self.log_search_pattern_list.append(project_pattern)
- # get the default profile pattern
+ # check if the search pattern is vaild
+ try:
+ re.compile(project_pattern['search'])
+ except re.error:
+ print("Non valid regex pattern")
+ print(project_pattern)
+ else:
+ self.log_search_pattern_list.append(project_pattern)
+ # get the default project pattern
# add if not pattern is in project ignore
+ self.project_pattern_ignore = yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('project_data')['uuid'])
for project_pattern in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuid(self.getProperty('default_project_data')['uuid'])):
- match = True
- for project_pattern_ignore in (yield
self.gentooci.db.projects.getProjectLogSearchPatternByUuidAndIgnore(self.getProperty('default_project_data')['uuid'])):
- if project_pattern['search'] ==
project_pattern_ignore['search']:
- match = False
- if match:
- self.log_search_pattern_list.append(project_pattern)
+ if not project_pattern['search'] in self.project_pattern_ignore:
+ # check if the search pattern is vaild
+ try:
+ re.compile(project_pattern['search'])
+ except re.error:
+ print("Non valid regex pattern")
+ print(project_pattern)
+ else:
+ self.log_search_pattern_list.append(project_pattern)
def search_buildlog(self, tmp_index):
# get text line to search
- text_line = self.logfile_text_dict[tmp_index]
+ text_line = self.ansiFilter(self.logfile_text_dict[tmp_index])
# loop true the pattern list for match
for search_pattern in self.log_search_pattern_list:
search_hit = False
- # we add all line that start with ' * ' as info
- # we add all line that start with '>>>' but not '>>> /' as info
- if text_line.startswith(' * ') or (text_line.startswith('>>>') and
not text_line.startswith('>>> /')):
- self.summery_dict[tmp_index] = {}
- self.summery_dict[tmp_index]['text'] = text_line
- self.summery_dict[tmp_index]['type'] = 'info'
- self.summery_dict[tmp_index]['status'] = 'info'
- self.summery_dict[tmp_index]['search_pattern_id'] = 0
if search_pattern['search_type'] == 'in':
if search_pattern['search'] in text_line:
search_hit = True
@@ -100,63 +107,74 @@ class ParserBuildLog(BuildStep):
if text_line.endswith(search_pattern['search']):
search_hit = True
if search_pattern['search_type'] == 'search':
- if search_pattern['search'] in text_line:
+ if re.search(search_pattern['search'], text_line):
search_hit = True
+ # add the line if the pattern match
if search_hit:
print(text_line)
- print(search_pattern['search'])
+ print(search_pattern)
+ print(tmp_index)
self.summery_dict[tmp_index] = {}
self.summery_dict[tmp_index]['text'] = text_line
self.summery_dict[tmp_index]['type'] = search_pattern['type']
self.summery_dict[tmp_index]['status'] =
search_pattern['status']
self.summery_dict[tmp_index]['search_pattern_id'] =
search_pattern['id']
# add upper text lines if requested
- # max 10
- if search_pattern['start'] != 0 and search_hit:
- i = tmp_index
- i_start = i - search_pattern['start']
- match = True
- while match:
- i = i - 1
- if i < 0 or i < i_start:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
+ # max 5
+ if search_pattern['start'] != 0:
+ i = tmp_index - search_pattern['start'] - 1
+ match = True
+ while match:
+ i = i + 1
+ if i < (tmp_index - 9) or i == tmp_index:
+ match = False
+ else:
+ if not i in self.summery_dict:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] =
self.ansiFilter(self.logfile_text_dict[i])
+ self.summery_dict[i]['type'] = 'info'
+ self.summery_dict[i]['status'] = 'info'
# add lower text lines if requested
- # max 10
- if search_pattern['end'] != 0 and search_hit:
- i = tmp_index
- i_end = i + search_pattern['end']
- match = True
- while match:
- i = i + 1
- if i > self.max_text_lines or i > i_end:
- match = False
- else:
- self.summery_dict[i] = {}
- self.summery_dict[i]['text'] =
self.logfile_text_dict[i]
- self.summery_dict[i]['type'] = search_pattern['type']
- self.summery_dict[i]['status'] = 'info'
+ # max 5
+ if search_pattern['end'] != 0:
+ i = tmp_index
+ end = tmp_index + search_pattern['end']
+ match = True
+ while match:
+ i = i + 1
+ if i > self.max_text_lines or i > end:
+ match = False
+ else:
+ if not i in self.summery_dict:
+ self.summery_dict[i] = {}
+ self.summery_dict[i]['text'] =
self.ansiFilter(self.logfile_text_dict[i])
+ self.summery_dict[i]['type'] = 'info'
+ self.summery_dict[i]['status'] = 'info'
+ else:
+ # we add all line that start with ' * ' as info
+ # we add all line that start with '>>>' but not '>>> /' as info
+ if text_line.startswith(' * ') or (text_line.startswith('>>>')
and not text_line.startswith('>>> /')):
+ if not tmp_index in self.summery_dict:
+ self.summery_dict[tmp_index] = {}
+ self.summery_dict[tmp_index]['text'] = text_line
+ self.summery_dict[tmp_index]['type'] = 'info'
+ self.summery_dict[tmp_index]['status'] = 'info'
@defer.inlineCallbacks
def run(self):
self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
- #FIXME:
- # get the log parser pattern from db
yield self.get_log_search_pattern()
# open the log file
# read it to a buffer
# make a dict of the buffer
- # maby use mulitiprocces to speed up the search
+ # maybe use mulitiprocces to speed up the search
print(self.getProperty('log_build_data'))
if self.getProperty('faild_cpv'):
log_cpv =
self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
else:
log_cpv =
self.getProperty('log_build_data')[self.getProperty('cpv')]
file_path = yield os.path.join(self.master.basedir, 'cpv_logs',
log_cpv['full_logname'])
+ #FIXME: decode it to utf-8
with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb')))
as f:
for text_line in f:
self.logfile_text_dict[self.index] = text_line.strip('\n')
@@ -168,6 +186,7 @@ class ParserBuildLog(BuildStep):
if self.index >= 20:
del self.logfile_text_dict[self.index - 19]
self.index = self.index + 1
+ self.max_text_lines = self.index
f.close()
# check last 10 lines in logfile_text_dict
yield self.search_buildlog(self.index - 10)
