Close #3287 --- rtems/config/4.10/rtems-autotools.bset | 5 + source-builder/sb/macros.py | 3 + source-builder/sb/reports.py | 283 +++++++++++++++++++++++---------- source-builder/sb/setbuilder.py | 138 ++++++++++++---- 4 files changed, 307 insertions(+), 122 deletions(-)
diff --git a/rtems/config/4.10/rtems-autotools.bset b/rtems/config/4.10/rtems-autotools.bset index f7f1929..a15aa1b 100644 --- a/rtems/config/4.10/rtems-autotools.bset +++ b/rtems/config/4.10/rtems-autotools.bset @@ -18,5 +18,10 @@ # %define _internal_autotools_path %{_tmppath}/sb-%{_uid}/${SB_PREFIX_CLEAN} +# +# Disable emailing reports of this building for RTEMS. +# +%define mail_disable + 4.10/rtems-autotools-internal 4.10/rtems-autotools-base diff --git a/source-builder/sb/macros.py b/source-builder/sb/macros.py index 2af8d36..28a52b2 100644 --- a/source-builder/sb/macros.py +++ b/source-builder/sb/macros.py @@ -447,6 +447,9 @@ class macros: if key in self.macros[map]: del self.macros[map][key] + def defined(self, key, globals = True, maps = None): + return self.get(key, globals, maps) is not None + def expand(self, _str): """Simple basic expander of config file macros.""" expanded = True diff --git a/source-builder/sb/reports.py b/source-builder/sb/reports.py index 5eb8bb8..9d3a342 100644 --- a/source-builder/sb/reports.py +++ b/source-builder/sb/reports.py @@ -24,6 +24,7 @@ from __future__ import print_function +import base64 import copy import datetime import os @@ -63,6 +64,18 @@ def _make_path(p, *args): p = path.join(p, arg) return os.path.abspath(path.host(p)) +def platform(mode = 'all'): + import platform + if mode == 'system': + return platform.system() + compact = platform.platform(aliased = True) + if mode == 'compact': + return compact + extended = ' '.join(platform.uname()) + if mode == 'extended': + return extended + return '%s (%s)' % (short, extended) + class formatter(object): def __init__(self): self.content = '' @@ -139,127 +152,212 @@ class formatter(object): def post_process(self): return self.content -class asciidoc_formatter(formatter): +class markdown_formatter(formatter): def __init__(self): - super(asciidoc_formatter, self).__init__() + super(markdown_formatter, self).__init__() + self.level_current = 1 + self.level_path = '0.' + self.levels = { '0.': 0 } + self.cols = [20, 55] + + def _heading(self, heading, level): + return '%s %s' % ('#' * level, heading) + + def _strong(self, s): + return '__' + s + '__' + + def _bold(self, s): + return '__' + s + '__' + + def _italic(self, s): + return '_' + s + '_' + + def _table_line(self): + l = '|' + for c in self.cols: + l += '-' * c + '|' + return l + + def _table_row(self, cols): + if len(cols) != len(self.cols): + raise error.general('invalid table column count') + l = '|' + for c in range(0, len(cols)): + l += '%-*s|' % (self.cols[c], cols[c]) + return l + + def _btext(self, level, text): + return '> ' * (level - 1) + text + + def _bline(self, level, text): + self.line(self._btext(level, text)) + + def _level(self, nest_level): + if nest_level > self.level_current: + self.level_path += '%d.' % (self.levels[self.level_path]) + if nest_level < self.level_current: + self.level_path = self.level_path[:-2] + if self.level_path not in self.levels: + self.levels[self.level_path] = 0 + self.level_current = nest_level + self.levels[self.level_path] += 1 + return '%s%d.' % (self.level_path[2:], self.levels[self.level_path]) def format(self): - return 'asciidoc' + return 'markdown' def ext(self): - return '.txt' + return '.md' def introduction(self, name, now, intro_text): - h = 'RTEMS Source Builder Report' - self.line(h) - self.line('=' * len(h)) - self.line(':doctype: book') - self.line(':toc2:') - self.line(':toclevels: 5') - self.line(':icons:') - self.line(':numbered:') - self.line(':data-uri:') + self.line('- - -') + self.line(self._heading('RTEMS Source Builder Report', 1)) + self.line(self._strong(_title)) self.line('') - self.line(_title) - self.line(now) - self.line('') - image = _make_path(self.sbpath, options.basepath, 'images', 'rtemswhitebg.jpg') - self.line('image:%s["RTEMS",width="20%%"]' % (image)) + self.line(self._bold('Generated: ' + now)) self.line('') if intro_text: self.line('%s' % ('\n'.join(intro_text))) + self.line('') + self.line('') + self.line('- - -') + self.line(self._heading('Table Of Contents', 2)) + self.line('') + self.line('[TOC]') + self.line('') def release_status(self, release_string): self.line('') - self.line("'''") + self.line(self._heading(_release_status_text, 2)) self.line('') - self.line('.%s' % (_release_status_text)) self.line('*Version*: %s;;' % (release_string)) self.line('') - self.line("'''") - self.line('') def git_status(self, valid, dirty, head, remotes): self.line('') - self.line("'''") - self.line('') - self.line('.%s' % (_git_status_text)) + self.line('- - -') + self.line(self._heading(_git_status_text, 2)) if valid: - self.line('*Remotes*:;;') + self.line(self._strong('Remotes:')) + self.line('') + rc = 1 for r in remotes: if 'url' in remotes[r]: text = remotes[r]['url'] else: text = 'no URL found' - text = '%s: %s' % (r, text) - self.line('. %s' % (text)) - self.line('*Status*:;;') + self.line('%d. %s: %s' % (rc, r, text)) + rc += 1 + self.line('') + self.line(self._strong('Status:')) + self.line('') if dirty: - self.line('_Repository is dirty_') + self.line('> ' + self._italic('Repository is dirty')) else: - self.line('Clean') - self.line('*Head*:;;') - self.line('Commit: %s' % (head)) + self.line('> Clean') + self.line('>') + self.line('> ' + self._bold('Head: ') + head) else: - self.line('_Not a valid GIT repository_') - self.line('') - self.line("'''") + self.line('> ' + self._italic('Not a valid GIT repository')) self.line('') def config(self, nest_level, name, _config): - self.line('*Package*: _%s_ +' % (name)) - self.line('*Config*: %s' % (_config.file_name())) - self.line('') + self._bline(nest_level, self._bold('Package:')) + self._bline(nest_level, '') + self._bline(nest_level + 1, self._table_row([self._bold('Item'), + self._bold('Description')])) + self._bline(nest_level + 1, self._table_line()) + self._bline(nest_level + 1, self._table_row(['Package', name])) + self._bline(nest_level + 1, self._table_row(['Config', + _config.file_name()])) def config_end(self, nest_level, name): - self.line('') - self.line("'''") - self.line('') + self._bline(nest_level + 1, '') def buildset_start(self, nest_level, name): - h = '%s' % (name) - self.line('=%s %s' % ('=' * int(nest_level), h)) + if nest_level == 1: + self.line('- - -') + self._bline(nest_level, + self._heading('RTEMS Source Builder Packages', 2)) + self._bline(nest_level, + self._heading('%s Build %s' % (self._level(nest_level), name), 3)) def info(self, nest_level, name, info, separated): - end = '' - if separated: - self.line('*%s:*::' % (name)) - self.line('') - else: - self.line('*%s:* ' % (name)) - end = ' +' - spaces = '' - for l in info: - self.line('%s%s%s' % (spaces, l, end)) - if separated: - self.line('') + self._bline(nest_level + 1, + self._table_row([name, ' '.join(info)])) def directive(self, nest_level, name, data): - self.line('') - self.line('*%s*:' % (name)) - self.line('--------------------------------------------') + self._bline(nest_level, '') + self._bline(nest_level, self._bold(name + ':')) for l in data: - self.line(l) - self.line('--------------------------------------------') + self._bline(nest_level + 1, ' ' * 4 + l) def files(self, nest_level, singular, plural, _files): - self.line('') - self.line('*' + plural + ':*::') + self._bline(nest_level, '') + self._bline(nest_level, self._bold(plural + ':')) + self._bline(nest_level, '') if len(_files) == 0: - self.line('No ' + plural.lower()) + self._bline(nest_level + 1, 'No ' + plural.lower()) + fc = 0 for name in _files: for s in _files[name]: - self.line('. %s' % (s[0])) + fc += 1 if s[1] is None: - h = 'No checksum' + h = self._bold('No checksum') else: hash = s[1].split() h = '%s: %s' % (hash[0], hash[1]) - self.line('+\n%s\n' % (h)) + self._bline(nest_level, + '%d. [%s](%s "%s %s")<br/>' % (fc, s[0], s[0], + name, singular.lower())) + self._bline(nest_level, + ' <span class=checksum>%s</span>' % (h)) -class html_formatter(asciidoc_formatter): +class html_formatter(markdown_formatter): def __init__(self): super(html_formatter, self).__init__() + self.html_header = '<!DOCTYPE html>' + os.linesep + \ + '<html lang="en">' + os.linesep + \ + '<head>' + os.linesep + \ + '<title>RTEMS RSB - @BUILD@</title>' + os.linesep + \ + '<meta http-equiv="content-type" content="text/html; charset=UTF-8" />' + os.linesep + \ + '<meta name="created" content="@NOW@" />' + os.linesep + \ + '<meta name="description" content="RTEMS RSB Report" />' + os.linesep + \ + '<meta name="keywords" content="RTEMS RSB" />' + os.linesep + \ + '<meta charset="utf-8">' + os.linesep + \ + '<meta http-equiv="X-UA-Compatible" content="IE=edge">' + os.linesep + \ + '<meta name="viewport" content="width=device-width, initial-scale=1">' + os.linesep + \ + '<style type="text/css">' + os.linesep + \ + 'body {' + os.linesep + \ + ' font-family: arial, helvetica, serif;' + os.linesep + \ + ' font-style: normal;' + os.linesep + \ + ' font-weight: 400;' + os.linesep + \ + '}' + os.linesep + \ + 'h1, h2 { margin: 10px 5px 10px 5px; }' + os.linesep + \ + 'h1 { font-size: 28px; }' + os.linesep + \ + 'h2 { font-size: 22px;}' + os.linesep + \ + 'h3 { font-size: 18px; }' + os.linesep + \ + 'p, ol, blockquote, h3, table, pre { margin: 1px 20px 2px 7px; }' + os.linesep + \ + 'table, th, td, pre { border: 1px solid gray; border-spacing: 0px; }' + os.linesep + \ + 'table { width: 100%; }' + os.linesep + \ + 'th, td { padding: 1px; }' + os.linesep + \ + 'pre { padding: 4px; }' + os.linesep + \ + '.checksum { font-size: 12px; }' + os.linesep + \ + '</style>' + os.linesep + \ + '</head>' + os.linesep + \ + '<body>' + os.linesep + self.html_footer = '</body>' + os.linesep + \ + '</html>' + os.linesep + + def _logo(self): + logo = _make_path(self.sbpath, options.basepath, 'images', 'rtemswhitebg.jpg') + try: + with open(logo, "rb") as image: + b64 = base64.b64encode(image.read()) + except: + raise error.general('installation error: no logo found') + logo = '<img alt="RTEMS Project" height="100" src="data:image/png;base64,' + b64 + '" />' + return logo def format(self): return 'html' @@ -267,24 +365,30 @@ class html_formatter(asciidoc_formatter): def ext(self): return '.html' + def introduction(self, name, now, intro_text): + self.name = name + self.now = now + super(html_formatter, self).introduction(name, now, intro_text) + def post_process(self): - import io - infile = io.StringIO(self.content) - outfile = io.StringIO() try: - import asciidocapi + import markdown except: - raise error.general('installation error: no asciidocapi found') - asciidoc_py = _make_path(self.sbpath, options.basepath, 'asciidoc', 'asciidoc.py') + raise error.general('installation error: no markdown found') try: - asciidoc = asciidocapi.AsciiDocAPI(asciidoc_py) + out = markdown.markdown(self.content, + output_format = 'html5', + extensions = ['markdown.extensions.toc', + 'markdown.extensions.tables', + 'markdown.extensions.sane_lists', + 'markdown.extensions.smarty']) except: - raise error.general('application error: asciidocapi failed') - asciidoc.execute(infile, outfile) - out = outfile.getvalue() - infile.close() - outfile.close() - return out + raise + raise error.general('application error: markdown failed') + header = self.html_header.replace('@BUILD@', self.name).replace('@NOW@', self.now) + footer = self.html_footer + logo = self._logo() + return header + logo + out + footer class text_formatter(formatter): def __init__(self): @@ -503,8 +607,8 @@ class report: if type(formatter) == str: if formatter == 'text': self.formatter = text_formatter() - elif formatter == 'asciidoc': - self.formatter = asciidoc_formatter() + elif formatter == 'markdown': + self.formatter = markdown_formatter() elif formatter == 'html': self.formatter = html_formatter() elif formatter == 'ini': @@ -736,6 +840,9 @@ class report: self.generate_ini_source(sources) self.generate_ini_hash(sources) + def get_output(self): + return self.formatter.post_process() + def write(self, name): self.out = self.formatter.post_process() if name is not None: @@ -784,9 +891,9 @@ def run(args): try: optargs = { '--list-bsets': 'List available build sets', '--list-configs': 'List available configurations', - '--format': 'Output format (text, html, asciidoc, ini, xml)', + '--format': 'Output format (text, html, markdown, ini, xml)', '--output': 'File name to output the report' } - opts = options.load(args, optargs) + opts = options.load(args, optargs, logfile = False) if opts.get_arg('--output') and len(opts.params()) > 1: raise error.general('--output can only be used with a single config') print('RTEMS Source Builder, Reporter, %s' % (version.str())) @@ -805,8 +912,8 @@ def run(args): raise error.general('invalid format option: %s' % ('='.join(format_opt))) if format_opt[1] == 'text': pass - elif format_opt[1] == 'asciidoc': - formatter = asciidoc_formatter() + elif format_opt[1] == 'markdown': + formatter = markdown_formatter() elif format_opt[1] == 'html': formatter = html_formatter() elif format_opt[1] == 'ini': @@ -824,7 +931,7 @@ def run(args): outname = output config = build.find_config(_config, configs) if config is None: - raise error.general('config file not found: %s' % (inname)) + raise error.general('config file not found: %s' % (_config)) r.create(config, outname) del r else: diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py index 9fa19ec..17b781a 100644 --- a/source-builder/sb/setbuilder.py +++ b/source-builder/sb/setbuilder.py @@ -30,6 +30,7 @@ import glob import operator import os import sys +import textwrap try: import build @@ -49,6 +50,23 @@ except: print('error: unknown application load error', file = sys.stderr) sys.exit(1) +class log_capture(object): + def __init__(self): + self.log = [] + log.capture = self.capture + + def __str__(self): + return os.linesep.join(self.log) + + def capture(self, text): + self.log += [l for l in text.replace(chr(13), '').splitlines()] + + def get(self): + return self.log + + def clear(self): + self.log = [] + class buildset: """Build a set builds a set of packages.""" @@ -71,30 +89,43 @@ class buildset: self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset) self.mail_header = '' self.mail_report = '' + self.mail_report_0subject = '' self.build_failure = None - def write_mail_header(self, text, prepend = False): - if len(text) == 0 or text[-1] != '\n' or text[-1] != '\r': + def write_mail_header(self, text = '', prepend = False): + if type(text) is list: + text = os.linesep.join(text) + text = text.replace('\r', '').replace('\n', os.linesep) + if len(text) == 0 or text[-1] != os.linesep: text += os.linesep if prepend: self.mail_header = text + self.mail_header else: self.mail_header += text + def get_mail_header(self): + return self.mail_header + def write_mail_report(self, text, prepend = False): - if len(text) == 0 or text[-1] != '\n' or text[-1] != '\r': + if type(text) is list: + text = os.linesep.join(text) + text = text.replace('\r', '').replace('\n', os.linesep) + if len(text) == 0 or text[-1] != os.linesep: text += os.linesep if prepend: self.mail_report = text + self.mail_report else: self.mail_report += text + def get_mail_report(self): + return self.mail_report + def copy(self, src, dst): log.output('copy: %s => %s' % (path.host(src), path.host(dst))) if not self.opts.dry_run(): path.copy_tree(src, dst) - def report(self, _config, _build, opts, macros, format = None): + def report(self, _config, _build, opts, macros, format = None, mail = None): if len(_build.main_package().name()) > 0 \ and not _build.macros.get('%{_disable_reporting}') \ and (not _build.opts.get_arg('--no-report') \ @@ -139,13 +170,13 @@ class buildset: _build.mkdir(outpath) r.write(outname) del r - if _build.opts.get_arg('--mail'): + if mail: r = reports.report('text', self.configs, copy.copy(opts), copy.copy(macros)) r.introduction(_build.config.file_name()) r.generate(_build.config.file_name()) r.epilogue(_build.config.file_name()) - self.write_mail_report(r.out) + self.write_mail_report(r.get_output()) del r def root_copy(self, src, dst): @@ -299,17 +330,20 @@ class buildset: configs = self.parse(bset) return configs - def build(self, deps = None, nesting_count = 0): + def build(self, deps = None, nesting_count = 0, mail = None): build_error = False nesting_count += 1 + if mail: + mail['output'].clear() + log.trace('_bset: %s: make' % (self.bset)) log.notice('Build Set: %s' % (self.bset)) - if self.opts.get_arg('--mail'): - mail_report_subject = '%s %s' % (self.bset, self.macros.expand('%{_host}')) + mail_subject = '%s on %s' % (self.bset, + self.macros.expand('%{_host}')) current_path = os.environ['PATH'] @@ -318,6 +352,9 @@ class buildset: mail_report = False have_errors = False + if mail: + mail['output'].clear() + try: configs = self.load() @@ -337,14 +374,17 @@ class buildset: if configs[s].endswith('.bset'): log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75)) bs = buildset(configs[s], self.configs, opts, macros) - bs.build(deps, nesting_count) + bs.build(deps, nesting_count, mail) del bs elif configs[s].endswith('.cfg'): - mail_report = self.opts.get_arg('--mail') + if mail: + mail_report = True log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75)) try: - b = build.build(configs[s], self.opts.get_arg('--pkg-tar-files'), - opts, macros) + b = build.build(configs[s], + self.opts.get_arg('--pkg-tar-files'), + opts, + macros) except: build_error = True raise @@ -354,12 +394,14 @@ class buildset: self.build_package(configs[s], b) self.report(configs[s], b, copy.copy(self.opts), - copy.copy(self.macros)) - # Always product an XML report. + copy.copy(self.macros), + mail = mail) + # Always produce an XML report. self.report(configs[s], b, copy.copy(self.opts), copy.copy(self.macros), - format = 'xml') + format = 'xml', + mail = mail) if s == len(configs) - 1 and not have_errors: self.bset_tar(b) else: @@ -428,25 +470,29 @@ class buildset: end = datetime.datetime.now() os.environ['PATH'] = current_path build_time = str(end - start) - if mail_report: - to_addr = self.opts.get_arg('--mail-to') - if to_addr is not None: - to_addr = to_addr[1] - else: - to_addr = self.macros.expand('%{_mail_tools_to}') - log.notice('Mailing report: %s' % (to_addr)) - self.write_mail_header('Build Time %s' % (build_time), True) - self.write_mail_header('') - m = mailer.mail(self.opts) + if mail_report and not self.macros.defined('mail_disable'): + self.write_mail_header('Build Time: %s' % (build_time), True) + self.write_mail_header('', True) if self.build_failure is not None: - mail_report_subject = 'Build: FAILED %s (%s)' %\ - (mail_report_subject, self.build_failure) - pass_fail = 'FAILED' + mail_subject = 'FAILED %s (%s)' % \ + (mail_subject, self.build_failure) else: - mail_report_subject = 'Build: PASSED %s' % (mail_report_subject) - if not self.opts.dry_run(): - m.send(to_addr, mail_report_subject, - self.mail_header + self.mail_report) + mail_subject = 'PASSED %s' % (mail_subject) + mail_subject = 'Build %s: %s' % (reports.platform(mode = 'system'), + mail_subject) + self.write_mail_header(mail['header'], True) + self.write_mail_header('') + log.notice('Mailing report: %s' % (mail['to'])) + body = self.get_mail_header() + body += 'Output' + os.linesep + body += '======' + os.linesep + os.linesep + body += os.linesep.join(mail['output'].get()) + body += os.linesep + os.linesep + body += 'Report' + os.linesep + body += '======' + os.linesep + os.linesep + body += self.get_mail_report() + if not opts.dry_run(): + mail['mail'].send(mail['to'], mail_subject, body) log.notice('Build Set: Time %s' % (build_time)) def list_bset_cfg_files(opts, configs): @@ -467,6 +513,7 @@ def run(): import sys ec = 0 setbuilder_error = False + mail = None try: optargs = { '--list-configs': 'List available configurations', '--list-bsets': 'List available build sets', @@ -477,10 +524,27 @@ def run(): '--report-format': 'The report format (text, html, asciidoc).' } mailer.append_options(optargs) opts = options.load(sys.argv, optargs) + if opts.get_arg('--mail'): + mail = { 'mail' : mailer.mail(opts), + 'output': log_capture() } + to_addr = opts.get_arg('--mail-to') + if to_addr is not None: + mail['to'] = to_addr[1] + else: + mail['to'] = opts.defaults.expand('%{_mail_tools_to}') + mail['from'] = mail['mail'].from_address() log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str())) opts.log_info() if not check.host_setup(opts): raise error.general('host build environment is not set up correctly') + if mail: + mail['header'] = os.linesep.join(mail['output'].get()) + os.linesep + mail['header'] += os.linesep + mail['header'] += 'Host: ' + reports.platform('compact') + os.linesep + indent = ' ' + for l in textwrap.wrap(reports.platform('extended'), + width = 80 - len(indent)): + mail['header'] += indent + l + os.linesep configs = build.get_configs(opts) if opts.get_arg('--list-deps'): deps = [] @@ -496,12 +560,14 @@ def run(): not opts.no_install() and \ not path.ispathwritable(prefix): raise error.general('prefix is not writable: %s' % (path.host(prefix))) + for bset in opts.params(): setbuilder_error = True b = buildset(bset, configs, opts) - b.build(deps) + b.build(deps, mail = mail) b = None setbuilder_error = False + if deps is not None: c = 0 for d in sorted(set(deps)): @@ -522,6 +588,10 @@ def run(): except KeyboardInterrupt: log.notice('abort: user terminated') ec = 1 + except: + raise + log.notice('abort: unknown error') + ec = 1 sys.exit(ec) if __name__ == "__main__": -- 2.14.3 (Apple Git-98) _______________________________________________ devel mailing list devel@rtems.org http://lists.rtems.org/mailman/listinfo/devel