From: Chris Johns <chr...@rtems.org> --- rtemstoolkit/check.py | 36 ++++---- rtemstoolkit/config.py | 60 +++++++------ rtemstoolkit/configuration.py | 35 ++++++-- rtemstoolkit/darwin.py | 13 ++- rtemstoolkit/execute.py | 16 ++-- rtemstoolkit/freebsd.py | 14 ++-- rtemstoolkit/git.py | 24 ++---- rtemstoolkit/host.py | 8 +- rtemstoolkit/linux.py | 18 ++-- rtemstoolkit/log.py | 15 ++-- rtemstoolkit/macros.py | 52 +++++++----- rtemstoolkit/mailer.py | 53 ++++++++---- rtemstoolkit/netbsd.py | 6 +- rtemstoolkit/options.py | 140 +++++++++++++++---------------- rtemstoolkit/path.py | 16 ++-- rtemstoolkit/reraise.py | 6 +- rtemstoolkit/rtems.py | 133 ++++++++++++++++++++++------- rtemstoolkit/solaris.py | 6 +- rtemstoolkit/stacktraces.py | 41 +++++++-- rtemstoolkit/textbox.py | 6 +- rtemstoolkit/version.py | 26 +++--- tester/rt/check.py | 190 +++++++++++++++++++++++------------------- tester/rt/run.py | 4 +- tester/rt/test.py | 4 +- 24 files changed, 538 insertions(+), 384 deletions(-)
diff --git a/rtemstoolkit/check.py b/rtemstoolkit/check.py index c6549bf..44cf6ba 100644 --- a/rtemstoolkit/check.py +++ b/rtemstoolkit/check.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.check +# + # # Check the defaults for a specific host. # @@ -36,24 +40,12 @@ from __future__ import print_function import os -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import execute - from . import log - from . import options - from . import path - from . import version -except (ValueError, SystemError): - import error - import execute - import log - import options - import path - import version +from rtemstoolkit import error +from rtemstoolkit import execute +from rtemstoolkit import log +from rtemstoolkit import options +from rtemstoolkit import path +from rtemstoolkit import version def _check_none(_opts, macro, value, constraint): return True @@ -97,12 +89,14 @@ def _check_exe(_opts, macro, value, constraint, silent = False): if _check_paths(value, paths): if absexe: if not silent: - log.notice('warning: exe: absolute exe found in path: (%s) %s' % (macro, orig_value)) + log.notice('warning: exe: absolute exe found in path: (%s) %s' % \ + (macro, orig_value)) return True if constraint == 'optional': if not silent: - log.trace('warning: exe: optional exe not found: (%s) %s' % (macro, orig_value)) + log.trace('warning: exe: optional exe not found: (%s) %s' % \ + (macro, orig_value)) return True if not silent: @@ -168,7 +162,7 @@ def run(): try: _opts = options.command_line(argv = sys.argv) options.load(_opts) - log.notice('RTEMS Source Builder - Check, v%s' % (version.string())) + log.notice('RTEMS Toolkit - Check, v%s' % (version.string())) if host_setup(_opts): print('Environment is ok') else: diff --git a/rtemstoolkit/config.py b/rtemstoolkit/config.py index a16261b..0ec93de 100644 --- a/rtemstoolkit/config.py +++ b/rtemstoolkit/config.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.config +# + # # This code is based on a tool I wrote to parse RPM spec files in the RTEMS # project. This is now a configuration file format that has moved away from the @@ -44,24 +48,12 @@ import os import re import sys -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import execute - from . import host - from . import log - from . import options - from . import path -except (ValueError, SystemError): - import error - import execute - import host - import log - import options - import path +from rtemstoolkit import error +from rtemstoolkit import execute +from rtemstoolkit import host +from rtemstoolkit import log +from rtemstoolkit import options +from rtemstoolkit import path def _check_bool(value): if value.isdigit(): @@ -756,13 +748,16 @@ class file(object): break configname = None if configname is None: - raise error.general('no config file found: %s' % (cfgname)) + raise error.general('no config file found: %s' % \ + (path.basename(cfgname))) try: - log.trace('config: %s: _open: %s' % (self.init_name, path.host(configname))) + log.trace('config: %s: _open: %s' % (self.init_name, + path.host(configname))) config = open(path.host(configname), 'r') except IOError as err: - raise error.general('error opening config file: %s' % (path.host(configname))) + raise error.general('error opening config file: %s' % \ + (path.host(configname))) self.configpath += [configname] self._includes += [configname] @@ -845,21 +840,34 @@ def run(): import sys try: # - # Run where defaults.mc is located + # Run from the top of RTEMS tools with: + # + # $ python -m rtemstoolkit.config \ + # --file tester/rtems/testing/gdb.cfg \ + # --config tester/rtems/testing:tester/config \ + # --rtdir tester # long_opts = { - # key macro handler param defs init - '--file' : ('_file', 'path', True, None, False) + # key macro handler param defs init + '--file' : ('_file', 'path', True, None, False), + '--config' : ('_configdir', 'string', True, None, False), + '--rtdir' : ('_rtdir', 'path', True, None, False) } opts = options.command_line(base_path = '.', argv = sys.argv, long_opts = long_opts) options.load(opts) - s = file(opts.defaults['_file'], opts) + if opts.defaults.get_value('_configdir') is None: + raise error.general('no --config argument') + f = opts.defaults.get_value('_file') + if f is None: + raise error.general('no --file argument') + s = file(f, opts) s.load(opts.defaults['_file']) print(s) del s except error.general as gerr: + raise print(gerr) sys.exit(1) except error.internal as ierr: diff --git a/rtemstoolkit/configuration.py b/rtemstoolkit/configuration.py index 10d97e5..04493f9 100644 --- a/rtemstoolkit/configuration.py +++ b/rtemstoolkit/configuration.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.configuration +# + # # Host specifics. # @@ -47,10 +51,12 @@ from rtemstoolkit import path class configuration: - def __init__(self): + def __init__(self, name = None): self.config = configparser.ConfigParser() self.ini = None self.macro_filter = re.compile('\$\{.+\}') + if name is not None: + self.load(name) def __str__(self): if self.ini is None: @@ -95,7 +101,7 @@ class configuration: pass return rec - def get_items(self, section, err = True, flatten = True): + def get_items(self, section, err = True, flatten = True, dictionary = False): try: items = [] for name, key in self.config.items(section): @@ -103,17 +109,24 @@ class configuration: items += [(name, key.replace(os.linesep, ' '))] else: items += [(name, key)] + if dictionary: + items = dict(items) return items except: if err: raise error.general('config: section "%s" not found' % (section)) + if dictionary: + return {} return [] - def comma_list(self, section, label, err = True): + def comma_list(self, section, label, err = True, sort = True): items = self.get_item(section, label, err) if items is None: return [] - return sorted(set([a.strip() for a in items.split(',')])) + items = set([a.strip() for a in items.split(',')]) + if sort: + return sorted(items) + return items def get_item_names(self, section, err = True): try: @@ -126,12 +139,13 @@ class configuration: def has_section(self, section): return self.config.has_section(section) - def load(self, name): + def load(self, name, trace = False): # # Load all the files. # - self.ini = { 'base' : path.dirname(name), - 'files' : [] } + if self.ini is None: + self.ini = { 'base' : path.dirname(name), + 'files' : [] } includes = [name] still_loading = True while still_loading: @@ -146,6 +160,8 @@ class configuration: include = rebased_inc if include not in self.ini['files']: try: + if trace: + print('config: read: %s' % (include)) self.config.read(include) except configparser.ParsingError as ce: raise error.general('config: %s' % (ce)) @@ -155,6 +171,11 @@ class configuration: if still_loading: for section in self.config.sections(): includes += self.comma_list(section, 'include', err = False) + if trace and len(includes) > 0: + print('config: includes: %s' % (','.join(includes))) def files(self): return self.ini['files'] + +if __name__ == "__main__": + print('No unittest, please add') diff --git a/rtemstoolkit/darwin.py b/rtemstoolkit/darwin.py index 0400174..57fca85 100644 --- a/rtemstoolkit/darwin.py +++ b/rtemstoolkit/darwin.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.darwin +# + # # This code is based on what ever doco about spec files I could find and # RTEMS project's spec files. @@ -35,14 +39,7 @@ import os -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import execute -except (ValueError, SystemError): - import execute +from rtemstoolkit import execute def cpus(): sysctl = '/usr/sbin/sysctl ' diff --git a/rtemstoolkit/execute.py b/rtemstoolkit/execute.py index 8e09b81..09532e5 100755 --- a/rtemstoolkit/execute.py +++ b/rtemstoolkit/execute.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.config +# + # # Execute commands or scripts. # @@ -46,16 +50,8 @@ import threading import time import traceback -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import log -except (ValueError, SystemError): - import error - import log +from rtemstoolkit import error +from rtemstoolkit import log # Trace exceptions trace_threads = False diff --git a/rtemstoolkit/freebsd.py b/rtemstoolkit/freebsd.py index 2842394..609fef4 100644 --- a/rtemstoolkit/freebsd.py +++ b/rtemstoolkit/freebsd.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2017 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.freebsd +# + # # This code is based on what ever doco about spec files I could find and # RTEMS project's spec files. @@ -39,12 +43,8 @@ import os # Support to handle use in a package and as a unit test. # If there is a better way to let us know. # -try: - from . import check - from . import execute -except (ValueError, SystemError): - import check - import execute +from rtemstoolkit import check +from rtemstoolkit import execute def cpus(): sysctl = '/sbin/sysctl ' diff --git a/rtemstoolkit/git.py b/rtemstoolkit/git.py index 5f3af58..ba6e8c0 100644 --- a/rtemstoolkit/git.py +++ b/rtemstoolkit/git.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -25,26 +25,20 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.git +# + # # Provide some basic access to the git command. # import os -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import execute - from . import log - from . import path -except (ValueError, SystemError): - import error - import execute - import log - import path +from rtemstoolkit import error +from rtemstoolkit import execute +from rtemstoolkit import log +from rtemstoolkit import path class repo: """An object to manage a git repo.""" diff --git a/rtemstoolkit/host.py b/rtemstoolkit/host.py index ea23b24..f284eef 100644 --- a/rtemstoolkit/host.py +++ b/rtemstoolkit/host.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2017 Chris Johns (chr...@rtems.org) +# Copyright 2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.host +# + # # Host specifics. # @@ -113,9 +117,9 @@ def label(mode = 'all'): if __name__ == '__main__': import pprint - pprint.pprint(platform()) _load() print('Name : %s' % (name)) + print('Label : %s' % (label())) if is_windows: status = 'Yes' else: diff --git a/rtemstoolkit/linux.py b/rtemstoolkit/linux.py index 1d7f577..03a2226 100644 --- a/rtemstoolkit/linux.py +++ b/rtemstoolkit/linux.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2017 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.linux +# + # # This code is based on what ever doco about spec files I could find and # RTEMS project's spec files. @@ -36,16 +40,8 @@ import os import platform -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import execute - from . import path -except (ValueError, SystemError): - import execute - import path +from rtemstoolkit import execute +from rtemstoolkit import path def cpus(): processors = '/bin/grep processor /proc/cpuinfo' diff --git a/rtemstoolkit/log.py b/rtemstoolkit/log.py index 3eb2c1b..2a3d1d8 100755 --- a/rtemstoolkit/log.py +++ b/rtemstoolkit/log.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-testing'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.freebsd +# + # # Log output to stdout and/or a file. # @@ -38,14 +42,7 @@ import os import sys import threading -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error -except (ValueError, SystemError): - import error +from rtemstoolkit import error # # A global log. diff --git a/rtemstoolkit/macros.py b/rtemstoolkit/macros.py index ed8cd96..9422705 100644 --- a/rtemstoolkit/macros.py +++ b/rtemstoolkit/macros.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.macros +# + # # Macro tables. # @@ -40,24 +44,17 @@ import re import os import string -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import log - from . import path -except (ValueError, SystemError): - import error - import log - import path +from rtemstoolkit import error +from rtemstoolkit import log +from rtemstoolkit import path # # Macro tables # class macros: + empty = {} + class macro_iterator: def __init__(self, keys): self.keys = keys @@ -97,21 +94,22 @@ class macros: self.read_maps = [] self.read_map_locked = False self.write_map = 'global' - self.rtpath = path.abspath(path.dirname(inspect.getfile(macros))) - if path.dirname(self.rtpath).endswith('/share/rtems'): - self.prefix = path.dirname(self.rtpath)[:-len('/share/rtems')] - else: - self.prefix = '.' self.macros['global'] = {} self.macros['global']['nil'] = ('none', 'none', '') self.macros['global']['_cwd'] = ('dir', 'required', path.abspath(os.getcwd())) - self.macros['global']['_prefix'] = ('dir', 'required', self.prefix) - self.macros['global']['_rtdir'] = ('dir', - 'required', - path.abspath(self.expand(rtdir))) - self.macros['global']['_rttop'] = ('dir', 'required', self.prefix) + if rtdir is not None: + self.rtpath = path.abspath(path.dirname(inspect.getfile(macros))) + if path.dirname(self.rtpath).endswith('/share/rtems'): + self.prefix = path.dirname(self.rtpath)[:-len('/share/rtems')] + else: + self.prefix = '.' + self.macros['global']['_rttop'] = ('dir', 'required', self.prefix) + self.macros['global']['_prefix'] = ('dir', 'required', self.prefix) + self.macros['global']['_rtdir'] = ('dir', + 'required', + path.abspath(self.expand(rtdir))) else: self.macros = {} for m in original.macros: @@ -195,6 +193,8 @@ class macros: if type(key) is not str: raise TypeError('bad key type (want str): %s' % (type(key))) if type(value) is not tuple: + if type(value) is not str or type(value) is not unicode: + value = str(value) value = self._unicode_to_str(value) if type(value) is str: value = ('none', 'none', value) @@ -463,6 +463,8 @@ class macros: def expand(self, _str): """Simple basic expander of config file macros.""" + if type(_str) is not str and type(_str) is not buffer: + raise TypeError('bad str type: %s' % (type(_str))) start_str = _str expanded = True count = 0 @@ -523,6 +525,10 @@ class macros: return True def lock_read_map(self): + '''Lock the read map so it cannot be changed. This is useful if you wish to + override and force a map onto code that sets a new read mao, + + ''' self.read_map_locked = True def unlock_read_map(self): diff --git a/rtemstoolkit/mailer.py b/rtemstoolkit/mailer.py index b5a0a1e..0ff7164 100644 --- a/rtemstoolkit/mailer.py +++ b/rtemstoolkit/mailer.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2013-2016 Chris Johns (chr...@rtems.org) +# Copyright 2013-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.mailer +# + # # Manage emailing results or reports. # @@ -38,18 +42,9 @@ import os import smtplib import socket -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import options - from . import path -except (ValueError, SystemError): - import error - import options - import path +from rtemstoolkit import error +from rtemstoolkit import options +from rtemstoolkit import path _options = { '--mail' : 'Send email report or results.', @@ -72,7 +67,7 @@ class mail: self.opts = opts def _args_are_macros(self): - return type(self.opts) is 'command_line' + return isinstance(self.opts, options.command_line) def _get_arg(self, arg): if self._args_are_macros(): @@ -87,6 +82,14 @@ class mail: value = None return value + def to_address(self, default = None): + addr = self._get_arg('--mail-to') + if addr is None: + addr = default + if addr is None: + raise error.general('no valid to address for mail') + return addr + def from_address(self): def _clean(l): @@ -130,7 +133,7 @@ class mail: def smtp_host(self): host = self._get_arg('--smtp-host') if host is not None: - return host[1] + return host if self._args_are_macros(): host = self.opts.defaults.get_value('%{_mail_smtp_host}') if host is not None: @@ -160,11 +163,27 @@ class mail: self.send(to_addr, from_addr, body) if __name__ == '__main__': + import copy import sys optargs = {} + args = sys.argv + for o in ['--mail', + '--smtp-host=localhost', + '--mail-to=you@sthere', + '--mail-from=me@here']: + if o.split('=')[0] not in args: + args += [o] append_options(optargs) - opts = options.load(sys.argv, optargs = optargs, defaults = 'defaults.mc') + opts = options.command_line(base_path = '.', + argv = args, + optargs = optargs, + defaults = 'rtemstoolkit/defaults.mc') + options.load(opts) m = mail(opts) + print('To: %s' % (m.to_address())) print('From: %s' % (m.from_address())) print('SMTP Host: %s' % (m.smtp_host())) - m.send(m.from_address(), 'Test mailer.py', 'This is a test') + try: + m.send(m.from_address(), 'Test mailer.py', 'This is a test') + except error.general as gerr: + print(gerr) diff --git a/rtemstoolkit/netbsd.py b/rtemstoolkit/netbsd.py index 112fba1..42af372 100644 --- a/rtemstoolkit/netbsd.py +++ b/rtemstoolkit/netbsd.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2017 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -19,6 +19,10 @@ # along with RTEMS Tools. If not, see <http://www.gnu.org/licenses/>. # +# +# Unittest: python -m rtemstoolkit.netbsd +# + # # This code is based on what ever doco about spec files I could find and # RTEMS project's spec files. diff --git a/rtemstoolkit/options.py b/rtemstoolkit/options.py index 77d9593..4425fe4 100644 --- a/rtemstoolkit/options.py +++ b/rtemstoolkit/options.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2016 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.options +# + # # Determine the defaults and load the specific file. # @@ -42,31 +46,70 @@ import os import string import sys -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import execute - from . import git - from . import host - from . import log - from . import macros - from . import path - from . import version -except (ValueError, SystemError): - import error - import execute - import git - import host - import log - import macros - import path - import version +from rtemstoolkit import error +from rtemstoolkit import execute +from rtemstoolkit import git +from rtemstoolkit import host +from rtemstoolkit import log +from rtemstoolkit import macros +from rtemstoolkit import path +from rtemstoolkit import version basepath = 'tb' +def jobs(cpus, opt = None, macros = None): + try: + cpus = int(cpus) + except: + raise error.general('jobs: invalid host cpu value') + if opt is not None: + if opt == 'default': + if macros is None: + _jobs = cpus + else: + _jobs = macros.get_value('jobs') + if _jobs is not None: + if _jobs == 'none': + cpus = 0 + elif _jobs == 'max': + pass + elif _jobs == 'half': + cpus = cpus / 2 + else: + try: + cpus = int(_jobs) + except: + raise error.general('jobs: invalid %%{jobs} macro: %s' % (_jobs)) + else: + opt = 'max' + if opt != 'default': + if opt == 'none': + cpus = 0 + elif opt == 'max': + pass + elif opt == 'half': + cpus = cpus / 2 + else: + ok = False + try: + i = int(opt) + cpus = i + ok = True + except: + pass + if not ok: + try: + f = float(opt) + cpus = f * cpus + ok = True + except: + pass + if not ok: + raise error.internal('bad jobs option: %s' % (opt)) + if cpus <= 0: + cpu = 1 + return cpus + class command_line(object): """Process the command line in a common way for all Tool Builder commands.""" @@ -90,6 +133,8 @@ class command_line(object): if defaults is None: defaults = macros.macros() + elif type(defaults) is str: + defaults = macros.macros(name = defaults) self.long_opts = { # key macro handler param defs init @@ -442,54 +487,7 @@ class command_line(object): return um if len(um) else None def jobs(self, cpus): - try: - cpus = int(cpus) - except: - raise error.general('invalid host cpu value') - opt_jobs = self.opts['jobs'] - if opt_jobs == 'default': - _jobs = self.defaults.get_value('jobs') - if _jobs is not None: - if _jobs == 'none': - cpus = 0 - elif _jobs == 'max': - pass - elif _jobs == 'half': - cpus = cpus / 2 - else: - try: - cpus = int(_jobs) - except: - raise error.general('invalid %%{jobs} value: %s' % (_jobs)) - else: - opt_jobs = 'max' - if opt_jobs != 'default': - if opt_jobs == 'none': - cpus = 0 - elif opt_jobs == 'max': - pass - elif opt_jobs == 'half': - cpus = cpus / 2 - else: - ok = False - try: - i = int(opt_jobs) - cpus = i - ok = True - except: - pass - if not ok: - try: - f = float(opt_jobs) - cpus = f * cpus - ok = True - except: - pass - if not ok: - raise error.internal('bad jobs option: %s' % (opt_jobs)) - if cpus <= 0: - cpu = 1 - return cpus + return jobs(cpus, opt = self.opts['jobs'], macros = self.defaults) def params(self): return self.opts['params'] diff --git a/rtemstoolkit/path.py b/rtemstoolkit/path.py index 760f4bd..cc7f213 100644 --- a/rtemstoolkit/path.py +++ b/rtemstoolkit/path.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.path +# + # # Manage paths locally. The internally the path is in Unix or shell format and # we convert to the native format when performing operations at the Python @@ -41,16 +45,8 @@ import os import shutil import string -# -# Support to handle use in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import log -except (ValueError, SystemError): - import error - import log +from rtemstoolkit import error +from rtemstoolkit import log windows = os.name == 'nt' diff --git a/rtemstoolkit/reraise.py b/rtemstoolkit/reraise.py index 4b80861..f1b8891 100644 --- a/rtemstoolkit/reraise.py +++ b/rtemstoolkit/reraise.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2013-2017 Chris Johns (chr...@rtems.org) +# Copyright 2013-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.reraise +# + from __future__ import print_function import sys diff --git a/rtemstoolkit/rtems.py b/rtemstoolkit/rtems.py index 8aa22e5..cbd80b5 100755 --- a/rtemstoolkit/rtems.py +++ b/rtemstoolkit/rtems.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.rtems +# + from __future__ import print_function import copy @@ -48,7 +52,7 @@ _prefix_path = '/opt/rtems' def default_prefix(): from rtemstoolkit import version - return path.join(_prefix_path, version.version()) + return path.join(_prefix_path, str(version.version())) def clean_windows_path(): '''On Windows MSYS2 prepends a path to itself to the environment path. This @@ -62,23 +66,41 @@ def clean_windows_path(): if 'msys' in cspath[0] and cspath[0].endswith('bin'): os.environ['PATH'] = os.pathsep.join(cspath[1:]) -def configuration_path(): - '''Return the path the configuration data path for RTEMS. The path is relative - to the installed executable. Mangage the installed package and the in source - tree when running from within the rtems-tools repo. +def shared_path(name): + '''Return the path to a shared file folder for RTEMS. The path is relative to + the installed executable or we are testing and running from within the + rtems-tools repo. + + ''' + exec_name = os.path.abspath(sys.argv[0]) + for top in [path.dirname(exec_name), + path.dirname(path.dirname(exec_name))]: + shared = path.join(top, 'share', 'rtems', name) + if path.exists(shared): + break + shared = path.join(top, name) + if path.exists(shared): + break + shared = None + return shared + +def shared_file(name): + '''Return the path to a shared file for RTEMS. The path is relative to the + installed executable or we are testing and running from within the + rtems-tools repo. ''' exec_name = os.path.abspath(sys.argv[0]) - for top in [os.path.dirname(exec_name), - os.path.dirname(os.path.dirname(exec_name))]: - config_path = path.join(top, 'share', 'rtems', 'config') - if path.exists(config_path): + for top in [path.dirname(exec_name), + path.dirname(path.dirname(exec_name))]: + name_path = path.join(top, 'share', 'rtems', name) + if path.exists(name_path): break - config_path = path.join(top, 'config') - if path.exists(config_path): + name_path = path.join(top, name) + if path.exists(name_path): break - config_path = None - return config_path + name_path = None + return name_path def configuration_file(config): '''Return the path to a configuration file for RTEMS. The path is relative to @@ -86,9 +108,9 @@ def configuration_file(config): rtems-tools repo. ''' - return path.join(configuration_path(), config) + return shared_file(os.path.join('config', config)) -def bsp_configuration_file(): +def default_configuration_file(): '''Return the path to the BSP configuration file for RTEMS. The path is relative to the installed executable or we are testing and running from within the rtems-tools repo. @@ -96,6 +118,14 @@ def bsp_configuration_file(): ''' return configuration_file('rtems-bsps.ini') +def arch_bsp(ab): + '''Split an arch/bsp string into it's arch and bsp parts.''' + if isinstance(ab, str): + abl = ab.split('/') + if len(abl) == 2: + return abl[0], abl[1] + return None, None + class configuration: def __init__(self): @@ -133,7 +163,7 @@ class configuration: options += [opt] return options - def load(self, name, build): + def load(self, name, build = None): self.config.load(name) archs = [] self.profiles['profiles'] = \ @@ -243,7 +273,9 @@ class configuration: return sorted(self.archs[arch]['bsps']) def bsp_present(self, arch, bsp): - return bsp in self.archs[arch]['bsps'] + if arch in self.archs: + return bsp in self.archs[arch]['bsps'] + return False def bsp_excludes(self, arch, bsp): excludes = self.archs[arch]['excludes'].keys() @@ -275,20 +307,22 @@ class configuration: raise error.general('invalid profile arch: %s' % (arch)) return ['%s/%s' % (arch, bsp) for bsp in self.profiles[profile]['bsps_%s' % (arch)]] - def report(self, profiles = True, builds = True, architectures = True): + def report(self, profiles = True, builds = True, architectures = True, files = False): width = 70 cols_1 = [width] cols_2 = [10, width - 10] - s = textbox.line(cols_1, line = '=', marker = '+', indent = 1) - s1 = ' File(s)' - for f in self.config.files(): - colon = ':' - for l in textwrap.wrap(f, width = cols_2[1] - 3): - s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1) - colon = ' ' - s1 = ' ' * len(s1) - s += textbox.line(cols_1, marker = '+', indent = 1) - s += os.linesep + s = '' + if files: + s += textbox.line(cols_1, line = '=', marker = '+', indent = 1) + s1 = ' File(s)' + for f in self.config.files(): + colon = ':' + for l in textwrap.wrap(f, width = cols_2[1] - 3): + s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1) + colon = ' ' + s1 = ' ' * len(s1) + s += textbox.line(cols_1, marker = '+', indent = 1) + s += os.linesep if profiles: s += textbox.line(cols_1, line = '=', marker = '+', indent = 1) profiles = sorted(self.profiles['profiles']) @@ -413,3 +447,46 @@ class configuration: s += textbox.line(cols_b, marker = '+', indent = 1) s += os.linesep return s + + def report_args(self, opts): + if 'rtems_config_report' in opts: + profiles = False + builds = False + archs = False + files = False + if opts.rtems_config_report == 'all': + profiles = True + builds = True + archs = True + files = True + elif opts.rtems_config_report == 'profiles': + profiles = True + elif opts.rtems_config_report == 'builds': + builds = True + elif opts.rtems_config_report == 'archs': + archs = True + elif opts.rtems_config_report == 'files': + files = True + return self.report(profiles, builds, archs, files) + return None + + +def add_arguments(argsp, config_report_default = 'all'): + '''RTEMS Module arguments.''' + + import argparse + + argsp.add_argument('--rtems-config', + help = 'RTEMS configuration file (default: %(default)s).', + type = str, + default = default_configuration_file()) + argsp.add_argument('--rtems-config-report', + help = 'Report the RTEMS configuration (default: %(const)s).', + type = str, + default = argparse.SUPPRESS, + const = config_report_default, + nargs = '?', + choices = ['all', 'profiles', 'builds', 'archs', 'files']) + +if __name__ == '__main__': + print('no unittest, please add') diff --git a/rtemstoolkit/solaris.py b/rtemstoolkit/solaris.py index dc3e490..9fcbab8 100644 --- a/rtemstoolkit/solaris.py +++ b/rtemstoolkit/solaris.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2010-2017 Chris Johns (chr...@rtems.org) +# Copyright 2010-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -17,6 +17,10 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# Unittest: python -m rtemstoolkit.solaris +# + # # This code is based on what ever doco about spec files I could find and # RTEMS project's spec files. diff --git a/rtemstoolkit/stacktraces.py b/rtemstoolkit/stacktraces.py index e589618..9a16362 100644 --- a/rtemstoolkit/stacktraces.py +++ b/rtemstoolkit/stacktraces.py @@ -28,16 +28,43 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.stacktraces +# + import sys +import threading import traceback -def trace(): +_lock = threading.Lock() + +def _thread_name(thread_id): + thread_name = '' + for t in threading.enumerate(): + if t.ident == thread_id: + thread_name = t.name + return '%s (%s)' % (thread_name, thread_id) + +def stacks(): code = [] - for threadId, stack in sys._current_frames().items(): - code.append("\n# thread-id: %s" % threadId) + for thread_id, stack in sys._current_frames().items(): + thread_label = _thread_name(thread_id) + code.append("] %s" % (thread_label)) for filename, lineno, name, line in traceback.extract_stack(stack): - code.append('file: "%s", line %d, in %s' % (filename, lineno, name)) - if line: - code.append(" %s" % (line.strip())) - return '\n'.join(code) + code.append(' %-30s : %s:%d' % (name, filename, lineno)) + if line: + code.append(' %s' % (line.strip())) + return code + +def trace(what, who, show_stacks = False): + _lock.acquire() + if show_stacks: + print('-' * 80) + if show_stacks: + for i in stacks(): + print(' ' + i) + _lock.release() +if __name__ == '__main__': + trace('test', 'tester', show_stacks = True) + print(' -- END --') diff --git a/rtemstoolkit/textbox.py b/rtemstoolkit/textbox.py index 8341bdd..6936acb 100644 --- a/rtemstoolkit/textbox.py +++ b/rtemstoolkit/textbox.py @@ -1,6 +1,6 @@ # # RTEMS Tools Project (http://www.rtems.org/) -# Copyright 2017 Chris Johns (chr...@rtems.org) +# Copyright 2017-2018 Chris Johns (chr...@rtems.org) # All rights reserved. # # This file is part of the RTEMS Tools package in 'rtems-tools'. @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.textbox +# + # # Manage paths locally. The internally the path is in Unix or shell format and # we convert to the native format when performing operations at the Python diff --git a/rtemstoolkit/version.py b/rtemstoolkit/version.py index d538d2a..d316804 100644 --- a/rtemstoolkit/version.py +++ b/rtemstoolkit/version.py @@ -28,6 +28,10 @@ # POSSIBILITY OF SUCH DAMAGE. # +# +# Unittest: python -m rtemstoolkit.version +# + # # Releasing RTEMS Tools # --------------------- @@ -88,19 +92,9 @@ try: except ImportError: import ConfigParser as configparser -# -# Support to handle importing when installed in a package and as a unit test. -# If there is a better way to let us know. -# -try: - from . import error - from . import git - from . import rtems -except (ValueError, SystemError): - import error - import git - import path - import rtems +from rtemstoolkit import error +from rtemstoolkit import git +from rtemstoolkit import rtems # # Default to an internal string. @@ -254,4 +248,8 @@ def revision(): return _revision if __name__ == '__main__': - print('Version: %s' % (str())) + print('Version : %s' % (version())) + print('Revision : %s' % (revision())) + print('Released : %s' % (released())) + print('Version control: %s' % (version_control())) + print('Version string : %s' % (string())) diff --git a/tester/rt/check.py b/tester/rt/check.py index f2addbd..39ee90d 100755 --- a/tester/rt/check.py +++ b/tester/rt/check.py @@ -1118,97 +1118,106 @@ class builder: log.notice('Profile(s): %s' % (', '.join(profiles))) self.run_jobs(self.profile_jobs(profiles)) + +class settings(object): + def __init__(self): + self.opts = None + self.mail = None + self.args = argparse.ArgumentParser() + self.args.add_argument('--prefix', + help = 'Prefix to build the BSP.', + default = rtems.default_prefix(), + type = str) + self.args.add_argument('--rtems-tools', + help = 'The RTEMS tools directory.', + default = rtems.default_prefix(), + type = str) + self.args.add_argument('--rtems', + help = 'The RTEMS source tree.', + type = str) + self.args.add_argument('--build-path', + help = 'Path to build in.', + default = 'bsp-builds', + type = str) + self.args.add_argument('--log', + help = 'Log file.', + default = 'bsp-builer-%s.txt' % (_now().strftime('%Y%m%d-%H%M%S')), + type = str) + self.args.add_argument('--warnings-report', + help = 'Report the warnings to a file.', + type = str, + default = None) + self.args.add_argument('--failures-report', + help = 'Report the failures to a file.', + type = str, + default = None) + self.args.add_argument('--stop-on-error', + help = 'Stop on an error.', + action = 'store_true') + self.args.add_argument('--no-clean', + help = 'Do not clean the build output.', + action = 'store_true') + self.args.add_argument('--profiles', + help = 'Build the listed profiles (profile,profile,..).', + type = str, + default = 'tier-1') + self.args.add_argument('--arch', + help = 'Build the architectures (arch,arch,..).', + type = str) + self.args.add_argument('--bsp', + help = 'Build the BSPs (arch/bsp,arch/bsp,..).', + type = str) + self.args.add_argument('--build', + help = 'Build name to build (see --rtems-config-report).', + type = str, + default = 'all') + self.args.add_argument('--jobs', + help = 'Number of jobs to run.', + type = str, + default = '1/%d' % (host.cpus())) + self.args.add_argument('--dry-run', + help = 'Do not run the actual builds.', + action = 'store_true') + rtems.add_arguments(self.args, config_report_default = 'profiles') + mailer.add_arguments(self.args) + + def parse(self, args): + self.opts = self.args.parse_args(args) + if self.opts.mail: + self.mail = mailer.mail(self.opts) + # Request these now to generate any errors. + self.mail_to_addr = self.mail.to_address() + self.mail_from_addr = self.mail.from_address() + self.mali_smtp_host = self.mail.smtp_host() + def run_args(args): b = None ec = 0 try: rtems.clean_windows_path() - start = _now() - prefix = '/opt/rtems/%s' % (rtems_version()) - tools = prefix - build_dir = 'bsp-builds' - logf = 'bsp-build-%s.txt' % (_now().strftime('%Y%m%d-%H%M%S')) - config_file = rtems.bsp_configuration_file() - - argsp = argparse.ArgumentParser() - argsp.add_argument('--prefix', help = 'Prefix to build the BSP.', - type = str) - argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.', - type = str) - argsp.add_argument('--rtems', help = 'The RTEMS source tree.', - type = str) - argsp.add_argument('--build-path', help = 'Path to build in.', - type = str) - argsp.add_argument('--log', help = 'Log file.', type = str) - argsp.add_argument('--config-report', help = 'Report the configuration.', - type = str, default = None, - choices = ['all', 'profiles', 'builds', 'archs']) - argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.', - type = str, default = None) - argsp.add_argument('--failures-report', help = 'Report the failures to a file.', - type = str, default = None) - argsp.add_argument('--stop-on-error', help = 'Stop on an error.', - action = 'store_true') - argsp.add_argument('--no-clean', help = 'Do not clean the build output.', - action = 'store_true') - argsp.add_argument('--profiles', help = 'Build the listed profiles (profile,profile,..).', - type = str, default = 'tier-1') - argsp.add_argument('--arch', help = 'Build the architectures (arch,arch,..).', - type = str) - argsp.add_argument('--bsp', help = 'Build the BSPs (arch/bsp,arch/bsp,..).', - type = str) - argsp.add_argument('--build', help = 'Build name to build (see --config-report).', - type = str, default='all') - argsp.add_argument('--jobs', help = 'Number of jobs to run.', - type = str, default = '1/%d' % (host.cpus())) - argsp.add_argument('--dry-run', help = 'Do not run the actual builds.', - action = 'store_true') - mailer.add_arguments(argsp) - - opts = argsp.parse_args(args[1:]) - mail = None - if opts.mail: - mail = mailer.mail(opts) - # Request these now to generate any errors. - from_addr = mail.from_address() - smtp_host = mail.smtp_host() - if 'mail_to' in opts and opts.mail_to is not None: - to_addr = opts.mail_to - else: - to_addr = 'bu...@rtems.org' - if opts.log is not None: - logf = opts.log - log.default = log.log([logf]) - log.notice(title()) - log.output(command_line()) - if mail: - log.notice('Mail: from:%s to:%s smtp:%s' % (from_addr, - to_addr, - smtp_host)) + settings_ = settings() + settings_.parse(args[1:]) config = rtems.configuration() - config.load(config_file, opts.build) - - if opts.config_report: - log.notice('Configuration Report: %s' % (opts.config_report)) - c_profiles = False - c_builds = False - c_archs = False - if opts.config_report == 'all': - c_profiles = True - c_builds = True - c_archs = True - elif opts.config_report == 'profiles': - c_profiles = True - elif opts.config_report == 'builds': - c_builds = True - elif opts.config_report == 'archs': - c_archs = True - log.notice(config.report(c_profiles, c_builds, c_archs)) + config.load(settings_.opts.rtems_config, settings_.opts.build) + + config_report = config.report_args(settings_.opts) + if config_report: + report_type = settings_.opts.rtems_config_report + log.notice('RTEMS Configuration Report: %s' % (report_type)) + log.notice(config_report) sys.exit(0) - if opts.rtems is None: + log.default = log.log([settings_.opts.log]) + log.notice(title()) + log.output(command_line()) + + if settings_.mail: + log.notice('Mail: from:%s to:%s smtp:%s' % (settings_.mail_from_addr, + settings_.mail_to_addr, + settings_.mail_smtp_host)) + if settings_.opts.rtems is None: raise error.general('No RTEMS source provided on the command line') if opts.prefix is not None: prefix = path.shell(opts.prefix) @@ -1224,12 +1233,17 @@ def run_args(args): 'warnings-report' : opts.warnings_report, 'failures-report' : opts.failures_report } - b = builder(config, rtems_version(), prefix, tools, - path.shell(opts.rtems), build_dir, options) + b = builder(config, + rtems_version(), + settings_.opts.prefix, + settings_.opts.tools, + path.shell(settings_.opts.rtems), + settings_.opts.build_dir, + options) - profiles = comma_split(opts.profiles) - archs = comma_split(opts.arch) - bsps = comma_split(opts.bsp) + profiles = comma_split(settings_.opts.profiles) + archs = comma_split(settings_.opts.arch) + bsps = comma_split(settings_.opts.bsp) # # The default is build a profile. @@ -1251,7 +1265,7 @@ def run_args(args): # # Email the results of the build. # - if mail is not None: + if settings.mail is not None: subject = '[rtems-bsp-builder] %s: %s' % (str(start).split('.')[0], what) t = title() @@ -1278,7 +1292,7 @@ def run_args(args): body += 'Warnings Report' + os.linesep body += '===============' + os.linesep body += b.results.warnings_report(summary = True) - mail.send(to_addr, subject, body) + settings_.mail.send(settings_.mali_to_addr, subject, body) except error.general as gerr: print(gerr) diff --git a/tester/rt/run.py b/tester/rt/run.py index 1f4fa3a..aa192e0 100644 --- a/tester/rt/run.py +++ b/tester/rt/run.py @@ -164,9 +164,7 @@ def run(command_path = None): except KeyboardInterrupt: if opts is not None and opts.find_arg('--stacktrace'): print('}} dumping:', threading.active_count()) - for t in threading.enumerate(): - print('}} ', t.name) - print(stacktraces.trace()) + print(stacktraces.trace('dump on exit', 'run', show_stacks = True)) log.notice('abort: user terminated') sys.exit(1) finally: diff --git a/tester/rt/test.py b/tester/rt/test.py index c5d61d8..c525396 100644 --- a/tester/rt/test.py +++ b/tester/rt/test.py @@ -394,9 +394,7 @@ def run(command_path = None): except KeyboardInterrupt: if opts is not None and opts.find_arg('--stacktrace'): print('}} dumping:', threading.active_count()) - for t in threading.enumerate(): - print('}} ', t.name) - print(stacktraces.trace()) + print(stacktraces.trace('dump on exit', 'run', show_stacks = True)) log.notice('abort: user terminated') killall(tests) sys.exit(1) -- 2.15.1 _______________________________________________ devel mailing list devel@rtems.org http://lists.rtems.org/mailman/listinfo/devel