This looks handy. I don't have an issue with it. On Tue, Apr 2, 2024 at 11:25 PM <chr...@rtems.org> wrote:
> From: Chris Johns <chr...@rtems.org> > > --- > source-builder/sb-rtems-pkg | 29 ++++ > source-builder/sb/download.py | 5 +- > source-builder/sb/git.py | 12 ++ > source-builder/sb/rtemspkg.py | 279 ++++++++++++++++++++++++++++++++++ > 4 files changed, 324 insertions(+), 1 deletion(-) > create mode 100755 source-builder/sb-rtems-pkg > create mode 100644 source-builder/sb/rtemspkg.py > > diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg > new file mode 100755 > index 0000000..99ed26c > --- /dev/null > +++ b/source-builder/sb-rtems-pkg > @@ -0,0 +1,29 @@ > +#! /usr/bin/env python > +# > +# RTEMS Tools Project (http://www.rtems.org/) > +# Copyright 2024 Chris Johns (chr...@rtems.org) > +# All rights reserved. > +# > +# This file is part of the RTEMS Tools package in 'rtems-tools'. > +# > +# Permission to use, copy, modify, and/or distribute this software for any > +# purpose with or without fee is hereby granted, provided that the above > +# copyright notice and this permission notice appear in all copies. > +# > +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES > +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF > +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR > +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES > +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN > +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF > +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. > + > +from __future__ import print_function > + > +try: > + import sb.rtemspkg > + sb.rtemspkg.run() > +except ImportError: > + import sys > + print("Incorrect Source Builder installation", file = sys.stderr) > + sys.exit(1) > diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py > index 6e6f9f2..0201675 100644 > --- a/source-builder/sb/download.py > +++ b/source-builder/sb/download.py > @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True): > hash = hash.split() > if len(hash) != 2: > raise error.internal('invalid hash format: %s' % (file_)) > + if hash[0] == 'NO-HASH': > + return not failed > try: > hashlib_algorithms = hashlib.algorithms > except: > @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts): > else: > repo.clean(['-f', '-d']) > repo.reset('--hard') > - repo.checkout('master') > + default_branch = repo.default_branch() > + repo.checkout(default_branch) > for a in us[1:]: > _as = a.split('=') > if _as[0] == 'branch' or _as[0] == 'checkout': > diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py > index 237e690..0aa4da9 100644 > --- a/source-builder/sb/git.py > +++ b/source-builder/sb/git.py > @@ -226,6 +226,18 @@ class repo: > hash = l1[len('commit '):] > return hash > > + def default_branch(self): > + ec, output = self._run(['remote', 'show']) > + if ec == 0: > + origin = output.split('\n')[0] > + ec, output = self._run(['remote', 'show', origin]) > + if ec == 0: > + for l in output.split('\n'): > + l = l.strip() > + if l.startswith('HEAD branch: '): > + return l[len('HEAD branch: '):] > + return None > + > if __name__ == '__main__': > import os.path > import sys > diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py > new file mode 100644 > index 0000000..0b43d23 > --- /dev/null > +++ b/source-builder/sb/rtemspkg.py > @@ -0,0 +1,279 @@ > +# > +# RTEMS Tools Project (http://www.rtems.org/) > +# Copyright 2024 Chris Johns (chr...@rtems.org) > +# All rights reserved. > +# > +# This file is part of the RTEMS Tools package in 'rtems-tools'. > +# > +# Permission to use, copy, modify, and/or distribute this software for any > +# purpose with or without fee is hereby granted, provided that the above > +# copyright notice and this permission notice appear in all copies. > +# > +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES > +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF > +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR > +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES > +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN > +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF > +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. > + > +# > +# This code builds a package compiler tool suite given a tool set. A tool > +# set lists the various tools. These are specific tool configurations. > +# > + > +from __future__ import print_function > + > +import argparse > +import base64 > +import copy > +import datetime > +import hashlib > +import os > +import sys > + > +try: > + from . import build > + from . import download > + from . import error > + from . import git > + from . import log > + from . import path > + from . import simhost > + from . import version > +except KeyboardInterrupt: > + print('abort: user terminated', file=sys.stderr) > + sys.exit(1) > +except: > + raise > + > +# > +# RTEMS Packages we maintian a git hash of in the RSB > +# > +rtems_pkg_cfgs = [ > + [ > + 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg', > + 'rtems_tools_version', 'git://git.rtems.org/rtems-tools', > + 'rtems-tools.git', 'master', > + ' > https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2 > ', > + 'rtems-tools-%{rtems_tools_version}.tar.bz2' > + ], > + [ > + 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg', > + 'rtems_kernel_version', 'git://git.rtems.org/rtems', 'rtems.git', > + 'master', > + ' > https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2 > ', > + 'rtems-kernel-%{rtems_kernel_version}.tar.bz2' > + ], > + [ > + 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg', > + 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd', > + 'rtems-libbsd.git', '6-freebsd-12', > + ' > https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext} > ', > + 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}' > + ], > + [ > + 'RTEMS Net Legacy', 'tools/rtems-net-legacy-%{rtems_version}.cfg', > + 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy', > + 'rtems-net-legacy.git', 'main', > + ' > https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext} > ', > + 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}' > + ], > + [ > + 'RTEMS Net Services', 'net/net-services-1.cfg', > + 'rtems_net_services_version', 'git:// > git.rtems.org/rtems-net-services', > + 'rtems-net-services.git', 'master', > + ' > https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext} > ', > + > 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}' > + ], > +] > + > + > +def clean_line(line): > + line = line[0:-1] > + b = line.find('#') > + if b >= 0: > + line = line[1:b] + ('\\' if line[-1] == '\\' else '') > + return line.strip() > + > + > +def clean_and_pack(line, last_line): > + leading_ws = ' ' if len(line) > 0 and line[0].isspace() else '' > + line = clean_line(line) > + if len(last_line) > 0: > + line = last_line + leading_ws + line > + return line > + > + > +def config_patch(configdir, config, version_label, config_hash, repo_hash, > + checksum): > + for cd in configdir.split(':'): > + cf = path.join(cd, config) > + if path.exists(cf): > + try: > + with open(cf) as f: > + lines = f.readlines() > + except IOError as err: > + raise error.general('config: %s: read error: %s' % > + (config, str(err))) > + new_config = [] > + new_lines = [] > + last_line = '' > + for line in lines: > + new_lines += [line] > + line = clean_and_pack(line, last_line) > + if len(line) > 0: > + if line[-1] == '\\': > + last_line = line[:-1] > + continue > + last_line = '' > + if version_label in line and not 'rsb_version' in > line: > + if line.startswith('%define ' + version_label): > + new_lines = [ > + '%define ' + version_label + ' ' + > repo_hash + > + os.linesep > + ] > + elif line.startswith('%hash '): > + ls = line.split() > + if len(ls) != 4: > + raise error.general('invalid %hash: ' + > line) > + new_lines = [ > + ' '.join(ls[0:3]) + ' \\' + os.linesep, > + ' ' + checksum + os.linesep > + ] > + new_config += new_lines > + new_lines = [] > + try: > + with open(cf, 'w') as f: > + f.writelines(new_config) > + except IOError as err: > + raise error.general('config: %s: write error: %s' % > + (config, str(err))) > + return > + raise error.general('could not find: ' + config) > + > + > +def checksum_shar512_base64(tarball): > + hasher = hashlib.new('sha512') > + try: > + with open(path.host(tarball), 'rb') as f: > + hasher.update(f.read()) > + except IOError as err: > + log.notice('hash: %s: read error: %s' % (file_, str(err))) > + except: > + raise > + raise error.general('cannot hash the tar file') > + hash_hex = hasher.hexdigest() > + hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8') > + return hash_base64 > + > + > +def run(args=sys.argv): > + ec = 0 > + output = [] > + try: > + # > + # The RSB options support cannot be used because it loads the > defaults > + # for the host which we cannot do here. > + # > + description = 'RTEMS Track Dependencies a build set has for all > hosts.' > + > + argsp = argparse.ArgumentParser(prog='sb-rtems-pkg', > + description=description) > + argsp.add_argument('--rtems-version', > + help='Set the RTEMS version.', > + type=str, > + default=version.version()) > + argsp.add_argument('--output', > + help='Output file.', > + type=str, > + default=None) > + argsp.add_argument('--log', > + help='Log file.', > + type=str, > + default=simhost.log_default('rtems-pkg')) > + argsp.add_argument('--trace', > + help='Enable trace logging for debugging.', > + action='store_true') > + argsp.add_argument('--dry-run', > + help='Dry run, do not update the > configurations', > + action='store_true') > + argsp.add_argument('bsets', nargs='*', help='Build sets.') > + > + argopts = argsp.parse_args(args[1:]) > + > + simhost.load_log(argopts.log) > + log.notice('RTEMS Source Builder - RTEMS Package Update, %s' % > + (version.string())) > + log.tracing = argopts.trace > + > + opts = simhost.load_options(args, argopts, > extras=['--with-download']) > + opts.defaults['_rsb_getting_source'] = '1' > + opts.defaults[ > + 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}' > + host = 'freebsd' > + > + for cfg in rtems_pkg_cfgs: > + b = None > + try: > + bopts = copy.copy(opts) > + bmacros = copy.copy(opts.defaults) > + b = build.build(cfg[1], False, bopts, bmacros) > + git_hash_key = b.macros.find(cfg[2]) > + if len(git_hash_key) == 0: > + raise error.general(cfg[0] + ': cannot find version > macro') > + source_dir = b.macros.expand('%{_sourcedir}') > + config_hash = b.macros.expand('%{' + cfg[2] + '}') > + repo_path = path.join(source_dir, cfg[4]) > + download.get_file(cfg[3] + '?fetch?checkout=' + cfg[5], > + repo_path, bopts, b) > + repo = git.repo(repo_path) > + repo_hash = repo.head() > + if config_hash != repo_hash: > + update = True > + update_str = 'UPDATE' > + else: > + update = False > + update_str = 'matching' > + print(cfg[0] + ': ' + update_str + ' config:' + > config_hash + > + ' repo:' + repo_hash) > + b.macros[cfg[2]] = repo_hash > + tarball = b.macros.expand(cfg[7]) > + b.macros.set_write_map('hashes') > + b.macros[tarball] = 'NO-HASH NO-HASH' > + b.macros.unset_write_map() > + tarball_path = path.join(source_dir, > b.macros.expand(cfg[7])) > + download.get_file(b.macros.expand(cfg[6]), tarball_path, > bopts, > + b) > + tarball_hash = checksum_shar512_base64(tarball_path) > + if update: > + config_patch(b.macros.expand('%{_configdir}'), > + b.macros.expand(cfg[1]), cfg[2], > config_hash, > + repo_hash, tarball_hash) > + del b > + except error.general as gerr: > + log.stderr(str(gerr)) > + log.stderr('Build FAILED') > + b = None > + except error.general as gerr: > + log.stderr(str(gerr)) > + log.stderr('Build FAILED') > + ec = 1 > + except error.internal as ierr: > + log.stderr(str(ierr)) > + log.stderr('Internal Build FAILED') > + ec = 1 > + except error.exit as eerr: > + pass > + except KeyboardInterrupt: > + log.notice('abort: user terminated') > + ec = 1 > + except: > + raise > + log.notice('abort: unknown error') > + ec = 1 > + sys.exit(ec) > + > + > +if __name__ == "__main__": > + run() > -- > 2.42.0 > > _______________________________________________ > devel mailing list > devel@rtems.org > http://lists.rtems.org/mailman/listinfo/devel >
_______________________________________________ devel mailing list devel@rtems.org http://lists.rtems.org/mailman/listinfo/devel
