Chris.. if you post a V2 addressing those, I will commit it. On Wed, Apr 3, 2024, 11:52 AM Kinsey Moore <kinsey.mo...@oarcorp.com> wrote:
> Looks fine overall. Minor nits: > "host" is set to "freebsd" and is never used. > Numeric indexes for repo config details are nice for brevity, but not for > readability. > > Kinsey > > On Wed, Apr 3, 2024 at 11:29 AM Joel Sherrill <joel.sherr...@gmail.com> > wrote: > >> This looks handy. I don't have an issue with it. >> >> On Tue, Apr 2, 2024 at 11:25 PM <chr...@rtems.org> wrote: >> >>> From: Chris Johns <chr...@rtems.org> >>> >>> --- >>> source-builder/sb-rtems-pkg | 29 ++++ >>> source-builder/sb/download.py | 5 +- >>> source-builder/sb/git.py | 12 ++ >>> source-builder/sb/rtemspkg.py | 279 ++++++++++++++++++++++++++++++++++ >>> 4 files changed, 324 insertions(+), 1 deletion(-) >>> create mode 100755 source-builder/sb-rtems-pkg >>> create mode 100644 source-builder/sb/rtemspkg.py >>> >>> diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg >>> new file mode 100755 >>> index 0000000..99ed26c >>> --- /dev/null >>> +++ b/source-builder/sb-rtems-pkg >>> @@ -0,0 +1,29 @@ >>> +#! /usr/bin/env python >>> +# >>> +# RTEMS Tools Project (http://www.rtems.org/) >>> +# Copyright 2024 Chris Johns (chr...@rtems.org) >>> +# All rights reserved. >>> +# >>> +# This file is part of the RTEMS Tools package in 'rtems-tools'. >>> +# >>> +# Permission to use, copy, modify, and/or distribute this software for >>> any >>> +# purpose with or without fee is hereby granted, provided that the above >>> +# copyright notice and this permission notice appear in all copies. >>> +# >>> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL >>> WARRANTIES >>> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF >>> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE >>> FOR >>> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES >>> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN >>> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT >>> OF >>> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. >>> + >>> +from __future__ import print_function >>> + >>> +try: >>> + import sb.rtemspkg >>> + sb.rtemspkg.run() >>> +except ImportError: >>> + import sys >>> + print("Incorrect Source Builder installation", file = sys.stderr) >>> + sys.exit(1) >>> diff --git a/source-builder/sb/download.py >>> b/source-builder/sb/download.py >>> index 6e6f9f2..0201675 100644 >>> --- a/source-builder/sb/download.py >>> +++ b/source-builder/sb/download.py >>> @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True): >>> hash = hash.split() >>> if len(hash) != 2: >>> raise error.internal('invalid hash format: %s' % (file_)) >>> + if hash[0] == 'NO-HASH': >>> + return not failed >>> try: >>> hashlib_algorithms = hashlib.algorithms >>> except: >>> @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts): >>> else: >>> repo.clean(['-f', '-d']) >>> repo.reset('--hard') >>> - repo.checkout('master') >>> + default_branch = repo.default_branch() >>> + repo.checkout(default_branch) >>> for a in us[1:]: >>> _as = a.split('=') >>> if _as[0] == 'branch' or _as[0] == 'checkout': >>> diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py >>> index 237e690..0aa4da9 100644 >>> --- a/source-builder/sb/git.py >>> +++ b/source-builder/sb/git.py >>> @@ -226,6 +226,18 @@ class repo: >>> hash = l1[len('commit '):] >>> return hash >>> >>> + def default_branch(self): >>> + ec, output = self._run(['remote', 'show']) >>> + if ec == 0: >>> + origin = output.split('\n')[0] >>> + ec, output = self._run(['remote', 'show', origin]) >>> + if ec == 0: >>> + for l in output.split('\n'): >>> + l = l.strip() >>> + if l.startswith('HEAD branch: '): >>> + return l[len('HEAD branch: '):] >>> + return None >>> + >>> if __name__ == '__main__': >>> import os.path >>> import sys >>> diff --git a/source-builder/sb/rtemspkg.py >>> b/source-builder/sb/rtemspkg.py >>> new file mode 100644 >>> index 0000000..0b43d23 >>> --- /dev/null >>> +++ b/source-builder/sb/rtemspkg.py >>> @@ -0,0 +1,279 @@ >>> +# >>> +# RTEMS Tools Project (http://www.rtems.org/) >>> +# Copyright 2024 Chris Johns (chr...@rtems.org) >>> +# All rights reserved. >>> +# >>> +# This file is part of the RTEMS Tools package in 'rtems-tools'. >>> +# >>> +# Permission to use, copy, modify, and/or distribute this software for >>> any >>> +# purpose with or without fee is hereby granted, provided that the above >>> +# copyright notice and this permission notice appear in all copies. >>> +# >>> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL >>> WARRANTIES >>> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF >>> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE >>> FOR >>> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES >>> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN >>> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT >>> OF >>> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. >>> + >>> +# >>> +# This code builds a package compiler tool suite given a tool set. A >>> tool >>> +# set lists the various tools. These are specific tool configurations. >>> +# >>> + >>> +from __future__ import print_function >>> + >>> +import argparse >>> +import base64 >>> +import copy >>> +import datetime >>> +import hashlib >>> +import os >>> +import sys >>> + >>> +try: >>> + from . import build >>> + from . import download >>> + from . import error >>> + from . import git >>> + from . import log >>> + from . import path >>> + from . import simhost >>> + from . import version >>> +except KeyboardInterrupt: >>> + print('abort: user terminated', file=sys.stderr) >>> + sys.exit(1) >>> +except: >>> + raise >>> + >>> +# >>> +# RTEMS Packages we maintian a git hash of in the RSB >>> +# >>> +rtems_pkg_cfgs = [ >>> + [ >>> + 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg', >>> + 'rtems_tools_version', 'git://git.rtems.org/rtems-tools', >>> + 'rtems-tools.git', 'master', >>> + ' >>> https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2 >>> ', >>> + 'rtems-tools-%{rtems_tools_version}.tar.bz2' >>> + ], >>> + [ >>> + 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg', >>> + 'rtems_kernel_version', 'git://git.rtems.org/rtems', >>> 'rtems.git', >>> + 'master', >>> + ' >>> https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2 >>> ', >>> + 'rtems-kernel-%{rtems_kernel_version}.tar.bz2' >>> + ], >>> + [ >>> + 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg', >>> + 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd', >>> + 'rtems-libbsd.git', '6-freebsd-12', >>> + ' >>> https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext} >>> ', >>> + 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}' >>> + ], >>> + [ >>> + 'RTEMS Net Legacy', >>> 'tools/rtems-net-legacy-%{rtems_version}.cfg', >>> + 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy', >>> + 'rtems-net-legacy.git', 'main', >>> + ' >>> https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext} >>> ', >>> + 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}' >>> + ], >>> + [ >>> + 'RTEMS Net Services', 'net/net-services-1.cfg', >>> + 'rtems_net_services_version', 'git:// >>> git.rtems.org/rtems-net-services', >>> + 'rtems-net-services.git', 'master', >>> + ' >>> https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext} >>> ', >>> + >>> 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}' >>> + ], >>> +] >>> + >>> + >>> +def clean_line(line): >>> + line = line[0:-1] >>> + b = line.find('#') >>> + if b >= 0: >>> + line = line[1:b] + ('\\' if line[-1] == '\\' else '') >>> + return line.strip() >>> + >>> + >>> +def clean_and_pack(line, last_line): >>> + leading_ws = ' ' if len(line) > 0 and line[0].isspace() else '' >>> + line = clean_line(line) >>> + if len(last_line) > 0: >>> + line = last_line + leading_ws + line >>> + return line >>> + >>> + >>> +def config_patch(configdir, config, version_label, config_hash, >>> repo_hash, >>> + checksum): >>> + for cd in configdir.split(':'): >>> + cf = path.join(cd, config) >>> + if path.exists(cf): >>> + try: >>> + with open(cf) as f: >>> + lines = f.readlines() >>> + except IOError as err: >>> + raise error.general('config: %s: read error: %s' % >>> + (config, str(err))) >>> + new_config = [] >>> + new_lines = [] >>> + last_line = '' >>> + for line in lines: >>> + new_lines += [line] >>> + line = clean_and_pack(line, last_line) >>> + if len(line) > 0: >>> + if line[-1] == '\\': >>> + last_line = line[:-1] >>> + continue >>> + last_line = '' >>> + if version_label in line and not 'rsb_version' in >>> line: >>> + if line.startswith('%define ' + version_label): >>> + new_lines = [ >>> + '%define ' + version_label + ' ' + >>> repo_hash + >>> + os.linesep >>> + ] >>> + elif line.startswith('%hash '): >>> + ls = line.split() >>> + if len(ls) != 4: >>> + raise error.general('invalid %hash: ' + >>> line) >>> + new_lines = [ >>> + ' '.join(ls[0:3]) + ' \\' + os.linesep, >>> + ' ' + checksum + os.linesep >>> + ] >>> + new_config += new_lines >>> + new_lines = [] >>> + try: >>> + with open(cf, 'w') as f: >>> + f.writelines(new_config) >>> + except IOError as err: >>> + raise error.general('config: %s: write error: %s' % >>> + (config, str(err))) >>> + return >>> + raise error.general('could not find: ' + config) >>> + >>> + >>> +def checksum_shar512_base64(tarball): >>> + hasher = hashlib.new('sha512') >>> + try: >>> + with open(path.host(tarball), 'rb') as f: >>> + hasher.update(f.read()) >>> + except IOError as err: >>> + log.notice('hash: %s: read error: %s' % (file_, str(err))) >>> + except: >>> + raise >>> + raise error.general('cannot hash the tar file') >>> + hash_hex = hasher.hexdigest() >>> + hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8') >>> + return hash_base64 >>> + >>> + >>> +def run(args=sys.argv): >>> + ec = 0 >>> + output = [] >>> + try: >>> + # >>> + # The RSB options support cannot be used because it loads the >>> defaults >>> + # for the host which we cannot do here. >>> + # >>> + description = 'RTEMS Track Dependencies a build set has for all >>> hosts.' >>> + >>> + argsp = argparse.ArgumentParser(prog='sb-rtems-pkg', >>> + description=description) >>> + argsp.add_argument('--rtems-version', >>> + help='Set the RTEMS version.', >>> + type=str, >>> + default=version.version()) >>> + argsp.add_argument('--output', >>> + help='Output file.', >>> + type=str, >>> + default=None) >>> + argsp.add_argument('--log', >>> + help='Log file.', >>> + type=str, >>> + default=simhost.log_default('rtems-pkg')) >>> + argsp.add_argument('--trace', >>> + help='Enable trace logging for debugging.', >>> + action='store_true') >>> + argsp.add_argument('--dry-run', >>> + help='Dry run, do not update the >>> configurations', >>> + action='store_true') >>> + argsp.add_argument('bsets', nargs='*', help='Build sets.') >>> + >>> + argopts = argsp.parse_args(args[1:]) >>> + >>> + simhost.load_log(argopts.log) >>> + log.notice('RTEMS Source Builder - RTEMS Package Update, %s' % >>> + (version.string())) >>> + log.tracing = argopts.trace >>> + >>> + opts = simhost.load_options(args, argopts, >>> extras=['--with-download']) >>> + opts.defaults['_rsb_getting_source'] = '1' >>> + opts.defaults[ >>> + 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}' >>> + host = 'freebsd' >>> + >>> + for cfg in rtems_pkg_cfgs: >>> + b = None >>> + try: >>> + bopts = copy.copy(opts) >>> + bmacros = copy.copy(opts.defaults) >>> + b = build.build(cfg[1], False, bopts, bmacros) >>> + git_hash_key = b.macros.find(cfg[2]) >>> + if len(git_hash_key) == 0: >>> + raise error.general(cfg[0] + ': cannot find version >>> macro') >>> + source_dir = b.macros.expand('%{_sourcedir}') >>> + config_hash = b.macros.expand('%{' + cfg[2] + '}') >>> + repo_path = path.join(source_dir, cfg[4]) >>> + download.get_file(cfg[3] + '?fetch?checkout=' + cfg[5], >>> + repo_path, bopts, b) >>> + repo = git.repo(repo_path) >>> + repo_hash = repo.head() >>> + if config_hash != repo_hash: >>> + update = True >>> + update_str = 'UPDATE' >>> + else: >>> + update = False >>> + update_str = 'matching' >>> + print(cfg[0] + ': ' + update_str + ' config:' + >>> config_hash + >>> + ' repo:' + repo_hash) >>> + b.macros[cfg[2]] = repo_hash >>> + tarball = b.macros.expand(cfg[7]) >>> + b.macros.set_write_map('hashes') >>> + b.macros[tarball] = 'NO-HASH NO-HASH' >>> + b.macros.unset_write_map() >>> + tarball_path = path.join(source_dir, >>> b.macros.expand(cfg[7])) >>> + download.get_file(b.macros.expand(cfg[6]), >>> tarball_path, bopts, >>> + b) >>> + tarball_hash = checksum_shar512_base64(tarball_path) >>> + if update: >>> + config_patch(b.macros.expand('%{_configdir}'), >>> + b.macros.expand(cfg[1]), cfg[2], >>> config_hash, >>> + repo_hash, tarball_hash) >>> + del b >>> + except error.general as gerr: >>> + log.stderr(str(gerr)) >>> + log.stderr('Build FAILED') >>> + b = None >>> + except error.general as gerr: >>> + log.stderr(str(gerr)) >>> + log.stderr('Build FAILED') >>> + ec = 1 >>> + except error.internal as ierr: >>> + log.stderr(str(ierr)) >>> + log.stderr('Internal Build FAILED') >>> + ec = 1 >>> + except error.exit as eerr: >>> + pass >>> + except KeyboardInterrupt: >>> + log.notice('abort: user terminated') >>> + ec = 1 >>> + except: >>> + raise >>> + log.notice('abort: unknown error') >>> + ec = 1 >>> + sys.exit(ec) >>> + >>> + >>> +if __name__ == "__main__": >>> + run() >>> -- >>> 2.42.0 >>> >>> _______________________________________________ >>> devel mailing list >>> devel@rtems.org >>> http://lists.rtems.org/mailman/listinfo/devel >>> >> _______________________________________________ >> devel mailing list >> devel@rtems.org >> http://lists.rtems.org/mailman/listinfo/devel > >
_______________________________________________ devel mailing list devel@rtems.org http://lists.rtems.org/mailman/listinfo/devel
