Looks fine overall. Minor nits: "host" is set to "freebsd" and is never used. Numeric indexes for repo config details are nice for brevity, but not for readability.
Kinsey On Wed, Apr 3, 2024 at 11:29 AM Joel Sherrill <joel.sherr...@gmail.com> wrote: > This looks handy. I don't have an issue with it. > > On Tue, Apr 2, 2024 at 11:25 PM <chr...@rtems.org> wrote: > >> From: Chris Johns <chr...@rtems.org> >> >> --- >> source-builder/sb-rtems-pkg | 29 ++++ >> source-builder/sb/download.py | 5 +- >> source-builder/sb/git.py | 12 ++ >> source-builder/sb/rtemspkg.py | 279 ++++++++++++++++++++++++++++++++++ >> 4 files changed, 324 insertions(+), 1 deletion(-) >> create mode 100755 source-builder/sb-rtems-pkg >> create mode 100644 source-builder/sb/rtemspkg.py >> >> diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg >> new file mode 100755 >> index 0000000..99ed26c >> --- /dev/null >> +++ b/source-builder/sb-rtems-pkg >> @@ -0,0 +1,29 @@ >> +#! /usr/bin/env python >> +# >> +# RTEMS Tools Project (http://www.rtems.org/) >> +# Copyright 2024 Chris Johns (chr...@rtems.org) >> +# All rights reserved. >> +# >> +# This file is part of the RTEMS Tools package in 'rtems-tools'. >> +# >> +# Permission to use, copy, modify, and/or distribute this software for >> any >> +# purpose with or without fee is hereby granted, provided that the above >> +# copyright notice and this permission notice appear in all copies. >> +# >> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL >> WARRANTIES >> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF >> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR >> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES >> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN >> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF >> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. >> + >> +from __future__ import print_function >> + >> +try: >> + import sb.rtemspkg >> + sb.rtemspkg.run() >> +except ImportError: >> + import sys >> + print("Incorrect Source Builder installation", file = sys.stderr) >> + sys.exit(1) >> diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py >> index 6e6f9f2..0201675 100644 >> --- a/source-builder/sb/download.py >> +++ b/source-builder/sb/download.py >> @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True): >> hash = hash.split() >> if len(hash) != 2: >> raise error.internal('invalid hash format: %s' % (file_)) >> + if hash[0] == 'NO-HASH': >> + return not failed >> try: >> hashlib_algorithms = hashlib.algorithms >> except: >> @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts): >> else: >> repo.clean(['-f', '-d']) >> repo.reset('--hard') >> - repo.checkout('master') >> + default_branch = repo.default_branch() >> + repo.checkout(default_branch) >> for a in us[1:]: >> _as = a.split('=') >> if _as[0] == 'branch' or _as[0] == 'checkout': >> diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py >> index 237e690..0aa4da9 100644 >> --- a/source-builder/sb/git.py >> +++ b/source-builder/sb/git.py >> @@ -226,6 +226,18 @@ class repo: >> hash = l1[len('commit '):] >> return hash >> >> + def default_branch(self): >> + ec, output = self._run(['remote', 'show']) >> + if ec == 0: >> + origin = output.split('\n')[0] >> + ec, output = self._run(['remote', 'show', origin]) >> + if ec == 0: >> + for l in output.split('\n'): >> + l = l.strip() >> + if l.startswith('HEAD branch: '): >> + return l[len('HEAD branch: '):] >> + return None >> + >> if __name__ == '__main__': >> import os.path >> import sys >> diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py >> new file mode 100644 >> index 0000000..0b43d23 >> --- /dev/null >> +++ b/source-builder/sb/rtemspkg.py >> @@ -0,0 +1,279 @@ >> +# >> +# RTEMS Tools Project (http://www.rtems.org/) >> +# Copyright 2024 Chris Johns (chr...@rtems.org) >> +# All rights reserved. >> +# >> +# This file is part of the RTEMS Tools package in 'rtems-tools'. >> +# >> +# Permission to use, copy, modify, and/or distribute this software for >> any >> +# purpose with or without fee is hereby granted, provided that the above >> +# copyright notice and this permission notice appear in all copies. >> +# >> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL >> WARRANTIES >> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF >> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR >> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES >> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN >> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF >> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. >> + >> +# >> +# This code builds a package compiler tool suite given a tool set. A tool >> +# set lists the various tools. These are specific tool configurations. >> +# >> + >> +from __future__ import print_function >> + >> +import argparse >> +import base64 >> +import copy >> +import datetime >> +import hashlib >> +import os >> +import sys >> + >> +try: >> + from . import build >> + from . import download >> + from . import error >> + from . import git >> + from . import log >> + from . import path >> + from . import simhost >> + from . import version >> +except KeyboardInterrupt: >> + print('abort: user terminated', file=sys.stderr) >> + sys.exit(1) >> +except: >> + raise >> + >> +# >> +# RTEMS Packages we maintian a git hash of in the RSB >> +# >> +rtems_pkg_cfgs = [ >> + [ >> + 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg', >> + 'rtems_tools_version', 'git://git.rtems.org/rtems-tools', >> + 'rtems-tools.git', 'master', >> + ' >> https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2 >> ', >> + 'rtems-tools-%{rtems_tools_version}.tar.bz2' >> + ], >> + [ >> + 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg', >> + 'rtems_kernel_version', 'git://git.rtems.org/rtems', >> 'rtems.git', >> + 'master', >> + ' >> https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2 >> ', >> + 'rtems-kernel-%{rtems_kernel_version}.tar.bz2' >> + ], >> + [ >> + 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg', >> + 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd', >> + 'rtems-libbsd.git', '6-freebsd-12', >> + ' >> https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext} >> ', >> + 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}' >> + ], >> + [ >> + 'RTEMS Net Legacy', >> 'tools/rtems-net-legacy-%{rtems_version}.cfg', >> + 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy', >> + 'rtems-net-legacy.git', 'main', >> + ' >> https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext} >> ', >> + 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}' >> + ], >> + [ >> + 'RTEMS Net Services', 'net/net-services-1.cfg', >> + 'rtems_net_services_version', 'git:// >> git.rtems.org/rtems-net-services', >> + 'rtems-net-services.git', 'master', >> + ' >> https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext} >> ', >> + >> 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}' >> + ], >> +] >> + >> + >> +def clean_line(line): >> + line = line[0:-1] >> + b = line.find('#') >> + if b >= 0: >> + line = line[1:b] + ('\\' if line[-1] == '\\' else '') >> + return line.strip() >> + >> + >> +def clean_and_pack(line, last_line): >> + leading_ws = ' ' if len(line) > 0 and line[0].isspace() else '' >> + line = clean_line(line) >> + if len(last_line) > 0: >> + line = last_line + leading_ws + line >> + return line >> + >> + >> +def config_patch(configdir, config, version_label, config_hash, >> repo_hash, >> + checksum): >> + for cd in configdir.split(':'): >> + cf = path.join(cd, config) >> + if path.exists(cf): >> + try: >> + with open(cf) as f: >> + lines = f.readlines() >> + except IOError as err: >> + raise error.general('config: %s: read error: %s' % >> + (config, str(err))) >> + new_config = [] >> + new_lines = [] >> + last_line = '' >> + for line in lines: >> + new_lines += [line] >> + line = clean_and_pack(line, last_line) >> + if len(line) > 0: >> + if line[-1] == '\\': >> + last_line = line[:-1] >> + continue >> + last_line = '' >> + if version_label in line and not 'rsb_version' in >> line: >> + if line.startswith('%define ' + version_label): >> + new_lines = [ >> + '%define ' + version_label + ' ' + >> repo_hash + >> + os.linesep >> + ] >> + elif line.startswith('%hash '): >> + ls = line.split() >> + if len(ls) != 4: >> + raise error.general('invalid %hash: ' + >> line) >> + new_lines = [ >> + ' '.join(ls[0:3]) + ' \\' + os.linesep, >> + ' ' + checksum + os.linesep >> + ] >> + new_config += new_lines >> + new_lines = [] >> + try: >> + with open(cf, 'w') as f: >> + f.writelines(new_config) >> + except IOError as err: >> + raise error.general('config: %s: write error: %s' % >> + (config, str(err))) >> + return >> + raise error.general('could not find: ' + config) >> + >> + >> +def checksum_shar512_base64(tarball): >> + hasher = hashlib.new('sha512') >> + try: >> + with open(path.host(tarball), 'rb') as f: >> + hasher.update(f.read()) >> + except IOError as err: >> + log.notice('hash: %s: read error: %s' % (file_, str(err))) >> + except: >> + raise >> + raise error.general('cannot hash the tar file') >> + hash_hex = hasher.hexdigest() >> + hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8') >> + return hash_base64 >> + >> + >> +def run(args=sys.argv): >> + ec = 0 >> + output = [] >> + try: >> + # >> + # The RSB options support cannot be used because it loads the >> defaults >> + # for the host which we cannot do here. >> + # >> + description = 'RTEMS Track Dependencies a build set has for all >> hosts.' >> + >> + argsp = argparse.ArgumentParser(prog='sb-rtems-pkg', >> + description=description) >> + argsp.add_argument('--rtems-version', >> + help='Set the RTEMS version.', >> + type=str, >> + default=version.version()) >> + argsp.add_argument('--output', >> + help='Output file.', >> + type=str, >> + default=None) >> + argsp.add_argument('--log', >> + help='Log file.', >> + type=str, >> + default=simhost.log_default('rtems-pkg')) >> + argsp.add_argument('--trace', >> + help='Enable trace logging for debugging.', >> + action='store_true') >> + argsp.add_argument('--dry-run', >> + help='Dry run, do not update the >> configurations', >> + action='store_true') >> + argsp.add_argument('bsets', nargs='*', help='Build sets.') >> + >> + argopts = argsp.parse_args(args[1:]) >> + >> + simhost.load_log(argopts.log) >> + log.notice('RTEMS Source Builder - RTEMS Package Update, %s' % >> + (version.string())) >> + log.tracing = argopts.trace >> + >> + opts = simhost.load_options(args, argopts, >> extras=['--with-download']) >> + opts.defaults['_rsb_getting_source'] = '1' >> + opts.defaults[ >> + 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}' >> + host = 'freebsd' >> + >> + for cfg in rtems_pkg_cfgs: >> + b = None >> + try: >> + bopts = copy.copy(opts) >> + bmacros = copy.copy(opts.defaults) >> + b = build.build(cfg[1], False, bopts, bmacros) >> + git_hash_key = b.macros.find(cfg[2]) >> + if len(git_hash_key) == 0: >> + raise error.general(cfg[0] + ': cannot find version >> macro') >> + source_dir = b.macros.expand('%{_sourcedir}') >> + config_hash = b.macros.expand('%{' + cfg[2] + '}') >> + repo_path = path.join(source_dir, cfg[4]) >> + download.get_file(cfg[3] + '?fetch?checkout=' + cfg[5], >> + repo_path, bopts, b) >> + repo = git.repo(repo_path) >> + repo_hash = repo.head() >> + if config_hash != repo_hash: >> + update = True >> + update_str = 'UPDATE' >> + else: >> + update = False >> + update_str = 'matching' >> + print(cfg[0] + ': ' + update_str + ' config:' + >> config_hash + >> + ' repo:' + repo_hash) >> + b.macros[cfg[2]] = repo_hash >> + tarball = b.macros.expand(cfg[7]) >> + b.macros.set_write_map('hashes') >> + b.macros[tarball] = 'NO-HASH NO-HASH' >> + b.macros.unset_write_map() >> + tarball_path = path.join(source_dir, >> b.macros.expand(cfg[7])) >> + download.get_file(b.macros.expand(cfg[6]), tarball_path, >> bopts, >> + b) >> + tarball_hash = checksum_shar512_base64(tarball_path) >> + if update: >> + config_patch(b.macros.expand('%{_configdir}'), >> + b.macros.expand(cfg[1]), cfg[2], >> config_hash, >> + repo_hash, tarball_hash) >> + del b >> + except error.general as gerr: >> + log.stderr(str(gerr)) >> + log.stderr('Build FAILED') >> + b = None >> + except error.general as gerr: >> + log.stderr(str(gerr)) >> + log.stderr('Build FAILED') >> + ec = 1 >> + except error.internal as ierr: >> + log.stderr(str(ierr)) >> + log.stderr('Internal Build FAILED') >> + ec = 1 >> + except error.exit as eerr: >> + pass >> + except KeyboardInterrupt: >> + log.notice('abort: user terminated') >> + ec = 1 >> + except: >> + raise >> + log.notice('abort: unknown error') >> + ec = 1 >> + sys.exit(ec) >> + >> + >> +if __name__ == "__main__": >> + run() >> -- >> 2.42.0 >> >> _______________________________________________ >> devel mailing list >> devel@rtems.org >> http://lists.rtems.org/mailman/listinfo/devel >> > _______________________________________________ > devel mailing list > devel@rtems.org > http://lists.rtems.org/mailman/listinfo/devel
_______________________________________________ devel mailing list devel@rtems.org http://lists.rtems.org/mailman/listinfo/devel
