From: Pietro Monteiro <[email protected]>
Build autoconf and automake and add autoregen.py from
https://sourceware.org/git/builder.git
Add forge action to build container images.
ChangeLog:
* .forgejo/workflows/build-containers.yaml: New file.
contrib/ChangeLog:
* ci-containers/README: New file.
* ci-containers/autoregen/Containerfile: New file.
* ci-containers/autoregen/autoregen.py: New file.
* ci-containers/build-image.sh: New file.
Signed-off-by: Pietro Monteiro <[email protected]>
---
.forgejo/workflows/build-containers.yaml | 55 +++++++
contrib/ci-containers/README | 47 ++++++
contrib/ci-containers/autoregen/Containerfile | 78 ++++++++++
contrib/ci-containers/autoregen/autoregen.py | 146 ++++++++++++++++++
contrib/ci-containers/build-image.sh | 104 +++++++++++++
5 files changed, 430 insertions(+)
create mode 100644 .forgejo/workflows/build-containers.yaml
create mode 100644 contrib/ci-containers/README
create mode 100644 contrib/ci-containers/autoregen/Containerfile
create mode 100755 contrib/ci-containers/autoregen/autoregen.py
create mode 100755 contrib/ci-containers/build-image.sh
diff --git a/.forgejo/workflows/build-containers.yaml
b/.forgejo/workflows/build-containers.yaml
new file mode 100644
index 000000000000..93eaeb7515f4
--- /dev/null
+++ b/.forgejo/workflows/build-containers.yaml
@@ -0,0 +1,55 @@
+on:
+# push:
+# branches:
+# - trunk
+# # run on changes to any file for ci containers or to this file
+# paths:
+# - .forgejo/workflows/build-containers.yaml
+# - 'contrib/ci-containers/**/*'
+ # similar for pull requests
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths:
+ - .forgejo/workflows/build-containers.yaml
+ - 'contrib/ci-containers/**/*'
+
+jobs:
+ containers:
+ runs-on: sourceware-runner
+ container:
+ image: fedora:latest
+ env:
+ # the default overlayfs doesn't work when running on docker, which
uses overlayfs
+ STORAGE_DRIVER: vfs
+ # we can't run containers in docker, so use a chroot to build the image
+ BUILDAH_ISOLATION: chroot
+ steps:
+ - name: install dependencies
+ run: |
+ dnf -y --setopt=install_weak_deps=False install buildah git nodejs
+
+ # Checkout sources
+ - uses: actions/checkout@v4
+
+ - name: build containers
+ run: |
+ echo "Building containers from contrib/ci-containers"
+ for DIR in ./contrib/ci-containers/*
+ do
+ ! [ -d "$DIR" ] && continue
+ CONTAINER="$(basename "$DIR")"
+ if [ "$FORGEJO_EVENT_NAME" = pull_request ]; then
+ # branch name in lowercase, replace non-alphanumerics with '-',
and remove leading and trailling '-'
+ TAG="$(echo "$FORGEJO_HEAD_REF" | sed -e 's/\(.*\)/\L\1/' -e
's/[^[:alnum:]-]/-/g' -e 's/^-\+//;s/-\+$//')"
+ else
+ # branch name
+ TAG="$FORGEJO_REF_NAME"
+ fi
+ echo "Building $CONTAINER with tag $TAG"
+ ./contrib/ci-containers/build-image.sh -d "$DIR" -t "$TAG" --
--network=host
+ echo "Built $CONTAINER:$TAG should push it somewhere"
+ buildah images --json "$CONTAINER:$TAG"
+ echo "Removing container image from localhost"
+ buildah rmi "$CONTAINER:$TAG"
+ buildah rmi --prune
+ done
diff --git a/contrib/ci-containers/README b/contrib/ci-containers/README
new file mode 100644
index 000000000000..bd985e9ebe4c
--- /dev/null
+++ b/contrib/ci-containers/README
@@ -0,0 +1,47 @@
+# CI Containers
+
+Each subdirectory under `contrib/ci-containers/` holds a hermetic description
of
+a container image that powers jobs on the [Sourceware
+Forge](https://forge.sourceware.org). The directory itself is used as the
build
+context, so any assets referenced by the `Containerfile` must be present
+in the subdirectory.
+
+Keeping the description self-contained guarantees reproducible builds.
+
+## Building Images
+
+Images are built with [buildah](https://buildah.io) via the helper script
+`build-image.sh`. A typical invocation looks like:
+
+```bash
+./contrib/ci-containers/build-image.sh \
+ -d ./contrib/ci-containers/foo \
+ -t v1.0 \
+ -- --layers --no-cache
+```
+
+* `-d` - Path to the directory containing the `Containerfile`.
+* `-t` - Tag to apply to the resulting image.
+* The trailing `--` passes additional flags directly to `buildah` (here we
+request layered output and disable the cache).
+
+The full image tag will be the basename of the directory, in this case `foo`,
+and the value passed to the `-t/--tag` argument. Our hypothetical image will
be
+tagged locally as `foo:v1.0`.
+
+### Verify the build
+
+```bash
+buildah images --json foo:v1.0
+```
+
+The command returns a JSON object with the image's ID, size, and other
metadata.
+
+### Test the image locally
+
+```bash
+podman run --rm -it foo:v1.0 /bin/bash
+```
+
+By running the image interactively you can confirm that the environment behaves
+as expected.
diff --git a/contrib/ci-containers/autoregen/Containerfile
b/contrib/ci-containers/autoregen/Containerfile
new file mode 100644
index 000000000000..8105c6bd16cc
--- /dev/null
+++ b/contrib/ci-containers/autoregen/Containerfile
@@ -0,0 +1,78 @@
+FROM debian:stable-slim
+
+# Run time deps
+RUN set -eux; \
+ apt-get update; \
+ apt-get upgrade -y; \
+ apt-get install -y --no-install-recommends \
+ autogen \
+ ca-certificates \
+ git \
+ m4 \
+ nodejs \
+ perl \
+ python3 \
+ python3-git \
+ python3-termcolor \
+ python3-unidiff \
+ wget; \
+ rm -rf /var/lib/apt/lists/*
+
+# Get and install the autoregen.py script
+COPY --chmod=755 autoregen.py /usr/local/bin/autoregen.py
+
+# Build and install autoconf-2.69 and automake-1.15.1
+# Automake depends on autoconf, which is built and installed first
+RUN set -eux; \
+ \
+ savedAptMark="$(apt-mark showmanual)"; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ gzip \
+ m4 \
+ tar \
+ wget \
+ ; \
+ rm -r /var/lib/apt/lists/*; \
+ \
+ builddir="$(mktemp -d)"; \
+ cd "${builddir}"; \
+ \
+ wget https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz; \
+ tar xf autoconf-2.69.tar.gz; \
+ cd autoconf-2.69; \
+ ./configure --program-suffix=-2.69; \
+ make; \
+ make install; \
+ cd .. ;\
+ rm -rf autoconf*; \
+ cd /usr/local/bin; \
+ ln -s autoconf-2.69 autoconf; \
+ ln -s autoheader-2.69 autoheader; \
+ ln -s autom4te-2.69 autom4te; \
+ ln -s autoreconf-2.69 autoreconf; \
+ ln -s autoscan-2.69 autoscan; \
+ ln -s autoupdate-2.69 autoupdate; \
+ \
+ cd "${builddir}"; \
+ wget https://ftp.gnu.org/gnu/automake/automake-1.15.1.tar.gz; \
+ tar xf automake-1.15.1.tar.gz; \
+ cd automake-1.15.1; \
+ ./configure --program-suffix=-1.15.1; \
+ make; \
+ make install; \
+ cd ..; \
+ rm -rf automake*; \
+ cd /usr/local/bin; \
+ ln -s aclocal-1.15.1 aclocal-1.15; \
+ ln -s aclocal-1.15.1 aclocal; \
+ ln -s automake-1.15.1 automake-1.15; \
+ ln -s automake-1.15.1 automake; \
+ \
+ rm -rf "${builddir}"; \
+ \
+ apt-mark auto '.*' > /dev/null; \
+ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; \
+ apt-get purge -y --auto-remove -o
APT::AutoRemove::RecommendsImportant=false
diff --git a/contrib/ci-containers/autoregen/autoregen.py
b/contrib/ci-containers/autoregen/autoregen.py
new file mode 100755
index 000000000000..7ac17c1b9dd8
--- /dev/null
+++ b/contrib/ci-containers/autoregen/autoregen.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+# This script helps to regenerate files managed by autotools and
+# autogen in binutils-gdb and gcc repositories.
+
+# It can be used by buildbots to check that the current repository
+# contents has been updated correctly, and by developers to update
+# such files as expected.
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+
+# On Gentoo, vanilla unpatched autotools are packaged separately.
+# We place the vanilla names first as we want to prefer those if both exist.
+AUTOCONF_NAMES = ["autoconf-vanilla-2.69", "autoconf-2.69", "autoconf"]
+AUTOMAKE_NAMES = ["automake-vanilla-1.15", "automake-1.15.1", "automake"]
+ACLOCAL_NAMES = ["aclocal-vanilla-1.15", "aclocal-1.15.1", "aclocal"]
+AUTOHEADER_NAMES = ["autoheader-vanilla-2.69", "autoheader-2.69", "autoheader"]
+AUTORECONF_NAMES = ["autoreconf-vanilla-2.69", "autoreconf-2.69", "autoreconf"]
+
+# Pick the first for each list that exists on this system.
+AUTOCONF_BIN = next(name for name in AUTOCONF_NAMES if shutil.which(name))
+AUTOMAKE_BIN = next(name for name in AUTOMAKE_NAMES if shutil.which(name))
+ACLOCAL_BIN = next(name for name in ACLOCAL_NAMES if shutil.which(name))
+AUTOHEADER_BIN = next(name for name in AUTOHEADER_NAMES if shutil.which(name))
+AUTORECONF_BIN = next(name for name in AUTORECONF_NAMES if shutil.which(name))
+
+AUTOGEN_BIN = "autogen"
+
+# autoconf-wrapper and automake-wrapper from Gentoo look at this environment
variable.
+# It's harmless to set it on other systems though.
+EXTRA_ENV = {
+ "WANT_AUTOCONF": AUTOCONF_BIN.split("-", 1)[1] if "-" in AUTOCONF_BIN else
"",
+ "WANT_AUTOMAKE": AUTOMAKE_BIN.split("-", 1)[1] if "-" in AUTOMAKE_BIN else
"",
+ "AUTOCONF": AUTOCONF_BIN,
+ "ACLOCAL": ACLOCAL_BIN,
+ "AUTOMAKE": AUTOMAKE_BIN,
+ "AUTOGEN": AUTOGEN_BIN,
+}
+ENV = os.environ.copy()
+ENV.update(EXTRA_ENV)
+
+
+# Directories we should skip entirely because they're vendored or have
different
+# autotools versions.
+SKIP_DIRS = [
+ # readline and minizip are maintained with different autotools versions
+ "readline",
+ "minizip",
+]
+
+MANUAL_CONF_DIRS = [
+ ".",
+ # autoreconf does not update aclocal.m4
+ "fixincludes",
+ ]
+
+# Run the shell command CMD.
+#
+# Print the command on stdout prior to running it.
+def run_shell(cmd: str):
+ print(f"+ {cmd}", flush=True)
+ res = subprocess.run(
+ f"{cmd}",
+ shell=True,
+ encoding="utf8",
+ env=ENV,
+ )
+ res.check_returncode()
+
+
+def regenerate_with_autoreconf():
+ run_shell(f"{AUTORECONF_BIN} -f")
+
+def regenerate_with_autogen():
+ run_shell(f"{AUTOGEN_BIN} Makefile.def")
+
+def regenerate_manually():
+ configure_lines = open("configure.ac").read().splitlines()
+ if folder.stem == "fixincludes" or any(
+ True for line in configure_lines if
line.startswith("AC_CONFIG_MACRO_DIR")
+ ):
+ include_arg = ""
+ include_arg2 = ""
+ if (folder / ".." / "config").is_dir():
+ include_arg = "-I../config"
+
+ if folder.stem == "fixincludes":
+ include_arg = "-I.."
+ include_arg2 = "-I../config"
+
+ # aclocal does not support the -f short option for force
+ run_shell(f"{ACLOCAL_BIN} --force {include_arg} {include_arg2}")
+
+ if (folder / "config.in").is_file() or any(
+ True for line in configure_lines if
line.startswith("AC_CONFIG_HEADERS")
+ ):
+ run_shell(f"{AUTOHEADER_BIN} -f")
+
+ # apparently automake is somehow unstable -> skip it for gotools
+ if any(
+ True for line in configure_lines if line.startswith("AM_INIT_AUTOMAKE")
+ ) and not str(folder).endswith("gotools"):
+ run_shell(f"{AUTOMAKE_BIN} -f")
+
+ run_shell(f"{AUTOCONF_BIN} -f")
+
+
+run_shell(f"{AUTOCONF_BIN} --version")
+run_shell(f"{AUTOMAKE_BIN} --version")
+run_shell(f"{ACLOCAL_BIN} --version")
+run_shell(f"{AUTOHEADER_BIN} --version")
+
+print(f"Extra environment: {EXTRA_ENV}", flush=True)
+
+config_folders: list[Path] = []
+autogen_folders: list[Path] = []
+repo_root = Path.cwd()
+
+for root, _, files in os.walk("."):
+ for file in files:
+ if file == "configure.ac":
+ config_folders.append(Path(root).resolve())
+ if file == "Makefile.tpl":
+ autogen_folders.append(Path(root).resolve())
+
+for folder in sorted(autogen_folders):
+ print(f"Entering directory {folder}", flush=True)
+ os.chdir(folder)
+ regenerate_with_autogen()
+
+for folder in sorted(config_folders):
+ if folder.stem in SKIP_DIRS:
+ print(f"Skipping directory {folder}", flush=True)
+ continue
+
+ print(f"Entering directory {folder}", flush=True)
+ os.chdir(folder)
+
+ if str(folder.relative_to(repo_root)) in MANUAL_CONF_DIRS:
+ regenerate_manually()
+ else:
+ regenerate_with_autoreconf()
diff --git a/contrib/ci-containers/build-image.sh
b/contrib/ci-containers/build-image.sh
new file mode 100755
index 000000000000..0df8fdfc1f3a
--- /dev/null
+++ b/contrib/ci-containers/build-image.sh
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+#
+# Build a container using buildah
+#
+set -euo pipefail
+
+usage() {
+ cat <<EOF
+Usage: build-image.sh -d <directory> -t <tag> [-e timestamp] [--
buildah-args...]
+
+Options:
+ -d, --dir <path> Directory with the Containerfile (required).
+ -t, --tag <tag> Tag to apply to the built image (required).
+ -e, --epoch <ts> Set the "created" timestamp for the built image to this
number of seconds since the epoch (optional).
+ Default is to use the timestamp of the current commit.
+ Needs buildah 1.41 or newer.
+ -h, --help Show this help message and exit.
+
+All arguments after a double-dash (--) are forwarded unchanged to 'buildah'.
+
+Example:
+ ./build-image.sh -d src -t v1.0 -- --layers --no-cache
+EOF
+ exit 1
+}
+
+DIR=""
+TAG=""
+EXTRA_ARGS=()
+
+while (( "$#" )); do
+ case "$1" in
+ -d|--dir)
+ if [[ -n "${2-}" ]]; then
+ DIR="$2"
+ shift 2
+ else
+ echo "error: --dir requires a value" >&2
+ exit 1
+ fi
+ ;;
+ -t|--tag)
+ if [[ -n "${2-}" ]]; then
+ TAG="$2"
+ shift 2
+ else
+ echo "error: --tag requires a value" >&2
+ exit 1
+ fi
+ ;;
+ -e|--epoch)
+ if [[ -n "${2-}" ]]; then
+ SOURCE_DATE_EPOCH="$2"
+ shift 2
+ else
+ echo "error: --source-date-epoch requires a value" >&2
+ exit 1
+ fi
+ ;;
+ -h|--help)
+ usage
+ ;;
+ --)
+ shift
+ EXTRA_ARGS+=("$@")
+ break
+ ;;
+ *)
+ echo "error: unknown option '$1'" >&2
+ usage
+ ;;
+ esac
+done
+
+if [[ -z "$DIR" ]]; then
+ echo "error: directory (-d/--dir) is required" >&2
+ usage
+fi
+
+if [[ -z "$TAG" ]]; then
+ echo "error: Tag (-t/--tag) is required." >&2
+ usage
+fi
+
+if [[ ! -e "${DIR}/Containerfile" ]]; then
+ echo "error: '${DIR}/Containerfile' does not exist." >&2
+ usage
+fi
+
+CONTAINER="$(basename "$DIR")"
+IMAGE_TAG="${CONTAINER}:${TAG}"
+
+if [[ -z "${SOURCE_DATE_EPOCH-}" ]]; then
+ SCRIPT_DIR="$(dirname "$0")"
+ SOURCE_DATE_EPOCH="$(cd "${SCRIPT_DIR}" && git log -1 --pretty=%ct)"
+fi
+export SOURCE_DATE_EPOCH
+
+
+buildah build \
+ -f "${DIR}/Containerfile" \
+ -t "$IMAGE_TAG" \
+ "${EXTRA_ARGS[@]}" \
+ "$DIR"
--
2.52.0
