commit: 2a77ebf7e7c02d43eb9dd385dc808efd87a54af0 Author: Konstantinos Smanis <konstantinos.smanis <AT> gmail <DOT> com> AuthorDate: Sun Aug 23 18:54:34 2020 +0000 Commit: Alexys Jacob <ultrabug <AT> gentoo <DOT> org> CommitDate: Thu Sep 3 20:34:00 2020 +0000 URL: https://gitweb.gentoo.org/proj/docker-images.git/commit/?id=2a77ebf7
Add CI builds for all supported stage3 architectures Build multiarch images using buildx [1] instead of modifying the image architecture post-creation with docker-copyedit. Although still experimental, buildx is the recommended way of building multi-platform images. All stage3 architectures that are supported by Docker [2] were added. Closes: #61 [1] https://docs.docker.com/buildx/working-with-buildx/#build-multi-platform-images [2] https://github.com/docker-library/official-images#architectures-other-than-amd64 Signed-off-by: Konstantinos Smanis <konstantinos.smanis <AT> gmail.com> Closes: https://github.com/gentoo/gentoo-docker-images/pull/92 Signed-off-by: Alexys Jacob <ultrabug <AT> gentoo.org> .gitmodules | 3 --- .travis.yml | 67 ++++++++++++++++++++++++++++++++------------------- README.md | 59 +++++++++++++++++++++++++++++++++++++-------- build-multiarch.sh | 9 ------- build.sh | 71 ++++++++++++++++++++++++++++++++++-------------------- docker-copyedit | 1 - portage.Dockerfile | 2 +- stage3.Dockerfile | 2 +- 8 files changed, 138 insertions(+), 76 deletions(-) diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 52c678d..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "docker-copyedit"] - path = docker-copyedit - url = https://github.com/gdraheim/docker-copyedit.git diff --git a/.travis.yml b/.travis.yml index 874e1ce..50ba3f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,46 +1,63 @@ -services: docker -language: bash +language: shell env: global: + - DOCKER_CLI_EXPERIMENTAL=enabled # required by buildx - ORG=gentoo - matrix: + jobs: - TARGET=portage - TARGET=stage3-amd64 - TARGET=stage3-amd64-hardened - TARGET=stage3-amd64-hardened-nomultilib + - TARGET=stage3-amd64-musl-hardened + - TARGET=stage3-amd64-musl-vanilla - TARGET=stage3-amd64-nomultilib - TARGET=stage3-amd64-systemd - - TARGET=stage3-amd64-musl-vanilla - - TARGET=stage3-amd64-musl-hardened + - TARGET=stage3-amd64-uclibc-hardened + - TARGET=stage3-amd64-uclibc-vanilla + - TARGET=stage3-arm64 + - TARGET=stage3-arm64-systemd + - TARGET=stage3-armv5tel + - TARGET=stage3-armv6j_hardfp + - TARGET=stage3-armv7a_hardfp + - TARGET=stage3-ppc64le + - TARGET=stage3-s390x - TARGET=stage3-x86 - TARGET=stage3-x86-hardened - - TARGET=stage3-armv7a - TARGET=stage3-x86-musl-vanilla - - TARGET=stage3-ppc - - TARGET=stage3-ppc64 - - TARGET=stage3-ppc64le + - TARGET=stage3-x86-systemd + - TARGET=stage3-x86-uclibc-hardened + - TARGET=stage3-x86-uclibc-vanilla before_install: - # Install latest Docker + # Install latest Docker (>=19.03.0 required by buildx) + # https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - sudo apt-get update -qq - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json - - sudo apt-get install qemu-user-static binfmt-support - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker info +before_script: + # Create multiarch buildx builder + - docker buildx create --driver docker-container --use script: - - VERSION="$(date -u +%Y%m%d)" - - sudo ./build.sh - - if [[ "${TARGET}" == stage* ]]; then - sudo docker run -it --rm ${ORG}/${TARGET}:${VERSION} /bin/bash -c "emerge --info" ; - fi - -# Travis is set up to push daily to dockerhub + # Build image + - ./build.sh after_success: - - if [[ "${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" == "master" && "${TRAVIS_EVENT_TYPE}" == "cron" ]]; then - VERSION=$(date -u +%Y%m%d); - echo "${DOCKER_PASSWORD}" | docker login -u="${DOCKER_USERNAME}" --password-stdin; - docker push "${ORG}/${TARGET}:latest"; - docker push "${ORG}/${TARGET}:${VERSION}"; + # Inspect built image + - docker image inspect "${ORG}/${TARGET}:latest" + # Run `emerge --info` for stage builds + - | + if [[ "${TARGET}" == stage* ]]; then + # Check if QEMU emulation support is required + if [[ ! "${TARGET}" =~ -(amd64|x86)($|-) ]]; then + # Enable execution of foreign binary formats (i.e., non-amd64/x86) + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + fi + docker run --rm "${ORG}/${TARGET}:latest" emerge --info + fi + # Push all built images to Docker Hub (cron daily task) + - | + if [[ "${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" == "master" && "${TRAVIS_EVENT_TYPE}" == "cron" ]]; then + echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin + docker push "${ORG}/${TARGET}" fi diff --git a/README.md b/README.md index eb5eba5..10eba12 100644 --- a/README.md +++ b/README.md @@ -14,19 +14,58 @@ https://hub.docker.com/u/gentoo/ ## Inventory -* portage -* stage3 - * stage3-amd64 - * stage3-amd64-hardened - * stage3-amd64-hardened-nomultilib - * stage3-amd64-nomultilib - * stage3-amd64-systemd - * stage3-x86 - * stage3-x86-hardened +The following targets are built by Travis (bold targets are also pushed to Docker Hub): + * **`portage`** + * `stage3` + * `amd64` + * **`stage3-amd64`** + * **`stage3-amd64-hardened`** + * **`stage3-amd64-hardened-nomultilib`** + * `stage3-amd64-musl-hardened` + * `stage3-amd64-musl-vanilla` + * **`stage3-amd64-nomultilib`** + * `stage3-amd64-systemd` + * `stage3-amd64-uclibc-hardened` + * `stage3-amd64-uclibc-vanilla` + * `arm64` + * `stage3-arm64` + * `stage3-arm64-systemd` + * `arm` + * `stage3-armv5tel` + * `stage3-armv6j_hardfp` + * `stage3-armv7a_hardfp` + * `ppc` + * `stage3-ppc64le` + * `s390` + * `stage3-s390x` + * `x86` + * **`stage3-x86`** + * **`stage3-x86-hardened`** + * `stage3-x86-musl-vanilla` + * `stage3-x86-systemd` + * `stage3-x86-uclibc-hardened` + * `stage3-x86-uclibc-vanilla` + +The following upstream stage3 targets are not built at all (see [rationale](https://github.com/gentoo/gentoo-docker-images/issues/75#issuecomment-680776939)): + * `amd64` + * `stage3-amd64-hardened-selinux` + * `stage3-amd64-hardened-selinux+nomultilib` + * `stage3-x32` + * `arm` + * `stage3-armv4tl` + * `stage3-armv6j` + * `stage3-armv7a` + * `ppc` + * `stage3-ppc` + * `stage3-ppc64` + * `s390` + * `stage3-s390` + * `x86` + * `stage3-i486` # Building the containers -The containers are created using a multi-stage build, which requires docker-17.05.0 or later. +The containers are created using a multi-stage build, which requires Docker >= 19.03.0. The container being built is defined by the TARGET environment variable: `` TARGET=stage3-amd64 ./build.sh `` diff --git a/build-multiarch.sh b/build-multiarch.sh deleted file mode 100755 index bc2fde9..0000000 --- a/build-multiarch.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -docker manifest create gentoo/stage3 \ - gentoo/stage3-amd64 \ - gentoo/stage3-x86 \ - gentoo/stage3-armv7a \ - gentoo/stage3-amd64 \ - gentoo/stage3-ppc \ - gentoo/stage3-ppc64 \ - gentoo/stage3-ppc64le diff --git a/build.sh b/build.sh index 10651cf..c0100d7 100755 --- a/build.sh +++ b/build.sh @@ -11,32 +11,45 @@ fi # Split the TARGET variable into three elements separated by hyphens IFS=- read -r NAME ARCH SUFFIX <<< "${TARGET}" -DOCKER_ARCH="${ARCH}" - -# Ensure upstream directories for stage3-amd64-hardened+nomultilib work -# unless we're building for musl targets (vanilla/hardened) -if [[ "${SUFFIX}" != *musl* ]]; then - SUFFIX=${SUFFIX/-/+} -fi VERSION=${VERSION:-$(date -u +%Y%m%d)} ORG=${ORG:-gentoo} -# x86 requires the i686 subfolder -if [[ "${ARCH}" == "x86" ]]; then - DOCKER_ARCH="386" - MICROARCH="i686" - BOOTSTRAP="multiarch/alpine:x86-v3.11" -elif [[ "${ARCH}" = ppc* ]]; then - MICROARCH="${ARCH}" - ARCH=ppc -elif [[ "${ARCH}" = arm* ]]; then - DOCKER_ARCH=$(echo $ARCH | sed -e 's-\(v.\).*-/\1-g') - MICROARCH="${ARCH}" - ARCH=arm -else - MICROARCH="${ARCH}" +case $ARCH in + "amd64" | "arm64") + DOCKER_ARCH="${ARCH}" + MICROARCH="${ARCH}" + ;; + "armv"*) + # armv6j_hardfp -> arm/v6 + # armv7a_hardfp -> arm/v7 + DOCKER_ARCH=$(echo "$ARCH" | sed -e 's#arm\(v.\).*#arm/\1#g') + MICROARCH="${ARCH}" + ARCH="arm" + ;; + "ppc64le") + DOCKER_ARCH="${ARCH}" + MICROARCH="${ARCH}" + ARCH="ppc" + ;; + "s390x") + DOCKER_ARCH="${ARCH}" + MICROARCH="${ARCH}" + ARCH="s390" + ;; + "x86") + DOCKER_ARCH="386" + MICROARCH="i686" + ;; + *) # portage + DOCKER_ARCH="amd64" + ;; +esac + +# Handle targets with special characters in the suffix +if [[ "${TARGET}" == "stage3-amd64-hardened-nomultilib" ]]; then + SUFFIX="hardened+nomultilib" fi # Prefix the suffix with a hyphen to make sure the URL works @@ -44,8 +57,14 @@ if [[ -n "${SUFFIX}" ]]; then SUFFIX="-${SUFFIX}" fi -set -x -docker build --build-arg ARCH="${ARCH}" --build-arg MICROARCH="${MICROARCH}" --build-arg BOOTSTRAP="${BOOTSTRAP}" --build-arg SUFFIX="${SUFFIX}" -t "${ORG}/${TARGET}:${VERSION}" -f "${NAME}.Dockerfile" . -docker-copyedit/docker-copyedit.py FROM "${ORG}/${TARGET}:${VERSION}" INTO "${ORG}/${TARGET}:${VERSION}" -vv \ - set arch ${DOCKER_ARCH} -docker tag "${ORG}/${TARGET}:${VERSION}" "${ORG}/${TARGET}:latest" +docker buildx build \ + --file "${NAME}.Dockerfile" \ + --build-arg ARCH="${ARCH}" \ + --build-arg MICROARCH="${MICROARCH}" \ + --build-arg SUFFIX="${SUFFIX}" \ + --tag "${ORG}/${TARGET}:latest" \ + --tag "${ORG}/${TARGET}:${VERSION}" \ + --platform "linux/${DOCKER_ARCH}" \ + --progress plain \ + --load \ + . diff --git a/docker-copyedit b/docker-copyedit deleted file mode 160000 index ab6bd5d..0000000 --- a/docker-copyedit +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ab6bd5d9f5ca3a9ba314e7124c2aac8ad7987a3b diff --git a/portage.Dockerfile b/portage.Dockerfile index 7f51c7e..23261d2 100644 --- a/portage.Dockerfile +++ b/portage.Dockerfile @@ -3,7 +3,7 @@ # docker-17.05.0 or later. It fetches a daily snapshot from the official # sources and verifies its checksum as well as its gpg signature. -FROM alpine:3.11 as builder +FROM --platform=$BUILDPLATFORM alpine:3.11 as builder WORKDIR /portage diff --git a/stage3.Dockerfile b/stage3.Dockerfile index a9dc594..b9909ee 100644 --- a/stage3.Dockerfile +++ b/stage3.Dockerfile @@ -4,7 +4,7 @@ # sources and verifies its checksum as well as its gpg signature. ARG BOOTSTRAP -FROM ${BOOTSTRAP:-alpine:3.11} as builder +FROM --platform=$BUILDPLATFORM ${BOOTSTRAP:-alpine:3.11} as builder WORKDIR /gentoo
