This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch jenkins-unify-ci-2 in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 07b98682606349c37fc04faba3bcc22aff9bf587 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Mon Sep 15 17:44:40 2025 -0400 Unify CI jobs Instead of having two separate jobs combine both. The new CI is mainly the previous "full" CI but with these stages and features copied from the "pr" CI: * Test multiple erlang versions. Add a test variant for the highest Erlang version. * Add an explicit source check step. * Add an explicit docs update check. Fast-forward if only docs changed. * Use an explicit "base" image. This is the image used for source check, docs checks and is usually a debian oldoldstable or oldstable version. --- build-aux/Jenkinsfile.full | 249 +++++++++++++------ build-aux/Jenkinsfile.pr | 594 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 650 insertions(+), 193 deletions(-) diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full index b47a9669f..db392bcce 100644 --- a/build-aux/Jenkinsfile.full +++ b/build-aux/Jenkinsfile.full @@ -13,13 +13,25 @@ // License for the specific language governing permissions and limitations under // the License. -// Erlang version embedded in binary packages +// This is image used to build the tarball, check source formatting and build +// the docs +DOCKER_IMAGE_BASE = 'apache/couchdbci-debian:bookworm-erlang' + +// Erlang version embedded in binary packages. Also the version most builds +// will run. ERLANG_VERSION = '26.2.5.15' // Erlang version used for rebar in release process. CouchDB will not build from // the release tarball on Erlang versions older than this MINIMUM_ERLANG_VERSION = '26.2.5.15' +// Highest support Erlang version. +MAXIMUM_ERLANG_VERSION = '28.0.4' + +// Use these to detect if just documents changed +docs_changed = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q '^src/docs/'" +other_changes = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q -v '^src/docs/'" + // We create parallel build / test / package stages for each OS using the metadata // in this map. Adding a new OS should ideally only involve adding a new entry here. meta = [ @@ -59,53 +71,62 @@ meta = [ image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" ], - 'bookworm-ppc64': [ + 'bullseye': [ + name: 'Debian x86_64', + spidermonkey_vsn: '78', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" + ], + + 'base-ppc64': [ name: 'Debian POWER', spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, quickjs_test262: true, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", node_label: 'ppc64le' ], - 'bookworm-s390x': [ + 'base-s390x': [ name: 'Debian s390x', spidermonkey_vsn: '78', with_nouveau: true, // QuickJS test262 shows a discrepancy typedarray-arg-set-values-same-buffer-other-type.js // Test262Error: 51539607552,42,0,4,5,6,7,8 quickjs_test262: false, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", node_label: 's390x' ], - 'bullseye': [ + 'base': [ name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, - quickjs_test262: true, - image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" + // Test this in in the bookworm-quickjs variant + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" ], - 'bookworm': [ + 'base-max-erlang': [ name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, - // Test this in in the bookworm-quickjs variant quickjs_test262: false, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" + image: "${DOCKER_IMAGE_BASE}-${MAXIMUM_ERLANG_VERSION}" ], - 'bookworm-quickjs': [ + 'base-quickjs': [ name: 'Debian 12 with QuickJS', disable_spidermonkey: true, with_nouveau: true, with_clouseau: true, quickjs_test262: true, - image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" ], 'freebsd-x86_64': [ @@ -408,11 +429,149 @@ pipeline { } stages { + + stage('Setup Env') { + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 10, unit: 'MINUTES') + } + steps { + script { + env.DOCS_CHANGED = '0' + env.ONLY_DOCS_CHANGED = '0' + if ( sh(returnStatus: true, script: docs_changed) == 0 ) { + env.DOCS_CHANGED = '1' + if (sh(returnStatus: true, script: other_changes) == 1) { + env.ONLY_DOCS_CHANGED = '1' + } + } + } + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage 'Setup Environment' + + stage('Docs Check') { + // Run docs `make check` stage if any docs changed + when { + beforeOptions true + expression { DOCS_CHANGED == '1' } + } + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 15, unit: 'MINUTES') + } + steps { + sh ''' + make python-black + ''' + sh ''' + (cd src/docs && make check) + ''' + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage Docs Check + + stage('Build Docs') { + // Build docs separately if only docs changed. If there are other changes, docs are + // already built as part of `make dist` + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '1' } + } + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 30, unit: 'MINUTES') + } + steps { + sh ''' + (cd src/docs && ./setup.sh ; make html) + ''' + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage Build Docs + + stage('Source Format Checks') { + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '0' } + } + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 15, unit: "MINUTES") + } + steps { + sh ''' + rm -rf apache-couchdb-* + ./configure --skip-deps --spidermonkey-version 78 + make erlfmt-check + make elixir-source-checks + make python-black + ''' + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage Erlfmt + stage('Build Release Tarball') { + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '0' } + } agent { docker { label 'docker' - image "apache/couchdbci-debian:bookworm-erlang-${MINIMUM_ERLANG_VERSION}" + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" args "${DOCKER_ARGS}" registryUrl 'https://docker.io/' registryCredentialsId 'dockerhub_creds' @@ -441,6 +600,10 @@ pipeline { } // stage Build Release Tarball stage('Test and Package') { + when { + beforeOptions true + expression { ONLY_DOCS_CHANGED == '0' } + } steps { script { // Including failFast: true in map fails the build immediately if any parallel step fails @@ -456,64 +619,6 @@ pipeline { } } } - - stage('Publish') { - - when { - expression { return env.BRANCH_NAME ==~ /main|2.*.x|3.*.x|4.*.x|jenkins-.*/ } - } - - agent { - docker { - image "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - skipDefaultCheckout() - timeout(time: 120, unit: "MINUTES") - } - - steps { - sh 'rm -rf ${WORKSPACE}/*' - unstash 'tarball' - unarchive mapping: ['pkgs/' : '.'] - - sh( label: 'Setup repo dirs', script: ''' - mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el8 $BRANCH_NAME/el9 $BRANCH_NAME/source - git clone https://github.com/apache/couchdb-pkg - ''' ) - - sh( label: 'Build Debian repo', script: ''' - for plat in bullseye bookworm - do - reprepro -b couchdb-pkg/repo includedeb $plat pkgs/$plat/*.deb - done - ''' ) - - sh( label: 'Build CentOS 8', script: ''' - (cd pkgs/centos8 && createrepo_c --database .) - ''' ) - - sh( label: 'Build CentOS 9', script: ''' - (cd pkgs/centos9 && createrepo_c --database .) - ''' ) - - sh( label: 'Build unified repo', script: ''' - mv couchdb-pkg/repo/pool $BRANCH_NAME/debian - mv couchdb-pkg/repo/dists $BRANCH_NAME/debian - mv pkgs/centos8/* $BRANCH_NAME/el8 - mv pkgs/centos9/* $BRANCH_NAME/el9 - mv apache-couchdb-*.tar.gz $BRANCH_NAME/source - cd $BRANCH_NAME/source - ls -1tr | head -n -10 | xargs -d '\n' rm -f -- - cd ../.. - ''' ) - } // steps - } // stage } // stages post { diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr index c250db1c6..db392bcce 100644 --- a/build-aux/Jenkinsfile.pr +++ b/build-aux/Jenkinsfile.pr @@ -13,91 +13,457 @@ // License for the specific language governing permissions and limitations under // the License. -build = ''' -mkdir -p ${ERLANG_VERSION} -cd ${ERLANG_VERSION} -rm -rf build -mkdir build -cd build -tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -cd apache-couchdb-* -./configure --with-nouveau --with-clouseau --js-engine=${JS_ENGINE} -''' +// This is image used to build the tarball, check source formatting and build +// the docs +DOCKER_IMAGE_BASE = 'apache/couchdbci-debian:bookworm-erlang' +// Erlang version embedded in binary packages. Also the version most builds +// will run. +ERLANG_VERSION = '26.2.5.15' + +// Erlang version used for rebar in release process. CouchDB will not build from +// the release tarball on Erlang versions older than this +MINIMUM_ERLANG_VERSION = '26.2.5.15' + +// Highest support Erlang version. +MAXIMUM_ERLANG_VERSION = '28.0.4' + +// Use these to detect if just documents changed docs_changed = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q '^src/docs/'" other_changes = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q -v '^src/docs/'" +// We create parallel build / test / package stages for each OS using the metadata +// in this map. Adding a new OS should ideally only involve adding a new entry here. +meta = [ + 'centos8': [ + name: 'CentOS 8', + spidermonkey_vsn: '60', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" + ], + + 'centos9': [ + name: 'CentOS 9', + spidermonkey_vsn: '78', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}" + ], + + 'jammy': [ + name: 'Ubuntu 22.04', + spidermonkey_vsn: '91', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" + ], + + 'noble': [ + name: 'Ubuntu 24.04', + spidermonkey_vsn: '115', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" + ], + + 'bullseye': [ + name: 'Debian x86_64', + spidermonkey_vsn: '78', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" + ], + + 'base-ppc64': [ + name: 'Debian POWER', + spidermonkey_vsn: '78', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", + node_label: 'ppc64le' + ], + + 'base-s390x': [ + name: 'Debian s390x', + spidermonkey_vsn: '78', + with_nouveau: true, + // QuickJS test262 shows a discrepancy typedarray-arg-set-values-same-buffer-other-type.js + // Test262Error: 51539607552,42,0,4,5,6,7,8 + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", + node_label: 's390x' + ], + + 'base': [ + name: 'Debian x86_64', + spidermonkey_vsn: '78', + with_nouveau: true, + with_clouseau: true, + // Test this in in the bookworm-quickjs variant + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" + ], + + 'base-max-erlang': [ + name: 'Debian x86_64', + spidermonkey_vsn: '78', + with_nouveau: true, + with_clouseau: true, + quickjs_test262: false, + image: "${DOCKER_IMAGE_BASE}-${MAXIMUM_ERLANG_VERSION}" + ], + + 'base-quickjs': [ + name: 'Debian 12 with QuickJS', + disable_spidermonkey: true, + with_nouveau: true, + with_clouseau: true, + quickjs_test262: true, + image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" + ], + + 'freebsd-x86_64': [ + name: 'FreeBSD x86_64', + spidermonkey_vsn: '91', + with_clouseau: true, + clouseau_java_home: '/usr/local/openjdk8-jre', + quickjs_test262: false, + gnu_make: 'gmake' + ], + + // Spidermonkey 91 has issues on ARM64 FreeBSD + // use QuickJS for now + 'freebsd-arm64': [ + name: 'FreeBSD ARM64 QuickJS', + disable_spidermonkey: true, + with_clouseau: true, + clouseau_java_home: '/usr/local/openjdk8-jre', + quickjs_test262: false, + gnu_make: 'gmake' + ], + + // Disable temporarily. Forks / shell execs seem to fail there currently + // + // + // 'macos': [ + // name: 'macOS', + // spidermonkey_vsn: '128', + // with_nouveau: false, + // with_clouseau: true, + // clouseau_java_home: '/opt/java/openjdk8/zulu-8.jre/Contents/Home', + // gnu_make: 'make' + // ], + + 'win2022': [ + name: 'Windows 2022', + spidermonkey_vsn: '128', + with_clouseau: true, + quickjs_test262: false, + node_label: 'win' + ] +] + +def String configure(config) { + if (config.disable_spidermonkey) { + result = "./configure --skip-deps --disable-spidermonkey" + } else { + result = "./configure --skip-deps --spidermonkey-version ${config.spidermonkey_vsn}" + } + if (config.with_nouveau) { + result += " --with-nouveau" + } + if (config.with_clouseau) { + result += " --with-clouseau" + } + return result +} + +// Credit to https://stackoverflow.com/a/69222555 for this technique. +// We can use the scripted pipeline syntax to dynamically generate stages, +// and inject them into a map that we pass to the `parallel` step in a script. +// While the scripting approach is very flexible, it's not able to use some +// functionality specific to Declarative Pipelines, like the `agent` and `post` +// directives, so you'll see alternatives like try-catch-finally used for flow +// control and the nested `node` and `docker` blocks in the container stage to +// configure the worker environment. + +// Returns a build stage suitable for non-containerized environments (currently +// macOS and FreeBSD). Coincidentally we do not currently support automated +// package generation on these platforms. This method in invoked when we create +// `parallelStagesMap` below. +def generateNativeStage(platform) { + return { + stage(platform) { + node(platform) { + timeout(time: 180, unit: "MINUTES") { + // Steps to configure and build CouchDB on *nix platforms + if (isUnix()) { + try { + // deleteDir is OK here because we're not inside of a Docker container! + deleteDir() + unstash 'tarball' + withEnv([ + 'HOME='+pwd(), + 'PATH+USRLOCAL=/usr/local/bin', + 'PATH+ERTS=/opt/homebrew/lib/erlang/bin', + 'MAKE='+meta[platform].gnu_make, + 'CLOUSEAU_JAVA_HOME='+meta[platform].clouseau_java_home ?: '' + ]) { + sh 'echo "JAIL_HOST: ${JAIL_HOST}"' + sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) + sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) + dir( "${platform}/build" ) { + sh "${configure(meta[platform])}" + sh '$MAKE' + retry (3) {sh '$MAKE eunit'} + if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} + retry (3) {sh '$MAKE elixir'} + retry (3) {sh '$MAKE elixir-search'} + retry (3) {sh '$MAKE mango-test'} + retry (3) {sh '$MAKE weatherreport-test'} + retry (3) {sh '$MAKE nouveau-test'} + } + } + } + catch (err) { + sh 'ls -l ${WORKSPACE}' + withEnv([ + 'HOME='+pwd(), + 'PATH+USRLOCAL=/usr/local/bin', + 'MAKE='+meta[platform].gnu_make + ]) { + dir( "${platform}/build" ) { + sh 'ls -l' + sh '${MAKE} build-report' + } + } + error("Build step failed with error: ${err.getMessage()}") + } + finally { + junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' + sh 'killall -9 beam.smp || true' + sh 'rm -rf ${WORKSPACE}/*' + } + } else { + //steps to configure and build CouchDB on Windows platforms + stage("${meta[platform].name} - build & test") { + try { + // deleteDir is OK here because we're not inside of a Docker container! + deleteDir() + unstash 'tarball' + powershell( script: "git clone https://github.com/apache/couchdb-glazier";, label: 'Cloning couchdb-glazier repository' ) + powershell( script: "New-Item -ItemType Directory -Path '${platform}/build' -Force", label: 'Create build directories' ) + powershell( script: "tar -xf (Get-Item apache-couchdb-*.tar.gz) -C '${platform}/build' --strip-components=1", label: 'Unpack release' ) + dir( "${platform}/build" ) { + withClouseau = meta[platform].with_clouseau ? '-WithClouseau' : '' + + powershell( script: """ + .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 + .\\configure.ps1 -SkipDeps -WithNouveau ${withClouseau} -SpiderMonkeyVersion ${meta[platform].spidermonkey_vsn} + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + make -f Makefile.win release + """, label: 'Configure and Build') + + //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win eunit", label: 'EUnit tests') + //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win elixir", label: 'Elixir tests') + + powershell( script: """ + .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + make -f Makefile.win elixir-search ERLANG_COOKIE=crumbles + """, label: 'Clouseau tests') + + powershell( script: """ + .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + make -f Makefile.win mango-test ERLANG_COOKIE=crumbles + """, label: 'Mango tests') + + powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win weatherreport-test"', label: 'N/A Weatherreport tests') + + // temporary exclude - random flaky tests on Windows + //powershell( script: """ + // .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 + // Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + // make -f Makefile.win nouveau-test + //""", label: 'Nouveau tests') + } + + powershell( script: """ + .\\couchdb-glazier\\bin\\shell.ps1 + .\\couchdb-glazier\\bin\\build_installer.ps1 -Path '${platform}/build' -IncludeGitSha -DisableICEChecks + """, label: 'Build Windows Installer file') + + archiveArtifacts artifacts: '*.msi', fingerprint: true, onlyIfSuccessful: true + } + catch (err) { + powershell( script: "Get-ChildItem ${WORKSPACE}") + dir( "${platform}/build" ) { + powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win build-report') + powershell( script: 'Get-Content test-results.log') + } + error("Build step failed with error: ${err.getMessage()}") + } + finally { + powershell( script: 'Get-ChildItem') + powershell( script: "Remove-Item -Path '${WORKSPACE}\\*' -Force -Recurse -ErrorAction SilentlyContinue") + powershell( script: 'Get-ChildItem') + } + } + } + } + } + } + } +} + +// Returns a build stage suitable for container-based deployments. This method +// is invoked when we create the `parallelStagesMap` in the pipeline below. +def generateContainerStage(platform) { + return { + // Important: the stage name here must match the parallelStagesMap key for the + // Jenkins UI to render the pipeline stages correctly. Don't ask why. -APK + stage(platform) { + node(meta[platform].get('node_label', 'docker')) { + docker.withRegistry('https://docker.io/', 'dockerhub_creds') { + docker.image(meta[platform].image).inside("${DOCKER_ARGS}") { + timeout(time: 180, unit: "MINUTES") { + stage("${meta[platform].name} - build & test") { + try { + sh( script: "rm -rf ${platform} apache-couchdb-*", label: 'Clean workspace' ) + unstash 'tarball' + sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) + sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) + quickjs_tests262 = meta[platform].quickjs_test262 + dir( "${platform}/build" ) { + sh "${configure(meta[platform])}" + sh 'make' + retry(3) {sh 'make eunit'} + if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} + retry(3) {sh 'make elixir'} + retry(3) {sh 'make elixir-search'} + retry(3) {sh 'make mango-test'} + retry(3) {sh 'make weatherreport-test'} + retry(3) {sh 'make nouveau-test'} + } + } + catch (err) { + sh 'ls -l ${WORKSPACE}' + dir( "${platform}/build" ) { + sh 'ls -l' + sh 'make build-report' + } + error("Build step failed with error: ${err.getMessage()}") + } + finally { + junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' + sh 'rm -rf ${WORKSPACE}/*' + } + } + + stage("${meta[platform].name} - package") { + try { + unstash 'tarball' + sh( script: "mkdir -p ${platform}/couchdb", label: 'Create build directory' ) + sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/couchdb", label: 'Unpack release' ) + sh( script: "cd ${platform} && git clone https://github.com/apache/couchdb-pkg";, label: 'Clone packaging helper repo' ) + dir( "${platform}/couchdb-pkg" ) { + sh( script: 'make', label: 'Build packages' ) + } + sh( label: 'Stage package artifacts for archival', script: """ + rm -rf pkgs/${platform} + mkdir -p pkgs/${platform} + mv ${platform}/rpmbuild/RPMS/\$(arch)/*rpm pkgs/${platform} || true + mv ${platform}/couchdb/*.deb pkgs/${platform} || true + """ ) + archiveArtifacts artifacts: 'pkgs/**', fingerprint: true, onlyIfSuccessful: true + } + catch (err) { + sh 'ls -l ${WORKSPACE}' + error("Build step failed with error: ${err.getMessage()}") + } + finally { + sh 'rm -rf ${WORKSPACE}/*' + } + } + } + } + } + } + } + } +} + +// Finally we have the actual Pipeline. It's mostly a Declarative Pipeline, +// except for the 'Test and Package' stage where we use the `script` step as an +// "escape hatch" to dynamically generate a set of parallel stages to execute. pipeline { // no top-level agent; agents must be declared for each stage agent none environment { - recipient = '[email protected]' // Following fix an issue with git <= 2.6.5 where no committer // name or email are present for reflog, required for git clone GIT_COMMITTER_NAME = 'Jenkins User' GIT_COMMITTER_EMAIL = '[email protected]' - // Parameters for the matrix build - DOCKER_IMAGE_BASE = 'apache/couchdbci-debian:bookworm-erlang' // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64 // We need the jenkins user mapped inside of the image // npm config cache below deals with /home/jenkins not mapping correctly // inside the image DOCKER_ARGS = '-e npm_config_cache=/home/jenkins/.npm -e HOME=. -e MIX_HOME=/home/jenkins/.mix -e HEX_HOME=/home/jenkins/.hex -e PIP_CACHE_DIR=/home/jenkins/.cache/pip -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group -v /home/jenkins/.gradle:/home/jenkins/.gradle:rw,z -v /home/jenkins/.hex:/home/jenkins/.hex:rw,z -v /home/jenkins/.npm:/home/jenkins/.npm:rw,z -v /home/jenkins/.cache/pip:/home/jenkins/.cache/pip:rw,z -v /home/jenkins/.mix:/home/jenkins/.mix:rw,z' - - // *** BE SURE TO ALSO CHANGE THE ERLANG VERSIONS FARTHER DOWN *** - // Search for ERLANG_VERSION - // see https://issues.jenkins.io/browse/JENKINS-61047 for why this cannot - // be done parametrically - LOW_ERLANG_VER = '26.2.5.15' } options { buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) - // This fails the build immediately if any parallel step fails - parallelsAlwaysFailFast() preserveStashes(buildCount: 10) - timeout(time: 3, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') timestamps() } stages { - stage('Setup Env') { - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - timeout(time: 10, unit: 'MINUTES') - } - steps { - script { - env.DOCS_CHANGED = '0' - env.ONLY_DOCS_CHANGED = '0' - if ( sh(returnStatus: true, script: docs_changed) == 0 ) { - env.DOCS_CHANGED = '1' - if (sh(returnStatus: true, script: other_changes) == 1) { - env.ONLY_DOCS_CHANGED = '1' - } - } - } - } - post { - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage 'Setup Environment' - - stage('Docs Check') { + stage('Setup Env') { + agent { + docker { + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" + label 'docker' + args "${DOCKER_ARGS}" + registryUrl 'https://docker.io/' + registryCredentialsId 'dockerhub_creds' + } + } + options { + timeout(time: 10, unit: 'MINUTES') + } + steps { + script { + env.DOCS_CHANGED = '0' + env.ONLY_DOCS_CHANGED = '0' + if ( sh(returnStatus: true, script: docs_changed) == 0 ) { + env.DOCS_CHANGED = '1' + if (sh(returnStatus: true, script: other_changes) == 1) { + env.ONLY_DOCS_CHANGED = '1' + } + } + } + } + post { + cleanup { + // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 + sh 'rm -rf ${WORKSPACE}/*' + } + } + } // stage 'Setup Environment' + + stage('Docs Check') { // Run docs `make check` stage if any docs changed when { beforeOptions true @@ -105,7 +471,7 @@ pipeline { } agent { docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" label 'docker' args "${DOCKER_ARGS}" registryUrl 'https://docker.io/' @@ -140,7 +506,7 @@ pipeline { } agent { docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" label 'docker' args "${DOCKER_ARGS}" registryUrl 'https://docker.io/' @@ -170,7 +536,7 @@ pipeline { } agent { docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" label 'docker' args "${DOCKER_ARGS}" registryUrl 'https://docker.io/' @@ -197,97 +563,83 @@ pipeline { } } // stage Erlfmt - - stage('Make Dist') { + stage('Build Release Tarball') { when { beforeOptions true expression { ONLY_DOCS_CHANGED == '0' } } agent { docker { - image "${DOCKER_IMAGE_BASE}-${LOW_ERLANG_VER}" label 'docker' + image "${DOCKER_IMAGE_BASE}-${MINIMUM_ERLANG_VERSION}" args "${DOCKER_ARGS}" registryUrl 'https://docker.io/' registryCredentialsId 'dockerhub_creds' } } - options { - timeout(time: 15, unit: "MINUTES") - } steps { - sh ''' - rm -rf apache-couchdb-* - ./configure --spidermonkey-version 78 --with-nouveau --with-clouseau - make dist - chmod -R a+w * . - ''' + timeout(time: 30, unit: "MINUTES") { + sh (script: 'rm -rf apache-couchdb-*', label: 'Clean workspace of any previous release artifacts' ) + sh "./configure --spidermonkey-version 78 --with-nouveau" + sh 'make dist' + } } post { success { stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball' + archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true + } + failure { + sh 'ls -l ${WORKSPACE}' } cleanup { // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 sh 'rm -rf ${WORKSPACE}/*' } } - } // stage Make Dist + } // stage Build Release Tarball - // TODO Rework once Improved Docker Pipeline Engine is released - // https://issues.jenkins-ci.org/browse/JENKINS-47962 - // https://issues.jenkins-ci.org/browse/JENKINS-48050 - - stage('Make Check') { + stage('Test and Package') { when { beforeOptions true expression { ONLY_DOCS_CHANGED == '0' } } - matrix { - axes { - axis { - name 'ERLANG_VERSION' - values '26.2.5.15', '27.3.4.3', '28.0.4' - } - axis { - name 'SM_VSN' - values '78' - } - axis { - name 'JS_ENGINE' - values 'quickjs', 'spidermonkey' - } - } - - stages { - stage('Build and Test') { - agent { - docker { - image "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" - label 'docker' - args "${DOCKER_ARGS}" - } - } - options { - skipDefaultCheckout() - timeout(time: 90, unit: "MINUTES") - } - steps { - unstash 'tarball' - sh( script: build ) - retry(3) {sh 'cd ${ERLANG_VERSION}/build/apache-couchdb-* && make check || (make build-report && false)'} + steps { + script { + // Including failFast: true in map fails the build immediately if any parallel step fails + parallelStagesMap = meta.collectEntries( [failFast: false] ) { key, values -> + if (values.image) { + ["${key}": generateContainerStage(key)] } - post { - always { - junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' - } - cleanup { - sh 'rm -rf ${WORKSPACE}/*' - } + else { + ["${key}": generateNativeStage(key)] } - } // stage - } // stages - } // matrix - } // stage "Make Check" + } + parallel parallelStagesMap + } + } + } } // stages + + post { + success { + mail to: '[email protected]', + replyTo: '[email protected]', + subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}", + body: "Yay, we passed. ${env.RUN_DISPLAY_URL}" + } + unstable { + mail to: '[email protected]', + replyTo: '[email protected]', + subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}", + body: "Eep! Build is unstable... ${env.RUN_DISPLAY_URL}" + } + failure { + mail to: '[email protected]', + replyTo: '[email protected]', + subject: "[Jenkins] FAILURE: ${currentBuild.fullDisplayName}", + body: "Boo, we failed. ${env.RUN_DISPLAY_URL}" + } + } + } // pipeline
