This revision was landed with ongoing or failed builds. This revision was automatically updated to reflect the committed changes. Closed by commit rGcf1a3d93581f: Implement the monolithic CI pipeline in the monorepo (authored by ldionne).
Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D158863/new/ https://reviews.llvm.org/D158863 Files: .ci/generate-buildkite-pipeline-premerge .ci/generate-buildkite-pipeline-scheduled .ci/monolithic-linux.sh .ci/monolithic-windows.sh
Index: .ci/monolithic-windows.sh =================================================================== --- /dev/null +++ .ci/monolithic-windows.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This script performs a monolithic build of the monorepo and runs the tests of +# most projects on Windows. This should be replaced by per-project scripts that +# run only the relevant tests. +# + +set -ex +set -o pipefail + +MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" +BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-windows}" + +rm -rf ${BUILD_DIR} + +sccache --zero-stats +function show-stats { + sccache --show-stats +} +trap show-stats EXIT + +projects="${1}" +targets="${2}" + +echo "--- cmake" +pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ + -D LLVM_ENABLE_PROJECTS="${projects}" \ + -G Ninja \ + -D CMAKE_BUILD_TYPE=Release \ + -D LLVM_ENABLE_ASSERTIONS=ON \ + -D LLVM_BUILD_EXAMPLES=ON \ + -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D COMPILER_RT_BUILD_ORC=OFF \ + -D CMAKE_C_COMPILER_LAUNCHER=sccache \ + -D CMAKE_CXX_COMPILER_LAUNCHER=sccache + +echo "--- ninja" +ninja -C ${BUILD_DIR} ${targets} Index: .ci/monolithic-linux.sh =================================================================== --- /dev/null +++ .ci/monolithic-linux.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This script performs a monolithic build of the monorepo and runs the tests of +# most projects on Linux. This should be replaced by per-project scripts that +# run only the relevant tests. +# + +set -ex +set -o pipefail + +MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" +BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-linux}" + +rm -rf ${BUILD_DIR} + +ccache --zero-stats +ccache --show-config +function show-stats { + ccache --print-stats +} +trap show-stats EXIT + +projects="${1}" +targets="${2}" + +echo "--- cmake" +pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ + -D LLVM_ENABLE_PROJECTS="${projects}" \ + -G Ninja \ + -D CMAKE_BUILD_TYPE=Release \ + -D LLVM_ENABLE_ASSERTIONS=ON \ + -D LLVM_BUILD_EXAMPLES=ON \ + -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D LLVM_ENABLE_LLD=ON \ + -D CMAKE_CXX_FLAGS=-gmlt \ + -D BOLT_CLANG_EXE=/usr/bin/clang \ + -D LLVM_CCACHE_BUILD=ON + +echo "--- ninja" +ninja -C ${BUILD_DIR} ${targets} Index: .ci/generate-buildkite-pipeline-scheduled =================================================================== --- .ci/generate-buildkite-pipeline-scheduled +++ .ci/generate-buildkite-pipeline-scheduled @@ -27,4 +27,43 @@ message: "${BUILDKITE_MESSAGE}" commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" -EOF + + - label: ':linux: x64 Debian' + artifact_paths: + - '*_result.json' + - 'build/monolithic-linux/test-results.xml' + agents: + queue: 'linux' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 120 + env: + CC: 'clang' + CXX: 'clang++' + commands: + - './.ci/monolithic-linux.sh "bolt;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"' + + - label: ':windows: x64 Windows' + artifact_paths: + - '*_result.json' + - 'build/monolithic-windows/test-results.xml' + agents: + queue: 'windows' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 150 + env: + CC: 'cl' + CXX: 'cl' + LD: 'link' + commands: + - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' + - 'bash .ci/monolithic-windows.sh "clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl" "check-all"' Index: .ci/generate-buildkite-pipeline-premerge =================================================================== --- .ci/generate-buildkite-pipeline-premerge +++ .ci/generate-buildkite-pipeline-premerge @@ -14,11 +14,6 @@ # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. # -if ! git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/|^clang/"; then - # libcxx/, libcxxabi/, libunwind/, runtimes/, cmake/ or clang/ are not affected - exit 0 -fi - reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')" if [[ "${reviewID}" != "" ]]; then buildMessage="https://llvm.org/${reviewID}"; @@ -51,3 +46,213 @@ branch: "${BUILDKITE_BRANCH}" EOF fi + +# +# If we're not running a more specific pipeline, generate a legacy monolithic pipeline +# based on which subdirectories have been modified. We have some heuristics for this +# to be reasonable. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. +# +function compute-projects-to-test() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + lld) + for p in bolt cross-project-tests; do + echo $p + done + ;; + llvm) + for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do + echo $p + done + ;; + clang) + for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do + echo $p + done + ;; + clang-tools-extra) + echo libc + ;; + mlir) + echo flang + ;; + *) + # Nothing to do + ;; + esac + done +} + +function add-dependencies() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + bolt) + for p in lld llvm; do + echo $p + done + ;; + cross-project-tests) + for p in lld clang; do + echo $p + done + ;; + clang-tools-extra) + for p in llvm clang; do + echo $p + done + ;; + compiler-rt|libc|openmp) + echo clang + ;; + flang|lldb) + for p in llvm clang; do + echo $p + done + ;; + lld|mlir|polly) + echo llvm + ;; + *) + # Nothing to do + ;; + esac + done +} + +function exclude-linux() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + lldb) ;; # tests failing + openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 + *) + echo "${project}" + ;; + esac + done +} + +function exclude-windows() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + compiler-rt) ;; # tests taking too long + openmp) ;; # TODO: having trouble with the Perl installation + libc) ;; # no Windows support + lldb) ;; # tests failing + bolt) ;; # tests are not supported yet + *) + echo "${project}" + ;; + esac + done +} + +function keep-modified-projects() { + projects=${@} + git_diff="$(git diff --name-only HEAD~1)" + for project in ${projects}; do + if echo "${git_diff}" | grep -q -E "^${project}/"; then + echo "${project}" + fi + done +} + +function check-targets() { + projects=${@} + for project in ${projects}; do + case ${project} in + clang-tools-extra) + echo "check-clang-tools" + ;; + compiler-rt) + echo "check-all" + ;; + cross-project-tests) + echo "check-cross-project" + ;; + lldb) + echo "check-all" # TODO: check-lldb may not include all the LLDB tests? + ;; + pstl) + echo "check-all" + ;; + libclc) + echo "check-all" + ;; + *) + echo "check-${project}" + ;; + esac + done +} + +# Figure out which projects need to be built on each platform +all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" +modified_projects="$(keep-modified-projects ${all_projects})" + +linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects})) +linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq) +linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq) + +windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects})) +windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq) +windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq) + +# Generate the appropriate pipeline +if [[ "${linux_projects}" != "" ]]; then + cat <<EOF +- label: ':linux: x64 Debian' + artifact_paths: + - '*_result.json' + - 'build/monolithic-linux/test-results.xml' + agents: + queue: 'linux' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 120 + env: + CC: 'clang' + CXX: 'clang++' + commands: + - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"' +EOF +fi + +if [[ "${windows_projects}" != "" ]]; then + cat <<EOF +- label: ':windows: x64 Windows' + artifact_paths: + - '*_result.json' + - 'build/monolithic-windows/test-results.xml' + agents: + queue: 'windows' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 150 + env: + CC: 'cl' + CXX: 'cl' + LD: 'link' + commands: + - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' + - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"' +EOF +fi
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
