ldionne updated this revision to Diff 554016. ldionne marked an inline comment as done. ldionne added a comment. Herald added a reviewer: bollu. Herald added subscribers: cfe-commits, libc-commits, openmp-commits, libcxx-commits, lldb-commits, Sanitizers, Enna1, yota9, ayermolo, jvesely. Herald added a reviewer: rafauler. Herald added a reviewer: Amir. Herald added a reviewer: maksfb. Herald added projects: Sanitizers, LLDB, libc++, OpenMP, libc-project, clang-tools-extra, Flang. Herald added a reviewer: libc++. This revision now requires review to proceed.
Test building all projects. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D158863/new/ https://reviews.llvm.org/D158863 Files: .ci/generate-buildkite-pipeline-premerge .ci/monolithic-linux.sh .ci/monolithic-windows.sh bolt/foo clang-tools-extra/foo compiler-rt/foo cross-project-tests/foo flang/foo libc/foo libclc/foo lld/foo lldb/foo llvm-libgcc/foo llvm/foo mlir/foo openmp/foo polly/foo pstl/foo
Index: .ci/monolithic-windows.sh =================================================================== --- /dev/null +++ .ci/monolithic-windows.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This script performs a monolithic build of the monorepo and runs the tests of +# most projects on Windows. This should be replaced by per-project scripts that +# run only the relevant tests. +# + +set -ex +set -o pipefail + +MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" +BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-windows}" + +rm -rf ${BUILD_DIR} + +sccache --zero-stats +function show-stats { + sccache --show-stats +} +trap show-stats EXIT + +projects="${1}" +targets="${2}" + +echo "--- cmake" +pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ + -D LLVM_ENABLE_PROJECTS="${projects}" \ + -G Ninja \ + -D CMAKE_BUILD_TYPE=Release \ + -D LLVM_ENABLE_ASSERTIONS=ON \ + -D LLVM_BUILD_EXAMPLES=ON \ + -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D COMPILER_RT_BUILD_ORC=OFF \ + -D CMAKE_C_COMPILER_LAUNCHER=sccache \ + -D CMAKE_CXX_COMPILER_LAUNCHER=sccache + +echo "--- ninja" +ninja -C ${BUILD_DIR} ${targets} Index: .ci/monolithic-linux.sh =================================================================== --- /dev/null +++ .ci/monolithic-linux.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This script performs a monolithic build of the monorepo and runs the tests of +# most projects on Linux. This should be replaced by per-project scripts that +# run only the relevant tests. +# + +set -ex +set -o pipefail + +MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" +BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-linux}" + +rm -rf ${BUILD_DIR} + +ccache --zero-stats +ccache --show-config +function show-stats { + ccache --print-stats +} +trap show-stats EXIT + +projects="${1}" +targets="${2}" + +echo "--- cmake" +pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ + -D LLVM_ENABLE_PROJECTS="${projects}" \ + -G Ninja \ + -D CMAKE_BUILD_TYPE=Release \ + -D LLVM_ENABLE_ASSERTIONS=ON \ + -D LLVM_BUILD_EXAMPLES=ON \ + -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D LLVM_ENABLE_LLD=ON \ + -D CMAKE_CXX_FLAGS=-gmlt \ + -D BOLT_CLANG_EXE=/usr/bin/clang \ + -D LLVM_CCACHE_BUILD=ON + +echo "--- ninja" +ninja -C ${BUILD_DIR} ${targets} Index: .ci/generate-buildkite-pipeline-premerge =================================================================== --- .ci/generate-buildkite-pipeline-premerge +++ .ci/generate-buildkite-pipeline-premerge @@ -14,11 +14,6 @@ # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. # -if ! git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/|^clang/"; then - # libcxx/, libcxxabi/, libunwind/, runtimes/, cmake/ or clang/ are not affected - exit 0 -fi - reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')" if [[ "${reviewID}" != "" ]]; then buildMessage="https://llvm.org/${reviewID}"; @@ -32,6 +27,7 @@ # If libc++ or one of the runtimes directories changed if git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/"; then + SPECIFIC_PIPELINE_AVAILABLE=1 cat <<EOF - trigger: "libcxx-ci" build: @@ -43,6 +39,7 @@ # If clang changed if git diff --name-only HEAD~1 | grep -q -E "^clang/"; then + SPECIFIC_PIPELINE_AVAILABLE=1 cat <<EOF - trigger: "clang-ci" build: @@ -51,3 +48,177 @@ branch: "${BUILDKITE_BRANCH}" EOF fi + +# +# If we're not running a more specific pipeline, generate a legacy monolithic pipeline +# based on which subdirectories have been modified. We have some heuristics for this +# to be reasonable. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. +# +function add-dependencies() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + bolt) + echo "llvm lld" + ;; + clang-tools-extra) + echo "clang llvm" + ;; + compiler-rt) + echo "clang" + ;; + flang) + echo "llvm mlir clang" + ;; + libc) + echo "clang lldb" + ;; + lld) + echo "llvm" + ;; + mlir) + echo "llvm" + ;; + polly) + echo "llvm" + ;; + *) + # Nothing to do + ;; + esac + done +} + +function exclude-linux() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + lldb) ;; # tests failing + openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 + *) + echo "${project}" + ;; + esac + done +} + +function exclude-windows() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + compiler-rt) ;; # tests taking too long + openmp) ;; # TODO: having trouble with the Perl installation + libc) ;; # no Windows support + lldb) ;; # tests failing + bolt) ;; # tests are not supported yet + *) + echo "${project}" + ;; + esac + done +} + +function keep-modified-projects() { + projects=${@} + git_diff="$(git diff --name-only HEAD~1)" + for project in ${projects}; do + if echo "${git_diff}" | grep -q -E "^${project}/"; then + echo "${project}" + fi + done +} + +function check-targets() { + projects=${@} + for project in ${projects}; do + case ${project} in + clang-tools-extra) + echo "check-clang-tools" + ;; + compiler-rt) + echo "check-all" + ;; + cross-project-tests) + echo "check-cross-project" + ;; + lldb) + echo "check-all" # TODO: check-lldb may not include all the LLDB tests? + ;; + pstl) + echo "check-all" + ;; + *) + echo "check-${project}" + ;; + esac + done +} + +if [[ ! ${SPECIFIC_PIPELINE_AVAILABLE} -eq 1 ]]; then + # Figure out which projects need to be built on each platform + all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm llvm-libgcc mlir openmp polly pstl" + modified_projects="$(keep-modified-projects ${all_projects})" + + linux_projects=$(add-dependencies $(exclude-linux ${modified_projects}) | sort | uniq) + linux_check_targets=$(check-targets ${linux_projects} | sort | uniq) + + windows_projects=$(add-dependencies $(exclude-windows ${modified_projects}) | sort | uniq) + windows_check_targets=$(check-targets ${windows_projects} | sort | uniq) + + # Generate the appropriate pipeline + if [[ "${linux_projects}" != "" ]]; then + cat <<EOF + - label: ':linux: x64 Debian' + artifact_paths: + - '*_result.json' + - 'build/monolithic-linux/test-results.xml' + agents: + queue: 'linux' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 120 + env: + CC: 'clang' + CXX: 'clang++' + commands: + - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"' +EOF + fi + + echo # Newline between the two steps + + if [[ "${windows_projects}" != "" ]]; then + cat <<EOF + - label: ':windows: x64 Windows' + artifact_paths: + - '*_result.json' + - 'build/monolithic-windows/test-results.xml' + agents: + queue: 'windows' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 150 + env: + CC: 'cl' + CXX: 'cl' + LD: 'link' + commands: + - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' + - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"' +EOF + fi +fi
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
