https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/120777
>From cb87398ea7f654559799ff7f79f292842832c29c Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 20 Nov 2024 20:59:38 -0800 Subject: [PATCH 01/12] Workflows: Use new depot runners for x86 Linux release builds --- .github/workflows/release-binaries.yml | 89 +++++++++++++++++++++----- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 1cde628d3f66c3..eeb92f2e039b98 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -60,6 +60,8 @@ jobs: enable-pgo: ${{ steps.vars.outputs.enable-pgo }} release-binary-basename: ${{ steps.vars.outputs.release-binary-basename }} release-binary-filename: ${{ steps.vars.outputs.release-binary-filename }} + runs-on: ${{ steps.vars.outputs.runs-on }} + multi-stage: ${{ steps.vars.outputs.multi-stage }} steps: # It's good practice to use setup-python, but this is also required on macos-14 @@ -144,12 +146,26 @@ jobs: echo "target-cmake-flags=$target_cmake_flags" >> $GITHUB_OUTPUT echo "build-flang=$build_flang" >> $GITHUB_OUTPUT + case "${{ inputs.runs-on }}" in + ubuntu-22.04) + runs_on="depot-${{ inputs.runs-on }}-16" + multi_stage="false" + ;; + *) + runs_on="${{ inputs.runs-on }}" + multi_stage="true" + ;; + esac + echo "runs-on=$runs_on" >> $GITHUB_OUTPUT + echo "multi-stage=$multi_stage" >> $GITHUB_OUTPUT build-stage1: name: "Build Stage 1" needs: prepare - if: github.repository == 'llvm/llvm-project' - runs-on: ${{ inputs.runs-on }} + if: >- + github.repository == 'llvm/llvm-project' && + needs.prepare.outputs.multi-stage == 'true' + runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions @@ -195,7 +211,7 @@ jobs: key: sccache-${{ runner.os }}-${{ runner.arch }}-release variant: sccache - - name: Build Stage 1 Clang + - name: Configure Stage 1 Clang id: build shell: bash run: | @@ -208,12 +224,44 @@ jobs: -DBOOTSTRAP_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}" \ -DCMAKE_C_COMPILER_LAUNCHER=sccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build - # There is a race condition on the MacOS builders and this command is here - # to help debug that when it happens. - ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build + - name: Build Stage 1 Clang + shell: bash + run: | + if "${{ steps.needs.prepare.outputs.multi-stage}}" = "true"; then + ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package + mv ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} . + else + ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build + # There is a race condition on the MacOS builders and this command is here + # to help debug that when it happens. + ls -ltr ${{ steps.setup-stage.outputs.build-prefix }}/build + fi + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: needs.prepare.outputs.multi-stage == "false" + with: + name: ${{ runner.os }}-${{ runner.arch }}-release-binary + # Due to path differences on Windows when running in bash vs running on node, + # we need to search for files in the current workspace. + path: | + ${{ needs.prepare.outputs.release-binary-filename }} + + # Clean up some build files to reduce size of artifact. + - name: Clean Up Build Directory + if: needs.prepare.outputs.multi-stage == "false" + shell: bash + run: | + find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname ${{ needs.prepare.outputs.release-binary-filename }} -delete + rm -Rf ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/_CPack_Packages + - name: Run Tests + shell: bash + if: needs.prepare.outputs.multi-stage == "false" + run: | + ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-check-all + - name: Save Stage + if: needs.prepare.outputs.multi-stage == "true" uses: ./workflows-main/.github/workflows/release-binaries-save-stage with: build-prefix: ${{ steps.setup-stage.outputs.build-prefix }} @@ -223,8 +271,10 @@ jobs: needs: - prepare - build-stage1 - if: github.repository == 'llvm/llvm-project' - runs-on: ${{ inputs.runs-on }} + if: >- + github.repository == 'llvm/llvm-project' && + needs.prepare.outputs.multi-stage == 'true' + runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -242,7 +292,9 @@ jobs: - name: Build Stage 2 # Re-enable once PGO builds are supported. - if: needs.prepare.outputs.enable-pgo == 'true' + if: >- + needs.prepare.outputs.enable-pgo == 'true' && + needs.prepare.outputs.multi-stage == 'true' shell: bash run: | ninja -C ${{ steps.setup-stage.outputs.build-prefix}}/build stage2-instrumented @@ -257,8 +309,10 @@ jobs: needs: - prepare - build-stage2 - if: github.repository == 'llvm/llvm-project' - runs-on: ${{ inputs.runs-on }} + if: >- + github.repository == 'llvm/llvm-project' && + needs.prepare.outputs.multi-stage == 'true' + runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -307,7 +361,9 @@ jobs: needs: - prepare - build-stage3-clang - runs-on: ${{ inputs.runs-on }} + if: >- + needs.prepare.outputs.multi-stage == 'true' + runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -357,7 +413,7 @@ jobs: needs: - prepare - build-stage3-flang - runs-on: ${{ inputs.runs-on }} + runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -409,6 +465,7 @@ jobs: needs: - prepare - build-stage3-all + - build-stage1 if: >- always() && github.event_name != 'pull_request' && @@ -469,8 +526,8 @@ jobs: - prepare - build-stage3-all if: >- - github.repository == 'llvm/llvm-project' - runs-on: ${{ inputs.runs-on }} + github.repository == 'llvm/llvm-project' && + runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 >From e0373d6b57b6a8ce2bb11ab54ea699311dbe88bf Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 20 Nov 2024 21:10:48 -0800 Subject: [PATCH 02/12] Fix up some typos --- .github/workflows/release-binaries.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index eeb92f2e039b98..82e9faaa0333fb 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -227,7 +227,7 @@ jobs: - name: Build Stage 1 Clang shell: bash run: | - if "${{ steps.needs.prepare.outputs.multi-stage}}" = "true"; then + if [ "${{ steps.needs.prepare.outputs.multi-stage}}" = "true" ]; then ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package mv ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} . else @@ -238,7 +238,7 @@ jobs: fi - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - if: needs.prepare.outputs.multi-stage == "false" + if: needs.prepare.outputs.multi-stage == 'false' with: name: ${{ runner.os }}-${{ runner.arch }}-release-binary # Due to path differences on Windows when running in bash vs running on node, @@ -248,7 +248,7 @@ jobs: # Clean up some build files to reduce size of artifact. - name: Clean Up Build Directory - if: needs.prepare.outputs.multi-stage == "false" + if: needs.prepare.outputs.multi-stage == 'false' shell: bash run: | find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname ${{ needs.prepare.outputs.release-binary-filename }} -delete @@ -256,12 +256,12 @@ jobs: - name: Run Tests shell: bash - if: needs.prepare.outputs.multi-stage == "false" + if: needs.prepare.outputs.multi-stage == 'false' run: | ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-check-all - name: Save Stage - if: needs.prepare.outputs.multi-stage == "true" + if: needs.prepare.outputs.multi-stage == 'true' uses: ./workflows-main/.github/workflows/release-binaries-save-stage with: build-prefix: ${{ steps.setup-stage.outputs.build-prefix }} >From 6568444066258c9873edf484dbd6793bc847087a Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 20 Nov 2024 21:11:46 -0800 Subject: [PATCH 03/12] Fix another typo --- .github/workflows/release-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 82e9faaa0333fb..8e9bd5b9a7449d 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -526,7 +526,7 @@ jobs: - prepare - build-stage3-all if: >- - github.repository == 'llvm/llvm-project' && + github.repository == 'llvm/llvm-project' runs-on: ${{ needs.prepare.outputs.runs-on }} steps: - name: Checkout Actions >From 1702aa6534b32dd8af52dbf2d0f1e9615ee4361f Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 20 Nov 2024 21:14:17 -0800 Subject: [PATCH 04/12] Fix wrong if condition --- .github/workflows/release-binaries.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 8e9bd5b9a7449d..dc7bf5906da029 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -163,8 +163,7 @@ jobs: name: "Build Stage 1" needs: prepare if: >- - github.repository == 'llvm/llvm-project' && - needs.prepare.outputs.multi-stage == 'true' + github.repository == 'llvm/llvm-project' runs-on: ${{ needs.prepare.outputs.runs-on }} steps: >From 72ff71f64661243c6782f63c1a64f6d7084f9afe Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 20 Nov 2024 21:18:37 -0800 Subject: [PATCH 05/12] Fix multi-stage condition --- .github/workflows/release-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index dc7bf5906da029..bd96d2b27f6044 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -226,7 +226,7 @@ jobs: - name: Build Stage 1 Clang shell: bash run: | - if [ "${{ steps.needs.prepare.outputs.multi-stage}}" = "true" ]; then + if [ "${{ steps.needs.prepare.outputs.multi-stage}}" = "false" ]; then ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package mv ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} . else >From e2ac546d0a5fc6a4b34f4797fec8b10b96e4dcba Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 20 Nov 2024 21:39:17 -0800 Subject: [PATCH 06/12] Fix variable name --- .github/workflows/release-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index bd96d2b27f6044..790817103db3f0 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -226,7 +226,7 @@ jobs: - name: Build Stage 1 Clang shell: bash run: | - if [ "${{ steps.needs.prepare.outputs.multi-stage}}" = "false" ]; then + if [ "${{ needs.prepare.outputs.multi-stage}}" = "false" ]; then ninja -v -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-package mv ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/${{ needs.prepare.outputs.release-binary-filename }} . else >From 137ed986d4995291959f0d58594a9ea0c6af2023 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Thu, 21 Nov 2024 07:01:54 -0800 Subject: [PATCH 07/12] Split out tests job --- .github/workflows/release-binaries.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 790817103db3f0..0ed4546920f118 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -253,14 +253,7 @@ jobs: find ${{ steps.setup-stage.outputs.build-prefix }}/build -iname ${{ needs.prepare.outputs.release-binary-filename }} -delete rm -Rf ${{ steps.setup-stage.outputs.build-prefix }}/build/tools/clang/stage2-bins/_CPack_Packages - - name: Run Tests - shell: bash - if: needs.prepare.outputs.multi-stage == 'false' - run: | - ninja -C ${{ steps.setup-stage.outputs.build-prefix }}/build stage2-check-all - - name: Save Stage - if: needs.prepare.outputs.multi-stage == 'true' uses: ./workflows-main/.github/workflows/release-binaries-save-stage with: build-prefix: ${{ steps.setup-stage.outputs.build-prefix }} @@ -525,8 +518,9 @@ jobs: - prepare - build-stage3-all if: >- + always() && github.repository == 'llvm/llvm-project' - runs-on: ${{ needs.prepare.outputs.runs-on }} + runs-on: ${{ input.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 @@ -540,7 +534,7 @@ jobs: id: setup-stage uses: ./workflows/.github/workflows/release-binaries-setup-stage with: - previous-artifact: build-stage3-all + previous-artifact: ${{ (needs.prepare.outputs.multi-stage == 'false' && 'build-stage1') || 'build-stage3-all' }} - name: Run Tests shell: bash >From 605e497f79aad69c3856a6f071b45a1b6a4aa574 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Thu, 21 Nov 2024 07:06:03 -0800 Subject: [PATCH 08/12] Fix typo --- .github/workflows/release-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 0ed4546920f118..9c9b947a7014bc 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -520,7 +520,7 @@ jobs: if: >- always() && github.repository == 'llvm/llvm-project' - runs-on: ${{ input.runs-on }} + runs-on: ${{ inputs.runs-on }} steps: - name: Checkout Actions uses: actions/checkout@v4 >From c60000d63df59b545f2cb28e3431e9e2cad8b59c Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Fri, 22 Nov 2024 09:30:47 -0800 Subject: [PATCH 09/12] Add sccache to test stage --- .github/workflows/release-binaries.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 9c9b947a7014bc..7a3f6870ebe291 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -536,6 +536,16 @@ jobs: with: previous-artifact: ${{ (needs.prepare.outputs.multi-stage == 'false' && 'build-stage1') || 'build-stage3-all' }} + # Need sccache installed, because some stage1 objects are being built for the tests. + # FIXME: This probably shouldn't be happening. + - name: Setup sccache + uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9 + with: + # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174 + max-size: 2G + key: sccache-${{ runner.os }}-${{ runner.arch }}-release + variant: sccache + - name: Run Tests shell: bash run: | >From 76675cb5e666b4f365755f1b8042ff96e771f7e9 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Fri, 20 Dec 2024 09:29:08 -0800 Subject: [PATCH 10/12] linux-release-pgo --- .github/workflows/release-binaries.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 7a3f6870ebe291..1fcf57814374c0 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -122,7 +122,11 @@ jobs: # Detect necessary CMake flags target="${{ runner.os }}-${{ runner.arch }}" - echo "enable-pgo=false" >> $GITHUB_OUTPUT + if [ "${{ runner.os }}" = "Linux" ]; then + echo "enable-pgo=true" >> $GITHUB_OUTPUT + else + echo "enable-pgo=false" >> $GITHUB_OUTPUT + fi target_cmake_flags="-DLLVM_RELEASE_ENABLE_PGO=OFF" # The macOS builds try to cross compile some libraries so we need to # add extra CMake args to disable them. >From 2fca06377d0a972ac0f5dbc21a25107058a70e6f Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Fri, 20 Dec 2024 09:30:26 -0800 Subject: [PATCH 11/12] Disable pgo --- .github/workflows/release-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 1fcf57814374c0..eceb0685935807 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -126,8 +126,8 @@ jobs: echo "enable-pgo=true" >> $GITHUB_OUTPUT else echo "enable-pgo=false" >> $GITHUB_OUTPUT + target_cmake_flags="-DLLVM_RELEASE_ENABLE_PGO=OFF" fi - target_cmake_flags="-DLLVM_RELEASE_ENABLE_PGO=OFF" # The macOS builds try to cross compile some libraries so we need to # add extra CMake args to disable them. # See https://github.com/llvm/llvm-project/issues/99767 >From f94970660bffe28e02759a31ad023e271ee57b92 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Sat, 21 Dec 2024 07:54:10 -0800 Subject: [PATCH 12/12] Fix package name --- .github/workflows/release-binaries.yml | 2 +- clang/cmake/caches/Release.cmake | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index eceb0685935807..67c8646f091ea1 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -222,9 +222,9 @@ jobs: # so we need to set some extra cmake flags to disable this. cmake -G Ninja -S llvm -B ${{ steps.setup-stage.outputs.build-prefix }}/build \ ${{ needs.prepare.outputs.target-cmake-flags }} \ + -DLLVM_RELEASE_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}" \ -C clang/cmake/caches/Release.cmake \ -DBOOTSTRAP_LLVM_PARALLEL_LINK_JOBS=1 \ - -DBOOTSTRAP_CPACK_PACKAGE_FILE_NAME="${{ needs.prepare.outputs.release-binary-basename }}" \ -DCMAKE_C_COMPILER_LAUNCHER=sccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - name: Build Stage 1 Clang diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake index 23e99493087ff0..f67eaa8664a909 100644 --- a/clang/cmake/caches/Release.cmake +++ b/clang/cmake/caches/Release.cmake @@ -40,6 +40,7 @@ set(LLVM_RELEASE_ENABLE_PROJECTS "clang;lld;lldb;clang-tools-extra;bolt;polly;ml # Note we don't need to add install here, since it is one of the pre-defined # steps. set(LLVM_RELEASE_FINAL_STAGE_TARGETS "clang;package;check-all;check-llvm;check-clang" CACHE STRING "") +set(LLVM_RELEASE_CPACK_PACKAGE_FILE_NAME "" CACHE STRING "") set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") # Stage 1 Options @@ -108,6 +109,8 @@ set_final_stage_var(LLVM_ENABLE_RUNTIMES "${LLVM_RELEASE_ENABLE_RUNTIMES}" STRIN set_final_stage_var(LLVM_ENABLE_PROJECTS "${LLVM_RELEASE_ENABLE_PROJECTS}" STRING) set_final_stage_var(CPACK_GENERATOR "TXZ" STRING) set_final_stage_var(CPACK_ARCHIVE_THREADS "0" STRING) +if (LLVM_RELEASE_CPACK_PACKAGE_FILE_NAME) + set_final_stage_var(CPACK_PACKAGE_FILE_NAME "${LLVM_RELEASE_CPACK_PACKAGE_FILE_NAME}") if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin") set_final_stage_var(LLVM_USE_STATIC_ZSTD "ON" BOOL) _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits