https://github.com/llvmbot created https://github.com/llvm/llvm-project/pull/149978
Backport 5004c59803fdeb3389d30439a6cc8b1ff874df0c edfec9c5829f4778bd38f3ca4019105eb481df49 Requested by: @bd1976bris >From 68fbf3cbb909497a75aa91072fdb88c0405c5cb9 Mon Sep 17 00:00:00 2001 From: bd1976bris <ben.dunbob...@sony.com> Date: Tue, 15 Jul 2025 15:06:46 +0100 Subject: [PATCH 1/2] [DTLTO][Clang] Add support for Integrated Distributed ThinLTO (#147265) This patch introduces support for Integrated Distributed ThinLTO (DTLTO) in Clang. DTLTO enables the distribution of ThinLTO backend compilations via external distribution systems, such as Incredibuild, during the traditional link step: https://llvm.org/docs/DTLTO.html. Testing: - `lit` test coverage has been added to Clang's Driver tests. - The DTLTO cross-project tests will use this Clang support. For the design discussion of the DTLTO feature, see: https://github.com/llvm/llvm-project/pull/126654 (cherry picked from commit 5004c59803fdeb3389d30439a6cc8b1ff874df0c) --- clang/docs/ThinLTO.rst | 32 +++++++++++++++ clang/include/clang/Driver/Options.td | 14 ++++++- clang/lib/Driver/ToolChains/CommonArgs.cpp | 11 +++++ clang/test/Driver/DTLTO/dtlto.c | 48 ++++++++++++++++++++++ cross-project-tests/dtlto/ld-dtlto.c | 31 ++++++-------- 5 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 clang/test/Driver/DTLTO/dtlto.c diff --git a/clang/docs/ThinLTO.rst b/clang/docs/ThinLTO.rst index c042547678919..569405ff5b2b7 100644 --- a/clang/docs/ThinLTO.rst +++ b/clang/docs/ThinLTO.rst @@ -240,6 +240,38 @@ The ``BOOTSTRAP_LLVM_ENABLE_LTO=Thin`` will enable ThinLTO for stage 2 and stage 3 in case the compiler used for stage 1 does not support the ThinLTO option. +Integrated Distributed ThinLTO (DTLTO) +-------------------------------------- + +Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend +ThinLTO compilations via external distribution systems, such as Incredibuild, +during the traditional link step. + +The implementation is documented here: https://llvm.org/docs/DTLTO.html. + +DTLTO requires the LLD linker (``-fuse-ld=lld``). + +``-fthinlto-distributor=<path>`` + - Specifies the ``<path>`` to the distributor process executable for DTLTO. + - If specified, ThinLTO backend compilations will be distributed by LLD. + +``-Xthinlto-distributor=<arg>`` + - Passes ``<arg>`` to the distributor process (see ``-fthinlto-distributor=``). + - Can be specified multiple times to pass multiple options. + - Multiple options can also be specified by separating them with commas. + +Examples: + - ``clang -flto=thin -fthinlto-distributor=incredibuild.exe -Xthinlto-distributor=--verbose,--j10 -fuse-ld=lld`` + - ``clang -flto=thin -fthinlto-distributor=$(which python) -Xthinlto-distributor=incredibuild.py -fuse-ld=lld`` + +If ``-fthinlto-distributor=`` is specified, Clang supplies the path to a +compiler to be executed remotely to perform the ThinLTO backend +compilations. Currently, this is Clang itself. + +Note that currently, DTLTO is only supported in some LLD flavors. Support can +be added to other LLD flavours in the future. +See `DTLTO <https://lld.llvm.org/dtlto.html>`_ for more information. + More Information ================ diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index e7c7e9d93fe21..bce29a76f3ac7 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -990,6 +990,13 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>, Visibility<[ClangOption, CLOption, FlangOption]>, HelpText<"Pass <arg> to the linker">, MetaVarName<"<arg>">, Group<Link_Group>; +def Xthinlto_distributor_EQ : CommaJoined<["-"], "Xthinlto-distributor=">, + Flags<[LinkOption]>, + Visibility<[ClangOption, CLOption]>, + HelpText<"Pass <arg> to the ThinLTO distributor process. Can be specified " + "multiple times or with comma-separated values.">, + MetaVarName<"<arg>">, + Group<Link_Group>; def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">, Visibility<[ClangOption, FlangOption]>, HelpText<"Pass <arg> to the offload linkers or the ones identified by -<triple>">, @@ -4249,7 +4256,12 @@ def ffinite_loops: Flag<["-"], "ffinite-loops">, Group<f_Group>, def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group<f_Group>, HelpText<"Do not assume that any loop is finite.">, Visibility<[ClangOption, CC1Option]>; - +def fthinlto_distributor_EQ : Joined<["-"], "fthinlto-distributor=">, + Group<f_Group>, + HelpText<"Path to the ThinLTO distributor process. If specified, " + "ThinLTO backend compilations will be distributed by LLD">, + MetaVarName<"<path>">, + Visibility<[ClangOption, CLOption]>; def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>, HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>; def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>, diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 069fd877bd7f8..826e2ea7eb06d 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1320,6 +1320,17 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, if (Args.hasArg(options::OPT_ftime_report)) CmdArgs.push_back( Args.MakeArgString(Twine(PluginOptPrefix) + "-time-passes")); + + if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) { + CmdArgs.push_back( + Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue()))); + CmdArgs.push_back( + Args.MakeArgString("--thinlto-remote-compiler=" + + Twine(ToolChain.getDriver().getClangProgramPath()))); + + for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ)) + CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A)); + } } void tools::addOpenMPRuntimeLibraryPath(const ToolChain &TC, diff --git a/clang/test/Driver/DTLTO/dtlto.c b/clang/test/Driver/DTLTO/dtlto.c new file mode 100644 index 0000000000000..d72e487e706b2 --- /dev/null +++ b/clang/test/Driver/DTLTO/dtlto.c @@ -0,0 +1,48 @@ +// REQUIRES: lld + +/// Check DTLTO options are forwarded to the linker. + +/// Check that options are forwarded as expected with --thinlto-distributor=. +// RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -Xthinlto-distributor=a1 -Xthinlto-distributor=a2,a3 \ +// RUN: -fthinlto-distributor=d.exe -Werror 2>&1 | \ +// RUN: FileCheck %s --check-prefix=FORWARD + +// FORWARD: ld.lld +// FORWARD-SAME: "--thinlto-distributor=d.exe" +// FORWARD-SAME: "--thinlto-remote-compiler={{.*}}clang{{[^\"]*}}" +// FORWARD-SAME: "--thinlto-distributor-arg=a1" +// FORWARD-SAME: "--thinlto-distributor-arg=a2" +// FORWARD-SAME: "--thinlto-distributor-arg=a3" + +/// Check that options are not added without --thinlto-distributor= and +/// that a warning is issued for unused -Xthinlto-distributor options. +// RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -Xthinlto-distributor=a1 -Xthinlto-distributor=a2,a3 2>&1 | \ +// RUN: FileCheck %s --check-prefix=NODIST --implicit-check-not=distributor \ +// RUN: --implicit-check-not=remote-compiler + +// NODIST: warning: argument unused during compilation: '-Xthinlto-distributor=a1' +// NODIST: warning: argument unused during compilation: '-Xthinlto-distributor=a2,a3' +// NODIST: ld.lld + +/// Check the expected arguments are forwarded by default with only +/// --thinlto-distributor=. +// RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -fthinlto-distributor=d.exe -Werror 2>&1 | \ +// RUN: FileCheck %s --check-prefix=DEFAULT --implicit-check-not=distributor \ +// RUN: --implicit-check-not=remote-compiler + +// DEFAULT: ld.lld +// DEFAULT-SAME: "--thinlto-distributor=d.exe" +// DEFAULT-SAME: "--thinlto-remote-compiler={{.*}}clang{{[^\"]*}}" + +/// Check that nothing is forwarded when the compiler is not in LTO mode, and that +/// appropriate unused option warnings are issued. +// RUN: %clang %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -fthinlto-distributor=d.exe 2>&1 | \ +// RUN: FileCheck %s --check-prefix=NOFLTO --implicit-check-not=distributor \ +// RUN: --implicit-check-not=remote-compiler + +// NOFLTO: warning: argument unused during compilation: '-fthinlto-distributor=d.exe' +// NOFLTO: ld.lld diff --git a/cross-project-tests/dtlto/ld-dtlto.c b/cross-project-tests/dtlto/ld-dtlto.c index 3ee962346bd4a..7dffe2e015bcb 100644 --- a/cross-project-tests/dtlto/ld-dtlto.c +++ b/cross-project-tests/dtlto/ld-dtlto.c @@ -5,13 +5,11 @@ // RUN: rm -rf %t && mkdir %t && cd %t -// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o - -// RUN: ld.lld dtlto.o \ -// RUN: --thinlto-distributor=%python \ -// RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ -// RUN: --thinlto-remote-compiler=%clang \ -// RUN: --thinlto-remote-compiler-arg=--save-temps +// RUN: %clang --target=x86_64-linux-gnu %s -flto=thin -fuse-ld=lld \ +// RUN: -fthinlto-distributor=%python \ +// RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \ +// RUN: -Wl,--thinlto-remote-compiler-arg=--save-temps \ +// RUN: -nostdlib -Werror /// Check that the required output files have been created. // RUN: ls | sort | FileCheck %s @@ -22,18 +20,15 @@ /// Linked ELF. // CHECK: {{^}}a.out{{$}} -/// Produced by the bitcode compilation. -// CHECK-NEXT: {{^}}dtlto.o{{$}} - /// --save-temps output for the backend compilation. -// CHECK-NEXT: {{^}}dtlto.s{{$}} -// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP:[a-zA-Z0-9_]+]].s{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.0.preopt.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.1.promote.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.2.internalize.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.3.import.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.4.opt.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.5.precodegen.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.resolution.txt{{$}} /// No files are expected after. // CHECK-NOT: {{.}} >From 71dc819fa112371cf1ebc8dfd33763e06d823576 Mon Sep 17 00:00:00 2001 From: bd1976bris <ben.dunbob...@sony.com> Date: Tue, 15 Jul 2025 19:13:07 +0100 Subject: [PATCH 2/2] [DTLTO][TEST] Fix Clang driver test failing on some build bots (#148908) Not all builds name the compiler executable `clang`. For example, the Fuchsia build bots use `llvm` as their single toolchain executable name, as they combine everything together in a busybot-style binary. Update the Clang driver test to simply check that a non-empty path is provided for the `--thinlto-remote-compiler` argument, rather than hardcoding the executable name. The cross-project test will verify that the path is valid later. Should fix #147265. (cherry picked from commit edfec9c5829f4778bd38f3ca4019105eb481df49) --- clang/test/Driver/DTLTO/dtlto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/Driver/DTLTO/dtlto.c b/clang/test/Driver/DTLTO/dtlto.c index d72e487e706b2..96795d9a4e6a4 100644 --- a/clang/test/Driver/DTLTO/dtlto.c +++ b/clang/test/Driver/DTLTO/dtlto.c @@ -10,7 +10,7 @@ // FORWARD: ld.lld // FORWARD-SAME: "--thinlto-distributor=d.exe" -// FORWARD-SAME: "--thinlto-remote-compiler={{.*}}clang{{[^\"]*}}" +// FORWARD-SAME: "--thinlto-remote-compiler={{[^"]+}}" // FORWARD-SAME: "--thinlto-distributor-arg=a1" // FORWARD-SAME: "--thinlto-distributor-arg=a2" // FORWARD-SAME: "--thinlto-distributor-arg=a3" _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
