llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang @llvm/pr-subscribers-clang-driver Author: None (llvmbot) <details> <summary>Changes</summary> Backport 5004c59803fdeb3389d30439a6cc8b1ff874df0c edfec9c5829f4778bd38f3ca4019105eb481df49 Requested by: @<!-- -->bd1976bris --- Full diff: https://github.com/llvm/llvm-project/pull/149978.diff 5 Files Affected: - (modified) clang/docs/ThinLTO.rst (+32) - (modified) clang/include/clang/Driver/Options.td (+13-1) - (modified) clang/lib/Driver/ToolChains/CommonArgs.cpp (+11) - (added) clang/test/Driver/DTLTO/dtlto.c (+48) - (modified) cross-project-tests/dtlto/ld-dtlto.c (+13-18) ``````````diff diff --git a/clang/docs/ThinLTO.rst b/clang/docs/ThinLTO.rst index c042547678919..569405ff5b2b7 100644 --- a/clang/docs/ThinLTO.rst +++ b/clang/docs/ThinLTO.rst @@ -240,6 +240,38 @@ The ``BOOTSTRAP_LLVM_ENABLE_LTO=Thin`` will enable ThinLTO for stage 2 and stage 3 in case the compiler used for stage 1 does not support the ThinLTO option. +Integrated Distributed ThinLTO (DTLTO) +-------------------------------------- + +Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend +ThinLTO compilations via external distribution systems, such as Incredibuild, +during the traditional link step. + +The implementation is documented here: https://llvm.org/docs/DTLTO.html. + +DTLTO requires the LLD linker (``-fuse-ld=lld``). + +``-fthinlto-distributor=<path>`` + - Specifies the ``<path>`` to the distributor process executable for DTLTO. + - If specified, ThinLTO backend compilations will be distributed by LLD. + +``-Xthinlto-distributor=<arg>`` + - Passes ``<arg>`` to the distributor process (see ``-fthinlto-distributor=``). + - Can be specified multiple times to pass multiple options. + - Multiple options can also be specified by separating them with commas. + +Examples: + - ``clang -flto=thin -fthinlto-distributor=incredibuild.exe -Xthinlto-distributor=--verbose,--j10 -fuse-ld=lld`` + - ``clang -flto=thin -fthinlto-distributor=$(which python) -Xthinlto-distributor=incredibuild.py -fuse-ld=lld`` + +If ``-fthinlto-distributor=`` is specified, Clang supplies the path to a +compiler to be executed remotely to perform the ThinLTO backend +compilations. Currently, this is Clang itself. + +Note that currently, DTLTO is only supported in some LLD flavors. Support can +be added to other LLD flavours in the future. +See `DTLTO <https://lld.llvm.org/dtlto.html>`_ for more information. + More Information ================ diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index e7c7e9d93fe21..bce29a76f3ac7 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -990,6 +990,13 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>, Visibility<[ClangOption, CLOption, FlangOption]>, HelpText<"Pass <arg> to the linker">, MetaVarName<"<arg>">, Group<Link_Group>; +def Xthinlto_distributor_EQ : CommaJoined<["-"], "Xthinlto-distributor=">, + Flags<[LinkOption]>, + Visibility<[ClangOption, CLOption]>, + HelpText<"Pass <arg> to the ThinLTO distributor process. Can be specified " + "multiple times or with comma-separated values.">, + MetaVarName<"<arg>">, + Group<Link_Group>; def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">, Visibility<[ClangOption, FlangOption]>, HelpText<"Pass <arg> to the offload linkers or the ones identified by -<triple>">, @@ -4249,7 +4256,12 @@ def ffinite_loops: Flag<["-"], "ffinite-loops">, Group<f_Group>, def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group<f_Group>, HelpText<"Do not assume that any loop is finite.">, Visibility<[ClangOption, CC1Option]>; - +def fthinlto_distributor_EQ : Joined<["-"], "fthinlto-distributor=">, + Group<f_Group>, + HelpText<"Path to the ThinLTO distributor process. If specified, " + "ThinLTO backend compilations will be distributed by LLD">, + MetaVarName<"<path>">, + Visibility<[ClangOption, CLOption]>; def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>, HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>; def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>, diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 069fd877bd7f8..826e2ea7eb06d 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1320,6 +1320,17 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, if (Args.hasArg(options::OPT_ftime_report)) CmdArgs.push_back( Args.MakeArgString(Twine(PluginOptPrefix) + "-time-passes")); + + if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) { + CmdArgs.push_back( + Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue()))); + CmdArgs.push_back( + Args.MakeArgString("--thinlto-remote-compiler=" + + Twine(ToolChain.getDriver().getClangProgramPath()))); + + for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ)) + CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A)); + } } void tools::addOpenMPRuntimeLibraryPath(const ToolChain &TC, diff --git a/clang/test/Driver/DTLTO/dtlto.c b/clang/test/Driver/DTLTO/dtlto.c new file mode 100644 index 0000000000000..96795d9a4e6a4 --- /dev/null +++ b/clang/test/Driver/DTLTO/dtlto.c @@ -0,0 +1,48 @@ +// REQUIRES: lld + +/// Check DTLTO options are forwarded to the linker. + +/// Check that options are forwarded as expected with --thinlto-distributor=. +// RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -Xthinlto-distributor=a1 -Xthinlto-distributor=a2,a3 \ +// RUN: -fthinlto-distributor=d.exe -Werror 2>&1 | \ +// RUN: FileCheck %s --check-prefix=FORWARD + +// FORWARD: ld.lld +// FORWARD-SAME: "--thinlto-distributor=d.exe" +// FORWARD-SAME: "--thinlto-remote-compiler={{[^"]+}}" +// FORWARD-SAME: "--thinlto-distributor-arg=a1" +// FORWARD-SAME: "--thinlto-distributor-arg=a2" +// FORWARD-SAME: "--thinlto-distributor-arg=a3" + +/// Check that options are not added without --thinlto-distributor= and +/// that a warning is issued for unused -Xthinlto-distributor options. +// RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -Xthinlto-distributor=a1 -Xthinlto-distributor=a2,a3 2>&1 | \ +// RUN: FileCheck %s --check-prefix=NODIST --implicit-check-not=distributor \ +// RUN: --implicit-check-not=remote-compiler + +// NODIST: warning: argument unused during compilation: '-Xthinlto-distributor=a1' +// NODIST: warning: argument unused during compilation: '-Xthinlto-distributor=a2,a3' +// NODIST: ld.lld + +/// Check the expected arguments are forwarded by default with only +/// --thinlto-distributor=. +// RUN: %clang -flto=thin %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -fthinlto-distributor=d.exe -Werror 2>&1 | \ +// RUN: FileCheck %s --check-prefix=DEFAULT --implicit-check-not=distributor \ +// RUN: --implicit-check-not=remote-compiler + +// DEFAULT: ld.lld +// DEFAULT-SAME: "--thinlto-distributor=d.exe" +// DEFAULT-SAME: "--thinlto-remote-compiler={{.*}}clang{{[^\"]*}}" + +/// Check that nothing is forwarded when the compiler is not in LTO mode, and that +/// appropriate unused option warnings are issued. +// RUN: %clang %s -### -fuse-ld=lld --target=x86_64-linux-gnu \ +// RUN: -fthinlto-distributor=d.exe 2>&1 | \ +// RUN: FileCheck %s --check-prefix=NOFLTO --implicit-check-not=distributor \ +// RUN: --implicit-check-not=remote-compiler + +// NOFLTO: warning: argument unused during compilation: '-fthinlto-distributor=d.exe' +// NOFLTO: ld.lld diff --git a/cross-project-tests/dtlto/ld-dtlto.c b/cross-project-tests/dtlto/ld-dtlto.c index 3ee962346bd4a..7dffe2e015bcb 100644 --- a/cross-project-tests/dtlto/ld-dtlto.c +++ b/cross-project-tests/dtlto/ld-dtlto.c @@ -5,13 +5,11 @@ // RUN: rm -rf %t && mkdir %t && cd %t -// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o - -// RUN: ld.lld dtlto.o \ -// RUN: --thinlto-distributor=%python \ -// RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ -// RUN: --thinlto-remote-compiler=%clang \ -// RUN: --thinlto-remote-compiler-arg=--save-temps +// RUN: %clang --target=x86_64-linux-gnu %s -flto=thin -fuse-ld=lld \ +// RUN: -fthinlto-distributor=%python \ +// RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \ +// RUN: -Wl,--thinlto-remote-compiler-arg=--save-temps \ +// RUN: -nostdlib -Werror /// Check that the required output files have been created. // RUN: ls | sort | FileCheck %s @@ -22,18 +20,15 @@ /// Linked ELF. // CHECK: {{^}}a.out{{$}} -/// Produced by the bitcode compilation. -// CHECK-NEXT: {{^}}dtlto.o{{$}} - /// --save-temps output for the backend compilation. -// CHECK-NEXT: {{^}}dtlto.s{{$}} -// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP:[a-zA-Z0-9_]+]].s{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.0.preopt.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.1.promote.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.2.internalize.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.3.import.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.4.opt.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.5.precodegen.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.resolution.txt{{$}} /// No files are expected after. // CHECK-NOT: {{.}} `````````` </details> https://github.com/llvm/llvm-project/pull/149978 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
