llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-lld-coff Author: None (llvmbot) <details> <summary>Changes</summary> Backport bbbbc093febffcae262cde1baa429b950842d76e Requested by: @<!-- -->bd1976bris --- Full diff: https://github.com/llvm/llvm-project/pull/149979.diff 10 Files Affected: - (added) cross-project-tests/dtlto/link-archive-thin.test (+93) - (added) cross-project-tests/dtlto/link-dtlto.c (+41) - (modified) cross-project-tests/lit.cfg.py (+1-1) - (modified) lld/COFF/Config.h (+12) - (modified) lld/COFF/Driver.cpp (+17) - (modified) lld/COFF/LTO.cpp (+10-1) - (modified) lld/COFF/Options.td (+11) - (modified) lld/docs/DTLTO.rst (+35-2) - (added) lld/test/COFF/dtlto/files.test (+71) - (added) lld/test/COFF/dtlto/options.test (+56) ``````````diff diff --git a/cross-project-tests/dtlto/link-archive-thin.test b/cross-project-tests/dtlto/link-archive-thin.test new file mode 100644 index 0000000000000..fbd8fd67300cf --- /dev/null +++ b/cross-project-tests/dtlto/link-archive-thin.test @@ -0,0 +1,93 @@ +REQUIRES: lld-link + +## Test that a DTLTO link succeeds and outputs the expected set of files +## correctly when thin archives are present. + +RUN: rm -rf %t && split-file %s %t && cd %t + +## Compile bitcode. -O2 is required for cross-module importing. +RUN: %clang -O2 --target=x86_64-pc-windows-msvc -flto=thin -c \ +RUN: foo.c bar.c dog.c cat.c start.c + +## Generate thin archives. +RUN: lld-link /lib /llvmlibthin /out:foo.lib foo.o +## Create this bitcode thin archive in a subdirectory to test the expansion of +## the path to a bitcode file that is referenced using "..", e.g., in this case +## "../bar.o". +RUN: mkdir lib +RUN: lld-link /lib /llvmlibthin /out:lib/bar.lib bar.o +## Create this bitcode thin archive with an absolute path entry containing "..". +RUN: lld-link /lib /llvmlibthin /out:dog.lib %t/lib/../dog.o +RUN: lld-link /lib /llvmlibthin /out:cat.lib cat.o +RUN: lld-link /lib /llvmlibthin /out:start.lib start.o + +## Link from a different directory to ensure that thin archive member paths are +## resolved correctly relative to the archive locations. +RUN: mkdir %t/out && cd %t/out +RUN: lld-link /subsystem:console /machine:x64 /entry:start /out:my.exe \ +RUN: %t/foo.lib %t/lib/bar.lib ../start.lib %t/cat.lib \ +RUN: /includeoptional:dog ../dog.lib \ +RUN: -thinlto-distributor:%python \ +RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \ +RUN: -thinlto-remote-compiler:%clang \ +RUN: /lldsavetemps + +## Check that the required output files have been created. +RUN: ls | FileCheck %s --check-prefix=OUTPUTS --implicit-check-not=cat + +## JSON jobs description. +OUTPUTS-DAG: my.[[PID:[a-zA-Z0-9_]+]].dist-file.json + +## Individual summary index files. +OUTPUTS-DAG: start.1.[[PID]].native.o.thinlto.bc{{$}} +OUTPUTS-DAG: dog.2.[[PID]].native.o.thinlto.bc{{$}} +OUTPUTS-DAG: foo.3.[[PID]].native.o.thinlto.bc{{$}} +OUTPUTS-DAG: bar.4.[[PID]].native.o.thinlto.bc{{$}} + +## Native output object files. +OUTPUTS-DAG: start.1.[[PID]].native.o{{$}} +OUTPUTS-DAG: dog.2.[[PID]].native.o{{$}} +OUTPUTS-DAG: foo.3.[[PID]].native.o{{$}} +OUTPUTS-DAG: bar.4.[[PID]].native.o{{$}} + + +## It is important that cross-module inlining occurs for this test to show that Clang can +## successfully load the bitcode file dependencies recorded in the summary indices. +## Explicitly check that the expected importing has occurred. + +RUN: llvm-dis start.1.*.native.o.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,BAR,START + +RUN: llvm-dis dog.2.*.native.o.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,BAR,DOG,START + +RUN: llvm-dis foo.3.*.native.o.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,BAR,START + +RUN: llvm-dis bar.4.*.native.o.thinlto.bc -o - | \ +RUN: FileCheck %s --check-prefixes=FOO,BAR,START + +FOO-DAG: foo.o +BAR-DAG: bar.o +DOG-DAG: dog.o +START-DAG: start.o + + +#--- foo.c +extern int bar(int), start(int); +__attribute__((retain)) int foo(int x) { return x + bar(x) + start(x); } + +#--- bar.c +extern int foo(int), start(int); +__attribute__((retain)) int bar(int x) { return x + foo(x) + start(x); } + +#--- dog.c +extern int foo(int), bar(int), start(int); +__attribute__((retain)) int dog(int x) { return x + foo(x) + bar(x) + start(x); } + +#--- cat.c +__attribute__((retain)) void cat(int x) {} + +#--- start.c +extern int foo(int), bar(int); +__attribute__((retain)) int start(int x) { return x + foo(x) + bar(x); } diff --git a/cross-project-tests/dtlto/link-dtlto.c b/cross-project-tests/dtlto/link-dtlto.c new file mode 100644 index 0000000000000..0ab4ec57f115d --- /dev/null +++ b/cross-project-tests/dtlto/link-dtlto.c @@ -0,0 +1,41 @@ +// REQUIRES: lld-link + +/// Simple test that DTLTO works with a single input bitcode file and that +/// --save-temps can be applied to the remote compilation. + +// RUN: rm -rf %t && mkdir %t && cd %t + +// RUN: %clang --target=x86_64-pc-windows-msvc -c -flto=thin %s -o dtlto.obj + +// RUN: lld-link /subsystem:console /entry:_start dtlto.obj \ +// RUN: -thinlto-distributor:%python \ +// RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \ +// RUN: -thinlto-remote-compiler:%clang \ +// RUN: -thinlto-remote-compiler-arg:--save-temps + +/// Check that the required output files have been created. +// RUN: ls | sort | FileCheck %s + +/// No files are expected before. +// CHECK-NOT: {{.}} + +/// Linked ELF. +// CHECK: {{^}}dtlto.exe{{$}} + +/// Produced by the bitcode compilation. +// CHECK-NEXT: {{^}}dtlto.obj{{$}} + +/// --save-temps output for the backend compilation. +// CHECK-NEXT: {{^}}dtlto.s{{$}} +// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}} +// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}} +// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}} +// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}} +// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}} +// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}} +// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}} + +/// No files are expected after. +// CHECK-NOT: {{.}} + +int _start() { return 0; } diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py index b35c643ac898c..31c93923ac9ed 100644 --- a/cross-project-tests/lit.cfg.py +++ b/cross-project-tests/lit.cfg.py @@ -19,7 +19,7 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) # suffixes: A list of file extensions to treat as test files. -config.suffixes = [".c", ".cl", ".cpp", ".m"] +config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h index 9220c847f356d..91b6e632fa7ed 100644 --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -192,6 +192,18 @@ struct Configuration { // Used for /lldltocachepolicy=policy llvm::CachePruningPolicy ltoCachePolicy; + // Used for /thinlto-distributor:<path> + StringRef dtltoDistributor; + + // Used for /thinlto-distributor-arg:<arg> + llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs; + + // Used for /thinlto-remote-compiler:<path> + StringRef dtltoCompiler; + + // Used for /thinlto-remote-compiler-arg:<arg> + llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs; + // Used for /opt:[no]ltodebugpassmanager bool ltoDebugPassManager = false; diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp index 0f649ff1cf4f8..bcd4796588aef 100644 --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -2088,6 +2088,23 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { Fatal(ctx) << "/manifestinput: requires /manifest:embed"; } + // Handle /thinlto-distributor:<path> + config->dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor); + + // Handle /thinlto-distributor-arg:<arg> + for (auto *arg : args.filtered(OPT_thinlto_distributor_arg)) + config->dtltoDistributorArgs.push_back(arg->getValue()); + + // Handle /thinlto-remote-compiler:<path> + config->dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler); + if (!config->dtltoDistributor.empty() && config->dtltoCompiler.empty()) + Err(ctx) << "A value must be specified for /thinlto-remote-compiler if " + "/thinlto-distributor is specified."; + + // Handle /thinlto-remote-compiler-arg:<arg> + for (auto *arg : args.filtered(OPT_thinlto_compiler_arg)) + config->dtltoCompilerArgs.push_back(arg->getValue()); + // Handle /dwodir config->dwoDir = args.getLastArgValue(OPT_dwodir); diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 2a4d07cc2d015..1050874a1b10c 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -110,7 +110,16 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { // Initialize ltoObj. lto::ThinBackend backend; - if (ctx.config.thinLTOIndexOnly) { + if (!ctx.config.dtltoDistributor.empty()) { + backend = lto::createOutOfProcessThinBackend( + llvm::hardware_concurrency(ctx.config.thinLTOJobs), + /*OnWrite=*/nullptr, + /*ShouldEmitIndexFiles=*/false, + /*ShouldEmitImportFiles=*/false, ctx.config.outputFile, + ctx.config.dtltoDistributor, ctx.config.dtltoDistributorArgs, + ctx.config.dtltoCompiler, ctx.config.dtltoCompilerArgs, + !ctx.config.saveTempsArgs.empty()); + } else if (ctx.config.thinLTOIndexOnly) { auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); }; backend = lto::createWriteIndexesThinBackend( llvm::hardware_concurrency(ctx.config.thinLTOJobs), diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td index a887d7d351e18..2a82fb5cd8845 100644 --- a/lld/COFF/Options.td +++ b/lld/COFF/Options.td @@ -270,6 +270,17 @@ def thinlto_object_suffix_replace : P< def thinlto_prefix_replace: P< "thinlto-prefix-replace", "'old;new' replace old prefix with new prefix in ThinLTO outputs">; +def thinlto_distributor : P<"thinlto-distributor", + "Distributor to use for ThinLTO backend compilations. If specified, ThinLTO " + "backend compilations will be distributed">; +def thinlto_distributor_arg : P<"thinlto-distributor-arg", + "Arguments to pass to the ThinLTO distributor">; +def thinlto_compiler : P<"thinlto-remote-compiler", + "Compiler for the ThinLTO distributor to invoke for ThinLTO backend " + "compilations">; +def thinlto_compiler_arg : P<"thinlto-remote-compiler-arg", + "Compiler arguments for the ThinLTO distributor to pass for ThinLTO backend " + "compilations">; def lto_obj_path : P< "lto-obj-path", "output native object for merged LTO unit to this path">; diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst index 985decf6c7db8..54fcc034d1371 100644 --- a/lld/docs/DTLTO.rst +++ b/lld/docs/DTLTO.rst @@ -7,8 +7,7 @@ during the traditional link step. The implementation is documented here: https://llvm.org/docs/DTLTO.html. -Currently, DTLTO is only supported in ELF LLD. Support will be added to other -LLD flavours in the future. +Currently, DTLTO is only supported in ELF and COFF LLD. ELF LLD ------- @@ -40,3 +39,37 @@ The command-line interface is as follows: Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported. Currently, other options are silently accepted but do not have the intended effect. Support for such options will be expanded in the future. + +COFF LLD +-------- + +The command-line interface is as follows: + +- ``/thinlto-distributor:<path>`` + Specifies the file to execute as the distributor process. If specified, + ThinLTO backend compilations will be distributed. + +- ``/thinlto-remote-compiler:<path>`` + Specifies the path to the compiler that the distributor process will use for + backend compilations. The compiler invoked must match the version of LLD. + +- ``/thinlto-distributor-arg:<arg>`` + Specifies ``<arg>`` on the command line when invoking the distributor. + Can be specified multiple times. + +- ``/thinlto-remote-compiler-arg:<arg>`` + Appends ``<arg>`` to the remote compiler's command line. + Can be specified multiple times. + + Options that introduce extra input/output files may cause miscompilation if + the distribution system does not automatically handle pushing/fetching them to + remote nodes. In such cases, configure the distributor - possibly using + ``/thinlto-distributor-arg:`` - to manage these dependencies. See the + distributor documentation for details. + +Some LLD LTO options (e.g., ``/lto-sample-profile:<file>``) are supported. +Currently, other options are silently accepted but do not have the intended +effect. Support for such options could be expanded in the future. + +Currently, there is no DTLTO command line interface supplied for ``clang-cl``, +as users are expected to invoke LLD directly. \ No newline at end of file diff --git a/lld/test/COFF/dtlto/files.test b/lld/test/COFF/dtlto/files.test new file mode 100644 index 0000000000000..4297adac9bbf0 --- /dev/null +++ b/lld/test/COFF/dtlto/files.test @@ -0,0 +1,71 @@ +REQUIRES: x86 + +## Test that the LLD options /lldsavetemps and -thinlto-emit-imports-files +## function correctly with DTLTO we also check that index files +## (-thinlto-emit-index-files) are not emitted with DTLTO. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll + +## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the +## linker. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc +RUN: cp t1.bc t3.bc + +## Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.obj +RUN: llc t2.ll --filetype=obj -o t2.obj + +## Create response file containing shared ThinLTO linker arguments. +## -start-lib/-end-lib is used to test the special case where unused lazy +## bitcode inputs result in empty index/imports files. +## Note that mock.py does not do any compilation; instead, it simply writes +## the contents of the object files supplied on the command line into the +## output object files in job order. +RUN: echo "/entry:t1 /subsystem:console \ +RUN: t1.bc t2.bc -start-lib t3.bc -end-lib /out:my.exe \ +RUN: -thinlto-distributor:\"%python\" \ +RUN: -thinlto-distributor-arg:\"%llvm_src_root/utils/dtlto/mock.py\" \ +RUN: -thinlto-distributor-arg:t1.obj \ +RUN: -thinlto-distributor-arg:t2.obj \ +RUN: -thinlto-remote-compiler:fake.exe" > l.rsp + +## Check that without extra flags, no index/imports files are produced and +## backend temp files are removed. +RUN: lld-link @l.rsp +RUN: ls | FileCheck %s \ +RUN: --check-prefixes=NOBACKEND,NOOTHERS + +## Check that with /lldsavetemps and -thinlto-emit-imports-files backend +## tempoary files are retained and no index/imports files are produced. +RUN: rm -f *.imports *.thinlto.bc +RUN: lld-link @l.rsp /lldsavetemps -thinlto-emit-imports-files +RUN: ls | sort | FileCheck %s \ +RUN: --check-prefixes=BACKEND,NOOTHERS + +## JSON jobs description, retained with --save-temps. +## Note that DTLTO temporary files include a PID component. +NOBACKEND-NOT: {{^}}my.[[#]].dist-file.json{{$}} +BACKEND: {{^}}my.[[#]].dist-file.json{{$}} + +## Index/imports files for t1.bc. +NOOTHERS-NOT: {{^}}t1.bc.imports{{$}} +NOOTHERS-NOT: {{^}}t1.bc.thinlto.bc{{$}} + +## Index/imports files for t2.bc. +NOOTHERS-NOT: {{^}}t2.bc.imports{{$}} +NOOTHERS-NOT: {{^}}t2.bc.thinlto.bc{{$}} + +## Empty index/imports files for unused t3.bc. +NOOTHERS-NOT: {{^}}t3.bc.imports{{$}} +NOOTHERS-NOT: {{^}}t3.bc.thinlto.bc{{$}} + +#--- t1.ll +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +define void @t1() { + ret void +} diff --git a/lld/test/COFF/dtlto/options.test b/lld/test/COFF/dtlto/options.test new file mode 100644 index 0000000000000..023ecd2359101 --- /dev/null +++ b/lld/test/COFF/dtlto/options.test @@ -0,0 +1,56 @@ +REQUIRES: x86 + +## Test that DTLTO-specific options are handled correctly. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: opt -thinlto-bc foo.ll -o foo.obj + +## Not specifying a value for -thinlto-remote-compiler should result in an +## error if -thinlto-distributor is specified. +RUN: not lld-link /entry:foo /subsystem:console foo.obj /out:my.exe \ +RUN: -thinlto-distributor:fake.exe 2>&1 | FileCheck %s --check-prefix=COMPILER +RUN: lld-link /entry:foo /subsystem:console foo.obj /out:my.exe + +## Specifying an empty value for -thinlto-remote-compiler should result in an +## error if -thinlto-distributor is specified. +RUN: not lld-link /entry:foo /subsystem:console foo.obj /out:my.exe \ +RUN: -thinlto-distributor:fake.exe \ +RUN: -thinlto-remote-compiler:"" 2>&1 | FileCheck %s --check-prefix=COMPILER +RUN: lld-link /entry:foo /subsystem:console foo.obj /out:my.exe \ +RUN: -thinlto-remote-compiler:"" + +COMPILER: error: A value must be specified for /thinlto-remote-compiler if /thinlto-distributor is specified. + +## Test that DTLTO options are passed correctly to the distributor and +## remote compiler. +## Note: validate.py does not perform any compilation. Instead, it validates the +## received JSON, pretty-prints the JSON and the supplied arguments, and then +## exits with an error. This allows FileCheck directives to verify the +## distributor inputs. +RUN: not lld-link /entry:foo /subsystem:console foo.obj /out:my.exe \ +RUN: -thinlto-distributor:%python \ +RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/validate.py \ +RUN: -thinlto-distributor-arg:darg1=10 \ +RUN: -thinlto-distributor-arg:darg2=20 \ +RUN: -thinlto-remote-compiler:my_clang.exe \ +RUN: -thinlto-remote-compiler-arg:carg1=20 \ +RUN: -thinlto-remote-compiler-arg:carg2=30 2>&1 | FileCheck %s + +CHECK: distributor_args=['darg1=10', 'darg2=20'] + +CHECK: "linker_output": "my.exe" + +CHECK: "my_clang.exe" +CHECK: "carg1=20" +CHECK: "carg2=30" + +CHECK: error: DTLTO backend compilation: cannot open native object file: + +#--- foo.ll +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +define void @foo() { + ret void +} `````````` </details> https://github.com/llvm/llvm-project/pull/149979 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
