Author: tejohnson Date: Mon Dec 7 13:21:34 2015 New Revision: 254927 URL: http://llvm.org/viewvc/llvm-project?rev=254927&view=rev Log: [ThinLTO] Option to invoke ThinLTO backend passes and importing
Summary: Adds new option -fthinlto-index=<file> to invoke the LTO pipeline along with function importing via clang using the supplied function summary index file. This supports invoking the parallel ThinLTO backend processes in a distributed build environment via clang. Additionally, this causes the module linker to be invoked on the bitcode file being compiled to perform any necessary promotion and renaming of locals that are exported via the function summary index file. Add a couple tests that confirm we get expected errors when we try to use the new option on a file that isn't bitcode, or specify an invalid index file. The tests also confirm that we trigger the expected function import pass. Depends on D15024 Reviewers: joker.eph, dexonsmith Subscribers: joker.eph, davidxl, cfe-commits Differential Revision: http://reviews.llvm.org/D15025 Added: cfe/trunk/test/CodeGen/thinlto_backend.c cfe/trunk/test/Driver/thinlto_backend.c Modified: cfe/trunk/include/clang/CodeGen/BackendUtil.h cfe/trunk/include/clang/Driver/Options.td cfe/trunk/include/clang/Driver/Types.h cfe/trunk/include/clang/Frontend/CodeGenOptions.h cfe/trunk/lib/CodeGen/BackendUtil.cpp cfe/trunk/lib/CodeGen/CodeGenAction.cpp cfe/trunk/lib/Driver/Tools.cpp cfe/trunk/lib/Driver/Types.cpp cfe/trunk/lib/Frontend/CompilerInvocation.cpp cfe/trunk/test/CMakeLists.txt Modified: cfe/trunk/include/clang/CodeGen/BackendUtil.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/CodeGen/BackendUtil.h?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/include/clang/CodeGen/BackendUtil.h (original) +++ cfe/trunk/include/clang/CodeGen/BackendUtil.h Mon Dec 7 13:21:34 2015 @@ -11,6 +11,8 @@ #define LLVM_CLANG_CODEGEN_BACKENDUTIL_H #include "clang/Basic/LLVM.h" +#include "llvm/IR/FunctionInfo.h" +#include <memory> namespace llvm { class Module; @@ -31,10 +33,12 @@ namespace clang { Backend_EmitObj ///< Emit native object files }; - void EmitBackendOutput(DiagnosticsEngine &Diags, const CodeGenOptions &CGOpts, - const TargetOptions &TOpts, const LangOptions &LOpts, - StringRef TDesc, llvm::Module *M, BackendAction Action, - raw_pwrite_stream *OS); + void + EmitBackendOutput(DiagnosticsEngine &Diags, const CodeGenOptions &CGOpts, + const TargetOptions &TOpts, const LangOptions &LOpts, + StringRef TDesc, llvm::Module *M, BackendAction Action, + raw_pwrite_stream *OS, + std::unique_ptr<llvm::FunctionInfoIndex> Index = nullptr); } #endif Modified: cfe/trunk/include/clang/Driver/Options.td URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Driver/Options.td?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/include/clang/Driver/Options.td (original) +++ cfe/trunk/include/clang/Driver/Options.td Mon Dec 7 13:21:34 2015 @@ -693,6 +693,9 @@ def flto : Flag<["-"], "flto">, Flags<[C HelpText<"Enable LTO in 'full' mode">; def fno_lto : Flag<["-"], "fno-lto">, Group<f_Group>, HelpText<"Disable LTO mode (default)">; +def fthinlto_index_EQ : Joined<["-"], "fthinlto-index=">, + Flags<[CC1Option]>, Group<f_Group>, + HelpText<"Perform ThinLTO importing using provided function summary index">; def fmacro_backtrace_limit_EQ : Joined<["-"], "fmacro-backtrace-limit=">, Group<f_Group>, Flags<[DriverOption, CoreOption]>; def fmerge_all_constants : Flag<["-"], "fmerge-all-constants">, Group<f_Group>; Modified: cfe/trunk/include/clang/Driver/Types.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Driver/Types.h?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/include/clang/Driver/Types.h (original) +++ cfe/trunk/include/clang/Driver/Types.h Mon Dec 7 13:21:34 2015 @@ -63,6 +63,9 @@ namespace types { /// isCXX - Is this a "C++" input (C++ and Obj-C++ sources and headers). bool isCXX(ID Id); + /// Is this LLVM IR. + bool isLLVMIR(ID Id); + /// isCuda - Is this a CUDA input. bool isCuda(ID Id); Modified: cfe/trunk/include/clang/Frontend/CodeGenOptions.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/CodeGenOptions.h?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/include/clang/Frontend/CodeGenOptions.h (original) +++ cfe/trunk/include/clang/Frontend/CodeGenOptions.h Mon Dec 7 13:21:34 2015 @@ -167,6 +167,10 @@ public: /// Name of the profile file to use as input for -fprofile-instr-use std::string InstrProfileInput; + /// Name of the function summary index file to use for ThinLTO function + /// importing. + std::string ThinLTOIndexFile; + /// The EABI version to use std::string EABIVersion; Modified: cfe/trunk/lib/CodeGen/BackendUtil.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/CodeGen/BackendUtil.cpp?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/lib/CodeGen/BackendUtil.cpp (original) +++ cfe/trunk/lib/CodeGen/BackendUtil.cpp Mon Dec 7 13:21:34 2015 @@ -22,6 +22,7 @@ #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/FunctionInfo.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" @@ -53,6 +54,7 @@ class EmitAssemblyHelper { const clang::TargetOptions &TargetOpts; const LangOptions &LangOpts; Module *TheModule; + std::unique_ptr<FunctionInfoIndex> FunctionIndex; Timer CodeGenerationTime; @@ -113,15 +115,14 @@ private: bool AddEmitPasses(BackendAction Action, raw_pwrite_stream &OS); public: - EmitAssemblyHelper(DiagnosticsEngine &_Diags, - const CodeGenOptions &CGOpts, + EmitAssemblyHelper(DiagnosticsEngine &_Diags, const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts, - const LangOptions &LOpts, - Module *M) - : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts), - TheModule(M), CodeGenerationTime("Code Generation Time"), - CodeGenPasses(nullptr), PerModulePasses(nullptr), - PerFunctionPasses(nullptr) {} + const LangOptions &LOpts, Module *M, + std::unique_ptr<FunctionInfoIndex> Index) + : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts), + TheModule(M), FunctionIndex(std::move(Index)), + CodeGenerationTime("Code Generation Time"), CodeGenPasses(nullptr), + PerModulePasses(nullptr), PerFunctionPasses(nullptr) {} ~EmitAssemblyHelper() { delete CodeGenPasses; @@ -285,6 +286,29 @@ void EmitAssemblyHelper::CreatePasses() } PassManagerBuilderWrapper PMBuilder(CodeGenOpts, LangOpts); + + // Figure out TargetLibraryInfo. + Triple TargetTriple(TheModule->getTargetTriple()); + PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); + + switch (Inlining) { + case CodeGenOptions::NoInlining: + break; + case CodeGenOptions::NormalInlining: { + PMBuilder.Inliner = + createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize); + break; + } + case CodeGenOptions::OnlyAlwaysInlining: + // Respect always_inline. + if (OptLevel == 0) + // Do not insert lifetime intrinsics at -O0. + PMBuilder.Inliner = createAlwaysInlinerPass(false); + else + PMBuilder.Inliner = createAlwaysInlinerPass(); + break; + } + PMBuilder.OptLevel = OptLevel; PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize; PMBuilder.BBVectorize = CodeGenOpts.VectorizeBB; @@ -297,6 +321,17 @@ void EmitAssemblyHelper::CreatePasses() PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO; PMBuilder.RerollLoops = CodeGenOpts.RerollLoops; + legacy::PassManager *MPM = getPerModulePasses(); + + // If we are performing a ThinLTO importing compile, invoke the LTO + // pipeline and pass down the in-memory function index. + if (!CodeGenOpts.ThinLTOIndexFile.empty()) { + assert(FunctionIndex && "Expected non-empty function index"); + PMBuilder.FunctionIndex = FunctionIndex.get(); + PMBuilder.populateLTOPassManager(*MPM); + return; + } + PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible, addAddDiscriminatorsPass); @@ -361,27 +396,6 @@ void EmitAssemblyHelper::CreatePasses() addDataFlowSanitizerPass); } - // Figure out TargetLibraryInfo. - Triple TargetTriple(TheModule->getTargetTriple()); - PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); - - switch (Inlining) { - case CodeGenOptions::NoInlining: break; - case CodeGenOptions::NormalInlining: { - PMBuilder.Inliner = - createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize); - break; - } - case CodeGenOptions::OnlyAlwaysInlining: - // Respect always_inline. - if (OptLevel == 0) - // Do not insert lifetime intrinsics at -O0. - PMBuilder.Inliner = createAlwaysInlinerPass(false); - else - PMBuilder.Inliner = createAlwaysInlinerPass(); - break; - } - // Set up the per-function pass manager. legacy::FunctionPassManager *FPM = getPerFunctionPasses(); if (CodeGenOpts.VerifyModule) @@ -389,7 +403,6 @@ void EmitAssemblyHelper::CreatePasses() PMBuilder.populateFunctionPassManager(*FPM); // Set up the per-module pass manager. - legacy::PassManager *MPM = getPerModulePasses(); if (!CodeGenOpts.RewriteMapFiles.empty()) addSymbolRewriterPass(CodeGenOpts, MPM); @@ -660,8 +673,10 @@ void clang::EmitBackendOutput(Diagnostic const clang::TargetOptions &TOpts, const LangOptions &LOpts, StringRef TDesc, Module *M, BackendAction Action, - raw_pwrite_stream *OS) { - EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M); + raw_pwrite_stream *OS, + std::unique_ptr<FunctionInfoIndex> Index) { + EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M, + std::move(Index)); AsmHelper.EmitAssembly(Action, OS); Modified: cfe/trunk/lib/CodeGen/CodeGenAction.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/CodeGen/CodeGenAction.cpp?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/lib/CodeGen/CodeGenAction.cpp (original) +++ cfe/trunk/lib/CodeGen/CodeGenAction.cpp Mon Dec 7 13:21:34 2015 @@ -26,10 +26,12 @@ #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/FunctionInfo.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Linker/Linker.h" +#include "llvm/Object/FunctionIndexObjectFile.h" #include "llvm/Pass.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" @@ -39,6 +41,24 @@ using namespace clang; using namespace llvm; namespace clang { +/// Diagnostic handler used by invocations of Linker::LinkModules +static void linkerDiagnosticHandler(const DiagnosticInfo &DI, + const llvm::Module *LinkModule, + DiagnosticsEngine &Diags) { + if (DI.getSeverity() != DS_Error) + return; + + std::string MsgStorage; + { + raw_string_ostream Stream(MsgStorage); + DiagnosticPrinterRawOStream DP(Stream); + DI.print(DP); + } + + Diags.Report(diag::err_fe_cannot_link_module) + << LinkModule->getModuleIdentifier() << MsgStorage; +} + class BackendConsumer : public ASTConsumer { virtual void anchor(); DiagnosticsEngine &Diags; @@ -167,7 +187,7 @@ namespace clang { llvm::Module *LinkModule = I.second.get(); if (Linker::linkModules(*M, *LinkModule, [=](const DiagnosticInfo &DI) { - linkerDiagnosticHandler(DI, LinkModule); + linkerDiagnosticHandler(DI, LinkModule, Diags); }, LinkFlags)) return; @@ -233,9 +253,6 @@ namespace clang { ((BackendConsumer*)Context)->InlineAsmDiagHandler2(SM, Loc); } - void linkerDiagnosticHandler(const llvm::DiagnosticInfo &DI, - const llvm::Module *LinkModule); - static void DiagnosticHandler(const llvm::DiagnosticInfo &DI, void *Context) { ((BackendConsumer *)Context)->DiagnosticHandlerImpl(DI); @@ -545,22 +562,6 @@ void BackendConsumer::OptimizationFailur EmitOptimizationMessage(D, diag::warn_fe_backend_optimization_failure); } -void BackendConsumer::linkerDiagnosticHandler(const DiagnosticInfo &DI, - const llvm::Module *LinkModule) { - if (DI.getSeverity() != DS_Error) - return; - - std::string MsgStorage; - { - raw_string_ostream Stream(MsgStorage); - DiagnosticPrinterRawOStream DP(Stream); - DI.print(DP); - } - - Diags.Report(diag::err_fe_cannot_link_module) - << LinkModule->getModuleIdentifier() << MsgStorage; -} - /// \brief This function is invoked when the backend needs /// to report something to the user. void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) { @@ -785,11 +786,44 @@ void CodeGenAction::ExecuteAction() { TheModule->setTargetTriple(TargetOpts.Triple); } + auto DiagHandler = [&](const DiagnosticInfo &DI) { + linkerDiagnosticHandler(DI, TheModule.get(), + getCompilerInstance().getDiagnostics()); + }; + + // If we are performing ThinLTO importing compilation (indicated by + // a non-empty index file option), then we need promote to global scope + // and rename any local values that are potentially exported to other + // modules. Do this early so that the rest of the compilation sees the + // promoted symbols. + std::unique_ptr<FunctionInfoIndex> Index; + if (!CI.getCodeGenOpts().ThinLTOIndexFile.empty()) { + ErrorOr<std::unique_ptr<FunctionInfoIndex>> IndexOrErr = + llvm::getFunctionIndexForFile(CI.getCodeGenOpts().ThinLTOIndexFile, + DiagHandler); + if (std::error_code EC = IndexOrErr.getError()) { + std::string Error = EC.message(); + errs() << "Error loading index file '" + << CI.getCodeGenOpts().ThinLTOIndexFile << "': " << Error + << "\n"; + return; + } + Index = std::move(IndexOrErr.get()); + assert(Index); + // Currently this requires creating a new Module object. + std::unique_ptr<llvm::Module> RenamedModule = + renameModuleForThinLTO(TheModule, Index.get(), DiagHandler); + if (!RenamedModule) + return; + + TheModule = std::move(RenamedModule); + } + LLVMContext &Ctx = TheModule->getContext(); Ctx.setInlineAsmDiagnosticHandler(BitcodeInlineAsmDiagHandler); EmitBackendOutput(CI.getDiagnostics(), CI.getCodeGenOpts(), TargetOpts, CI.getLangOpts(), CI.getTarget().getDataLayoutString(), - TheModule.get(), BA, OS); + TheModule.get(), BA, OS, std::move(Index)); return; } Modified: cfe/trunk/lib/Driver/Tools.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/lib/Driver/Tools.cpp (original) +++ cfe/trunk/lib/Driver/Tools.cpp Mon Dec 7 13:21:34 2015 @@ -3419,6 +3419,13 @@ void Clang::ConstructJob(Compilation &C, Args.AddLastArg(CmdArgs, options::OPT_flto, options::OPT_flto_EQ); } + if (const Arg *A = Args.getLastArg(options::OPT_fthinlto_index_EQ)) { + if (!types::isLLVMIR(Input.getType())) + D.Diag(diag::err_drv_argument_only_allowed_with) << A->getAsString(Args) + << "-x ir"; + Args.AddLastArg(CmdArgs, options::OPT_fthinlto_index_EQ); + } + // We normally speed up the clang process a bit by skipping destructors at // exit, but when we're generating diagnostics we can rely on some of the // cleanup. Modified: cfe/trunk/lib/Driver/Types.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Types.cpp?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/lib/Driver/Types.cpp (original) +++ cfe/trunk/lib/Driver/Types.cpp Mon Dec 7 13:21:34 2015 @@ -128,6 +128,19 @@ bool types::isCXX(ID Id) { } } +bool types::isLLVMIR(ID Id) { + switch (Id) { + default: + return false; + + case TY_LLVM_IR: + case TY_LLVM_BC: + case TY_LTO_IR: + case TY_LTO_BC: + return true; + } +} + bool types::isCuda(ID Id) { switch (Id) { default: Modified: cfe/trunk/lib/Frontend/CompilerInvocation.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInvocation.cpp?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/lib/Frontend/CompilerInvocation.cpp (original) +++ cfe/trunk/lib/Frontend/CompilerInvocation.cpp Mon Dec 7 13:21:34 2015 @@ -521,6 +521,12 @@ static bool ParseCodeGenArgs(CodeGenOpti Opts.PrepareForLTO = Args.hasArg(OPT_flto, OPT_flto_EQ); const Arg *A = Args.getLastArg(OPT_flto, OPT_flto_EQ); Opts.EmitFunctionSummary = A && A->containsValue("thin"); + if (Arg *A = Args.getLastArg(OPT_fthinlto_index_EQ)) { + if (IK != IK_LLVM_IR) + Diags.Report(diag::err_drv_argument_only_allowed_with) + << A->getAsString(Args) << "-x ir"; + Opts.ThinLTOIndexFile = Args.getLastArgValue(OPT_fthinlto_index_EQ); + } Opts.MSVolatile = Args.hasArg(OPT_fms_volatile); Modified: cfe/trunk/test/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CMakeLists.txt?rev=254927&r1=254926&r2=254927&view=diff ============================================================================== --- cfe/trunk/test/CMakeLists.txt (original) +++ cfe/trunk/test/CMakeLists.txt Mon Dec 7 13:21:34 2015 @@ -61,6 +61,7 @@ if( NOT CLANG_BUILT_STANDALONE ) FileCheck count not llc llvm-bcanalyzer + llvm-lto llvm-objdump llvm-profdata llvm-readobj Added: cfe/trunk/test/CodeGen/thinlto_backend.c URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CodeGen/thinlto_backend.c?rev=254927&view=auto ============================================================================== --- cfe/trunk/test/CodeGen/thinlto_backend.c (added) +++ cfe/trunk/test/CodeGen/thinlto_backend.c Mon Dec 7 13:21:34 2015 @@ -0,0 +1,14 @@ +// RUN: %clang -target x86_64-unknown-linux -O2 %s -flto=thin -c -o %t.o +// RUN: llvm-lto -thinlto -o %t %t.o + +// Ensure clang -cc1 give expected error for incorrect input type +// RUN: not %clang_cc1 -target x86_64-unknown-linux -O2 -o %t1.o %s -c -fthinlto-index=%t.thinlto.bc 2>&1 | FileCheck %s -check-prefix=CHECK-WARNING +// CHECK-WARNING: error: invalid argument '-fthinlto-index={{.*}}' only allowed with '-x ir' + +// Ensure we get expected error for missing index file +// RUN: %clang -target x86_64-unknown-linux -O2 -o %t1.o -x ir %t.o -c -fthinlto-index=bad.thinlto.bc 2>&1 | FileCheck %s -check-prefix=CHECK-ERROR +// CHECK-ERROR: Error loading index file 'bad.thinlto.bc': No such file or directory + +// Ensure Function Importing pass added +// RUN: %clang -target x86_64-unknown-linux -O2 -o %t1.o -x ir %t.o -c -fthinlto-index=%t.thinlto.bc -mllvm -debug-pass=Structure 2>&1 | FileCheck %s -check-prefix=CHECK-PASS +// CHECK-PASS: Function Importing Added: cfe/trunk/test/Driver/thinlto_backend.c URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Driver/thinlto_backend.c?rev=254927&view=auto ============================================================================== --- cfe/trunk/test/Driver/thinlto_backend.c (added) +++ cfe/trunk/test/Driver/thinlto_backend.c Mon Dec 7 13:21:34 2015 @@ -0,0 +1,10 @@ +// RUN: %clang -target x86_64-unknown-linux -O2 %s -flto=thin -c -o %t.o +// RUN: llvm-lto -thinlto -o %t %t.o + +// -fthinlto_index should be passed to cc1 +// RUN: %clang -target x86_64-unknown-linux -O2 -o %t1.o -x ir %t.o -c -fthinlto-index=%t.thinlto.bc -### 2>&1 | FileCheck %s -check-prefix=CHECK-THINLTOBE-ACTION +// CHECK-THINLTOBE-ACTION: -fthinlto-index= + +// Ensure clang driver gives the expected error for incorrect input type +// RUN: not %clang -target x86_64-unknown-linux -O2 -o %t1.o %s -c -fthinlto-index=%t.thinlto.bc 2>&1 | FileCheck %s -check-prefix=CHECK-WARNING +// CHECK-WARNING: error: invalid argument '-fthinlto-index={{.*}}' only allowed with '-x ir' _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
