================ @@ -0,0 +1,950 @@ +//===- Construction of code generation pass pipelines ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file provides the implementation of the PassBuilder based on our +/// static pass registry as well as related functionality. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/CodeGenPrepare.h" +#include "llvm/CodeGen/DwarfEHPrepare.h" +#include "llvm/CodeGen/ExpandLargeDivRem.h" +#include "llvm/CodeGen/ExpandLargeFpConvert.h" +#include "llvm/CodeGen/ExpandMemCmp.h" +#include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/FinalizeISel.h" +#include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GlobalMergeFunctions.h" +#include "llvm/CodeGen/LiveVariables.h" +#include "llvm/CodeGen/LocalStackSlotAllocation.h" +#include "llvm/CodeGen/LowerEmuTLS.h" +#include "llvm/CodeGen/MachineFunctionAnalysis.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/ReplaceWithVeclib.h" +#include "llvm/CodeGen/SafeStack.h" +#include "llvm/CodeGen/SelectOptimize.h" +#include "llvm/CodeGen/ShadowStackGCLowering.h" +#include "llvm/CodeGen/SjLjEHPrepare.h" +#include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/UnreachableBlockElim.h" +#include "llvm/CodeGen/WasmEHPrepare.h" +#include "llvm/CodeGen/WinEHPrepare.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only! +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/ObjCARC.h" +#include "llvm/Transforms/Scalar/ConstantHoisting.h" +#include "llvm/Transforms/Scalar/LoopStrengthReduce.h" +#include "llvm/Transforms/Scalar/MergeICmps.h" +#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" +#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" +#include "llvm/Transforms/Utils/LowerGlobalDtors.h" +#include "llvm/Transforms/Utils/LowerInvoke.h" + +namespace llvm { +extern cl::opt<std::string> FSRemappingFile; +} + +using namespace llvm; + +void PassBuilder::invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM) { + for (auto &C : CodeGenIREarlyEPCallbacks) + C(MPM); +} + +void PassBuilder::invokeGCLoweringEPCallbacks(FunctionPassManager &FPM) { + for (auto &C : GCLoweringEPCallbacks) + C(FPM); +} + +void PassBuilder::invokeISelPrepareEPCallbacks(ModulePassManager &MPM) { + for (auto &C : ISelPrepareEPCallbacks) + C(MPM); +} + +void PassBuilder::invokeMachineSSAOptimizationEarlyEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineSSAOptimizationEarlyEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMachineSSAOptimizationLastEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineSSAOptimizationLastEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreRegAllocEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreRegAllocEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreRegBankSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreRegBankSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreGlobalInstructionSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostGlobalInstructionSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : ILPOptsEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMachineLateOptimizationEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineLateOptimizationEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : MIEmitEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : PreEmitEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostRegAllocEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostRegAllocEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : PreSched2EPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostBBSectionsEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostBBSectionsEPCallbacks) + C(MFPM); +} + +void PassBuilder::addDefaultCodeGenPreparePasses(ModulePassManager &MPM) { + FunctionPassManager FPM; + // CodeGen prepare + if (TM->getOptLevel() != CodeGenOptLevel::None && !CGPBO.DisableCGP) + FPM.addPass(CodeGenPreparePass(TM)); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); +} + +Error PassBuilder::addDefaultRegAllocFastPasses( + MachineFunctionPassManager &MFPM) { + MFPM.addPass(PHIEliminationPass()); + MFPM.addPass(TwoAddressInstructionPass()); + if (auto Err = addRegAllocPass(MFPM)) + return Err; + return Error::success(); +} + +Error PassBuilder::addDefaultRegAllocOptimizedPasses( + MachineFunctionPassManager &MFPM) { + MFPM.addPass(DetectDeadLanesPass()); + MFPM.addPass(InitUndefPass()); + MFPM.addPass(ProcessImplicitDefsPass()); + + // LiveVariables currently requires pure SSA form. + // + // FIXME: Once TwoAddressInstruction pass no longer uses kill flags, + // LiveVariables can be removed completely, and LiveIntervals can be + // directly computed. (We still either need to regenerate kill flags after + // regalloc, or preferably fix the scavenger to not depend on them). + // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables. + // When LiveVariables is removed this has to be removed/moved either. + // Explicit addition of UnreachableMachineBlockElim allows stopping before + // or after it with -stop-before/-stop-after. + MFPM.addPass(UnreachableMachineBlockElimPass()); + // FIXME: Some X86 tests failed because of incomplete pipeline. + // MFPM.addPass(RequireAnalysisPass<LiveVariablesAnalysis, + // MachineFunction>()); + + // Edge splitting is smarter with machine loop info. + MFPM.addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>()); + MFPM.addPass(PHIEliminationPass()); + + if (CGPBO.EarlyLiveIntervals) + MFPM.addPass(RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>()); + + MFPM.addPass(TwoAddressInstructionPass()); + MFPM.addPass(RegisterCoalescerPass()); + + // The machine scheduler may accidentally create disconnected components + // when moving subregister definitions around, avoid this by splitting them + // to separate vregs before. Splitting can also improve reg. allocation + // quality. + MFPM.addPass(RenameIndependentSubregsPass()); + + // PreRA instruction scheduling. + MFPM.addPass(MachineSchedulerPass()); + + if (auto Err = addRegAllocPass(MFPM)) + return Err; + // Finally rewrite virtual registers. + MFPM.addPass(VirtRegRewriterPass()); + + // Regalloc scoring for ML-driven eviction - noop except when learning a new + // eviction policy. + MFPM.addPass(RegAllocScoringPass()); + + return Error::success(); +} + +bool PassBuilder::isOptimizedRegAlloc() const { + return CGPBO.OptimizeRegAlloc.value_or(TM->getOptLevel() != + CodeGenOptLevel::None); +} + +// Find the Profile remapping file name. The internal option takes the +// precedence before getting from TargetMachine. +static std::string getFSRemappingFile(const TargetMachine *TM, + const CGPassBuilderOption &Options) { + if (!Options.FSRemappingFile.empty()) + return Options.FSRemappingFile; + const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileRemappingFile; +} + +// Find the FSProfile file name. The internal option takes the precedence +// before getting from TargetMachine. +static std::string getFSProfileFile(const TargetMachine *TM, + const CGPassBuilderOption &Options) { + if (!Options.FSProfileFile.empty()) + return Options.FSProfileFile; + const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileFile; +} + +Error PassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) { + const MCAsmInfo *MCAI = TM->getMCAsmInfo(); + if (!MCAI) + return make_error<StringError>("No MCAsmInfo!", inconvertibleErrorCode()); + switch (MCAI->getExceptionHandlingType()) { + case ExceptionHandling::SjLj: + // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both + // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, + // catch info can get misplaced when a selector ends up more than one block + // removed from the parent invoke(s). This could happen when a landing + // pad is shared by multiple invokes and is also a target of a normal + // edge from elsewhere. + FPM.addPass(SjLjEHPreparePass(TM)); + [[fallthrough]]; + case ExceptionHandling::DwarfCFI: + case ExceptionHandling::ARM: + case ExceptionHandling::AIX: + case ExceptionHandling::ZOS: + FPM.addPass(DwarfEHPreparePass(TM)); + break; + case ExceptionHandling::WinEH: + // We support using both GCC-style and MSVC-style exceptions on Windows, so + // add both preparation passes. Each pass will only actually run if it + // recognizes the personality function. + FPM.addPass(WinEHPreparePass()); + FPM.addPass(DwarfEHPreparePass(TM)); + break; + case ExceptionHandling::Wasm: + // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs + // on catchpads and cleanuppads because it does not outline them into + // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we + // should remove PHIs there. + FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true)); + FPM.addPass(WasmEHPreparePass()); + break; + case ExceptionHandling::None: + FPM.addPass(LowerInvokePass()); + + // The lower invoke pass may create unreachable code. Remove it. + FPM.addPass(UnreachableBlockElimPass()); + break; + } + return Error::success(); +} + +Error PassBuilder::addInstructionSelectorPasses( + MachineFunctionPassManager &MFPM) { + CodeGenOptLevel OptLevel = TM->getOptLevel(); + + // Core ISel + // Enable FastISel with -fast-isel, but allow that to be overridden. + TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true)); + // Determine an instruction selector. + enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; + SelectorType Selector; + + CGPBO.EnableFastISelOption.value_or(false); + if (CGPBO.EnableFastISelOption.value_or(false)) + Selector = SelectorType::FastISel; + + else if (CGPBO.EnableGlobalISelOption.value_or(false) || + (TM->Options.EnableGlobalISel && + !CGPBO.EnableGlobalISelOption.value_or(false))) + Selector = SelectorType::GlobalISel; + else if (OptLevel == CodeGenOptLevel::None && TM->getO0WantsFastISel()) + Selector = SelectorType::FastISel; + else + Selector = SelectorType::SelectionDAG; + + // Set consistently TM.Options.EnableFastISel and EnableGlobalISel. + if (Selector == SelectorType::FastISel) { + TM->setFastISel(true); + TM->setGlobalISel(false); + } else if (Selector == SelectorType::GlobalISel) { + TM->setFastISel(false); + TM->setGlobalISel(true); + } + + // Add instruction selector passes. + if (Selector == SelectorType::GlobalISel) { + MFPM.addPass(IRTranslatorPass()); + MFPM.addPass(LegalizerPass()); + + // Before running the register bank selector, ask the target if it + // wants to run some passes. + invokePreRegBankSelectEPCallbacks(MFPM); + MFPM.addPass(RegBankSelectPass()); ---------------- optimisan wrote:
I think we also need a mechanism to override adding RegBankSelect for targets, AMDGPU switches between custom passes and the default one. https://github.com/llvm/llvm-project/pull/116913 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
