================ @@ -0,0 +1,950 @@ +//===- Construction of code generation pass pipelines ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file provides the implementation of the PassBuilder based on our +/// static pass registry as well as related functionality. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/CodeGenPrepare.h" +#include "llvm/CodeGen/DwarfEHPrepare.h" +#include "llvm/CodeGen/ExpandLargeDivRem.h" +#include "llvm/CodeGen/ExpandLargeFpConvert.h" +#include "llvm/CodeGen/ExpandMemCmp.h" +#include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/FinalizeISel.h" +#include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GlobalMergeFunctions.h" +#include "llvm/CodeGen/LiveVariables.h" +#include "llvm/CodeGen/LocalStackSlotAllocation.h" +#include "llvm/CodeGen/LowerEmuTLS.h" +#include "llvm/CodeGen/MachineFunctionAnalysis.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/ReplaceWithVeclib.h" +#include "llvm/CodeGen/SafeStack.h" +#include "llvm/CodeGen/SelectOptimize.h" +#include "llvm/CodeGen/ShadowStackGCLowering.h" +#include "llvm/CodeGen/SjLjEHPrepare.h" +#include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/UnreachableBlockElim.h" +#include "llvm/CodeGen/WasmEHPrepare.h" +#include "llvm/CodeGen/WinEHPrepare.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only! +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/ObjCARC.h" +#include "llvm/Transforms/Scalar/ConstantHoisting.h" +#include "llvm/Transforms/Scalar/LoopStrengthReduce.h" +#include "llvm/Transforms/Scalar/MergeICmps.h" +#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" +#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" +#include "llvm/Transforms/Utils/LowerGlobalDtors.h" +#include "llvm/Transforms/Utils/LowerInvoke.h" + +namespace llvm { +extern cl::opt<std::string> FSRemappingFile; +} + +using namespace llvm; + +void PassBuilder::invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM) { + for (auto &C : CodeGenIREarlyEPCallbacks) + C(MPM); +} + +void PassBuilder::invokeGCLoweringEPCallbacks(FunctionPassManager &FPM) { + for (auto &C : GCLoweringEPCallbacks) + C(FPM); +} + +void PassBuilder::invokeISelPrepareEPCallbacks(ModulePassManager &MPM) { + for (auto &C : ISelPrepareEPCallbacks) + C(MPM); +} + +void PassBuilder::invokeMachineSSAOptimizationEarlyEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineSSAOptimizationEarlyEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMachineSSAOptimizationLastEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineSSAOptimizationLastEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreRegAllocEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreRegAllocEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreRegBankSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreRegBankSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreGlobalInstructionSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostGlobalInstructionSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : ILPOptsEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMachineLateOptimizationEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineLateOptimizationEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : MIEmitEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : PreEmitEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostRegAllocEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostRegAllocEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : PreSched2EPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostBBSectionsEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostBBSectionsEPCallbacks) + C(MFPM); +} + +void PassBuilder::addDefaultCodeGenPreparePasses(ModulePassManager &MPM) { + FunctionPassManager FPM; + // CodeGen prepare + if (TM->getOptLevel() != CodeGenOptLevel::None && !CGPBO.DisableCGP) + FPM.addPass(CodeGenPreparePass(TM)); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); +} + +Error PassBuilder::addDefaultRegAllocFastPasses( + MachineFunctionPassManager &MFPM) { + MFPM.addPass(PHIEliminationPass()); + MFPM.addPass(TwoAddressInstructionPass()); + if (auto Err = addRegAllocPass(MFPM)) + return Err; + return Error::success(); +} + +Error PassBuilder::addDefaultRegAllocOptimizedPasses( + MachineFunctionPassManager &MFPM) { + MFPM.addPass(DetectDeadLanesPass()); + MFPM.addPass(InitUndefPass()); + MFPM.addPass(ProcessImplicitDefsPass()); + + // LiveVariables currently requires pure SSA form. + // + // FIXME: Once TwoAddressInstruction pass no longer uses kill flags, + // LiveVariables can be removed completely, and LiveIntervals can be + // directly computed. (We still either need to regenerate kill flags after + // regalloc, or preferably fix the scavenger to not depend on them). + // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables. + // When LiveVariables is removed this has to be removed/moved either. + // Explicit addition of UnreachableMachineBlockElim allows stopping before + // or after it with -stop-before/-stop-after. + MFPM.addPass(UnreachableMachineBlockElimPass()); + // FIXME: Some X86 tests failed because of incomplete pipeline. + // MFPM.addPass(RequireAnalysisPass<LiveVariablesAnalysis, + // MachineFunction>()); + + // Edge splitting is smarter with machine loop info. + MFPM.addPass(RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>()); + MFPM.addPass(PHIEliminationPass()); + + if (CGPBO.EarlyLiveIntervals) + MFPM.addPass(RequireAnalysisPass<LiveIntervalsAnalysis, MachineFunction>()); + + MFPM.addPass(TwoAddressInstructionPass()); + MFPM.addPass(RegisterCoalescerPass()); + + // The machine scheduler may accidentally create disconnected components + // when moving subregister definitions around, avoid this by splitting them + // to separate vregs before. Splitting can also improve reg. allocation + // quality. + MFPM.addPass(RenameIndependentSubregsPass()); + + // PreRA instruction scheduling. + MFPM.addPass(MachineSchedulerPass()); + + if (auto Err = addRegAllocPass(MFPM)) + return Err; + // Finally rewrite virtual registers. + MFPM.addPass(VirtRegRewriterPass()); + + // Regalloc scoring for ML-driven eviction - noop except when learning a new + // eviction policy. + MFPM.addPass(RegAllocScoringPass()); + + return Error::success(); +} + +bool PassBuilder::isOptimizedRegAlloc() const { + return CGPBO.OptimizeRegAlloc.value_or(TM->getOptLevel() != + CodeGenOptLevel::None); +} + +// Find the Profile remapping file name. The internal option takes the +// precedence before getting from TargetMachine. +static std::string getFSRemappingFile(const TargetMachine *TM, + const CGPassBuilderOption &Options) { + if (!Options.FSRemappingFile.empty()) + return Options.FSRemappingFile; + const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileRemappingFile; +} + +// Find the FSProfile file name. The internal option takes the precedence +// before getting from TargetMachine. +static std::string getFSProfileFile(const TargetMachine *TM, + const CGPassBuilderOption &Options) { + if (!Options.FSProfileFile.empty()) + return Options.FSProfileFile; + const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileFile; +} + +Error PassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) { + const MCAsmInfo *MCAI = TM->getMCAsmInfo(); + if (!MCAI) + return make_error<StringError>("No MCAsmInfo!", inconvertibleErrorCode()); + switch (MCAI->getExceptionHandlingType()) { + case ExceptionHandling::SjLj: + // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both + // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, + // catch info can get misplaced when a selector ends up more than one block + // removed from the parent invoke(s). This could happen when a landing + // pad is shared by multiple invokes and is also a target of a normal + // edge from elsewhere. + FPM.addPass(SjLjEHPreparePass(TM)); + [[fallthrough]]; + case ExceptionHandling::DwarfCFI: + case ExceptionHandling::ARM: + case ExceptionHandling::AIX: + case ExceptionHandling::ZOS: + FPM.addPass(DwarfEHPreparePass(TM)); + break; + case ExceptionHandling::WinEH: + // We support using both GCC-style and MSVC-style exceptions on Windows, so + // add both preparation passes. Each pass will only actually run if it + // recognizes the personality function. + FPM.addPass(WinEHPreparePass()); + FPM.addPass(DwarfEHPreparePass(TM)); + break; + case ExceptionHandling::Wasm: + // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs + // on catchpads and cleanuppads because it does not outline them into + // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we + // should remove PHIs there. + FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true)); + FPM.addPass(WasmEHPreparePass()); + break; + case ExceptionHandling::None: + FPM.addPass(LowerInvokePass()); + + // The lower invoke pass may create unreachable code. Remove it. + FPM.addPass(UnreachableBlockElimPass()); + break; + } + return Error::success(); +} + +Error PassBuilder::addInstructionSelectorPasses( + MachineFunctionPassManager &MFPM) { + CodeGenOptLevel OptLevel = TM->getOptLevel(); + + // Core ISel + // Enable FastISel with -fast-isel, but allow that to be overridden. + TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true)); + // Determine an instruction selector. + enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; + SelectorType Selector; + + CGPBO.EnableFastISelOption.value_or(false); + if (CGPBO.EnableFastISelOption.value_or(false)) + Selector = SelectorType::FastISel; + + else if (CGPBO.EnableGlobalISelOption.value_or(false) || + (TM->Options.EnableGlobalISel && + !CGPBO.EnableGlobalISelOption.value_or(false))) + Selector = SelectorType::GlobalISel; + else if (OptLevel == CodeGenOptLevel::None && TM->getO0WantsFastISel()) + Selector = SelectorType::FastISel; + else + Selector = SelectorType::SelectionDAG; + + // Set consistently TM.Options.EnableFastISel and EnableGlobalISel. + if (Selector == SelectorType::FastISel) { + TM->setFastISel(true); + TM->setGlobalISel(false); + } else if (Selector == SelectorType::GlobalISel) { + TM->setFastISel(false); + TM->setGlobalISel(true); + } + + // Add instruction selector passes. + if (Selector == SelectorType::GlobalISel) { + MFPM.addPass(IRTranslatorPass()); + MFPM.addPass(LegalizerPass()); + + // Before running the register bank selector, ask the target if it + // wants to run some passes. + invokePreRegBankSelectEPCallbacks(MFPM); + MFPM.addPass(RegBankSelectPass()); + + invokePreGlobalInstructionSelectEPCallbacks(MFPM); + MFPM.addPass(InstructionSelectPass()); + invokePostGlobalInstructionSelectEPCallbacks(MFPM); + + // Pass to reset the MachineFunction if the ISel failed. + MFPM.addPass(ResetMachineFunctionPass( + TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag, + TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable)); + + // Provide a fallback path when we do not want to abort on + // not-yet-supported input. + if (TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) { + if (!AddInstSelectorCallback) + return make_error<StringError>("No InstSelectorCallback!", + inconvertibleErrorCode()); + AddInstSelectorCallback(MFPM); + } + } else { + if (!AddInstSelectorCallback) + return make_error<StringError>("No InstSelectorCallback!", + inconvertibleErrorCode()); + AddInstSelectorCallback(MFPM); + } + return Error::success(); +} + +void PassBuilder::addMachineSSAOptimizationPasses( + MachineFunctionPassManager &MFPM) { + // Pre-ra tail duplication. + MFPM.addPass(EarlyTailDuplicatePass()); + + // Optimize PHIs before DCE: removing dead PHI cycles may make more + // instructions dead. + MFPM.addPass(OptimizePHIsPass()); + + // This pass merges large allocas. StackSlotColoring is a different pass + // which merges spill slots. + MFPM.addPass(StackColoringPass()); + + // If the target requests it, assign local variables to stack slots relative + // to one another and simplify frame index references where possible. + MFPM.addPass(LocalStackSlotAllocationPass()); + + // With optimization, dead code should already be eliminated. However + // there is one known exception: lowered code for arguments that are only + // used by tail calls, where the tail calls reuse the incoming stack + // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). + MFPM.addPass(DeadMachineInstructionElimPass()); + + // Allow targets to insert passes that improve instruction level parallelism, + // like if-conversion. Such passes will typically need dominator trees and + // loop info, just like LICM and CSE below. + invokeILPOptsEPCallbacks(MFPM); + + MFPM.addPass(EarlyMachineLICMPass()); + MFPM.addPass(MachineCSEPass()); + MFPM.addPass(MachineSinkingPass()); + MFPM.addPass(PeepholeOptimizerPass()); + // Clean-up the dead code that may have been generated by peephole + // rewriting. + MFPM.addPass(DeadMachineInstructionElimPass()); +} + +static Error setStartStop(ModulePassManager &MPM, + PassInstrumentationCallbacks *PIC) { + auto StartStopInfoOrErr = TargetPassConfig::getStartStopInfo(); + if (!StartStopInfoOrErr) + return StartStopInfoOrErr.takeError(); + auto &SSI = *StartStopInfoOrErr; + + if (SSI.StartPass.empty() && SSI.StopPass.empty()) + return Error::success(); + if (!PIC) { + return make_error<StringError>("Need PassInstrumentationCallbacks!", + inconvertibleErrorCode()); + } + + static const std::vector<StringRef> SpecialPasses = { + "InvalidateAnalysisPass", "MachineVerifierPass", "PrintMIRPass", + "PrintMIRPreparePass", "RequireAnalysisPass", "VerifierPass"}; + + bool Started = SSI.StartPass.empty(), Stopped = false; + // Return true if pass is skipped. + auto Filter = [&, StartInstanceNum = 0u, + StopInstanceNum = 0u](StringRef Name) mutable { + if (isSpecialPass(Name, SpecialPasses)) + return false; + + bool ShouldDrop = true; + StringRef CurPassName = PIC->getPassNameForClassName(Name); + + // Set instance counters correctly. + if (!SSI.StartPass.empty() && CurPassName == SSI.StartPass) { + ++StartInstanceNum; + if (StartInstanceNum == SSI.StartInstanceNum) + Started = true; + } + if (!SSI.StopPass.empty() && CurPassName == SSI.StopPass) { + ++StopInstanceNum; + if (StopInstanceNum == SSI.StopInstanceNum) + Stopped = true; + } + + // Border case. + const bool AtStartBorder = !SSI.StartPass.empty() && Started && + CurPassName == SSI.StartPass && + StartInstanceNum == SSI.StartInstanceNum; + const bool AtStopBorder = !SSI.StopPass.empty() && Stopped && + CurPassName == SSI.StopPass && + StopInstanceNum == SSI.StopInstanceNum; + if (AtStartBorder) + ShouldDrop = SSI.StartAfter; + if (AtStopBorder) + ShouldDrop = !SSI.StopAfter; + if (!AtStartBorder && !AtStopBorder) + ShouldDrop = !Started || Stopped; + + return ShouldDrop; + }; + + MPM.eraseIf(Filter); + if (!Started) { + return make_error<StringError>( + "Can't find start pass \"" + SSI.StartPass + "\".", + std::make_error_code(std::errc::invalid_argument)); + } + if (!Stopped && !SSI.StopPass.empty()) { + return make_error<StringError>( + "Can't find stop pass \"" + SSI.StopPass + "\".", + std::make_error_code(std::errc::invalid_argument)); + } + return Error::success(); +} + +Error PassBuilder::addRegisterAllocatorPasses( + MachineFunctionPassManager &MFPM) { + return isOptimizedRegAlloc() ? AddRegAllocOptimizedCallback(MFPM) + : AddRegAllocFastCallback(MFPM); +} + +Error PassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM, + StringRef FilterName) { + std::optional<RegAllocFilterFunc> FilterFunc = + parseRegAllocFilter(FilterName); + if (!FilterFunc) { + return make_error<StringError>( + formatv("Unknown register filter name: {0}", FilterName).str(), + std::make_error_code(std::errc::invalid_argument)); + } + + if (RegAllocPasses.contains(FilterName)) { + MFPM.addPass(std::move(RegAllocPasses[FilterName])); + return Error::success(); + } + + // Add default register allocator. + if (isOptimizedRegAlloc()) { + MFPM.addPass(RAGreedyPass()); + } else { + RegAllocFastPassOptions Opts; + Opts.Filter = *FilterFunc; + Opts.FilterName = FilterName; + MFPM.addPass(RegAllocFastPass(Opts)); + } + return Error::success(); +} + +Error PassBuilder::addMachinePasses(ModulePassManager &MPM, + FunctionPassManager &FPM, + MachineFunctionPassManager &MFPM) { + CodeGenOptLevel OptLevel = TM->getOptLevel(); + + // Expand pseudo-instructions emitted by ISel. Don't run the verifier before + // FinalizeISel. + MFPM.addPass(FinalizeISelPass()); + + // Add passes that optimize machine instructions in SSA form. + if (OptLevel != CodeGenOptLevel::None) { + invokeMachineSSAOptimizationEarlyEPCallbacks(MFPM); + addMachineSSAOptimizationPasses(MFPM); + invokeMachineSSAOptimizationLastEPCallbacks(MFPM); + } else { + MFPM.addPass(LocalStackSlotAllocationPass()); + } + + if (TM->Options.EnableIPRA) + MFPM.addPass(RegUsageInfoPropagationPass()); + + // Run pre-ra passes. + invokePreRegAllocEPCallbacks(MFPM); + + if (EnableFSDiscriminator) { + MFPM.addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1)); + const std::string ProfileFile = getFSProfileFile(TM, CGPBO); + if (!ProfileFile.empty() && !CGPBO.DisableRAFSProfileLoader) + MFPM.addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(TM, CGPBO), + sampleprof::FSDiscriminatorPass::Pass1, nullptr)); + } + + if (auto Err = addRegisterAllocatorPasses(MFPM)) + return Err; + + invokePostRegAllocEPCallbacks(MFPM); + + MFPM.addPass(RemoveRedundantDebugValuesPass()); + MFPM.addPass(FixupStatepointCallerSavedPass()); + + // Insert prolog/epilog code. Eliminate abstract frame index references... + if (OptLevel != CodeGenOptLevel::None) { + MFPM.addPass(PostRAMachineSinkingPass()); + MFPM.addPass(ShrinkWrapPass()); + } + + if (!CGPBO.DisablePrologEpilogInserterPass) + MFPM.addPass(PrologEpilogInserterPass()); + /// Add passes that optimize machine instructions after register allocation. + if (OptLevel != CodeGenOptLevel::None) + invokeMachineLateOptimizationEPCallbacks(MFPM); + + // Expand pseudo instructions before second scheduling pass. + MFPM.addPass(ExpandPostRAPseudosPass()); + + // Run pre-sched2 passes. + invokePreSched2EPCallbacks(MFPM); + + if (CGPBO.EnableImplicitNullChecks) + MFPM.addPass(ImplicitNullChecksPass()); + + // Second pass scheduler. + // Let Target optionally insert this pass by itself at some other + // point. + if (OptLevel != CodeGenOptLevel::None && + !TM->targetSchedulesPostRAScheduling()) { + if (CGPBO.MISchedPostRA) + MFPM.addPass(PostMachineSchedulerPass()); + else + MFPM.addPass(PostRASchedulerPass()); + } + + // GC, replacement for GCMachineCodeAnalysis + MFPM.addPass(GCMachineCodeInsertionPass()); + + // Basic block placement. + if (OptLevel != CodeGenOptLevel::None) { + if (EnableFSDiscriminator) { + MFPM.addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2)); + const std::string ProfileFile = getFSProfileFile(TM, CGPBO); + if (!ProfileFile.empty() && !CGPBO.DisableLayoutFSProfileLoader) + MFPM.addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(TM, CGPBO), + sampleprof::FSDiscriminatorPass::Pass2, nullptr)); + } + MFPM.addPass(MachineBlockPlacementPass()); + // Run a separate pass to collect block placement statistics. + if (CGPBO.EnableBlockPlacementStats) + MFPM.addPass(MachineBlockPlacementStatsPass()); + } + + // Insert before XRay Instrumentation. + MFPM.addPass(FEntryInserterPass()); + MFPM.addPass(XRayInstrumentationPass()); + MFPM.addPass(PatchableFunctionPass()); + + invokePreEmitEPCallbacks(MFPM); + + if (TM->Options.EnableIPRA) + // Collect register usage information and produce a register mask of + // clobbered registers, to be used to optimize call sites. + MFPM.addPass(RegUsageInfoCollectorPass()); + + // FIXME: Some backends are incompatible with running the verifier after + // addPreEmitPass. Maybe only pass "false" here for those targets? + MFPM.addPass(FuncletLayoutPass()); + + MFPM.addPass(StackMapLivenessPass()); + MFPM.addPass(LiveDebugValuesPass()); + MFPM.addPass(MachineSanitizerBinaryMetadata()); + + if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None && + CGPBO.EnableMachineOutliner != RunOutliner::NeverOutline) { + bool RunOnAllFunctions = + (CGPBO.EnableMachineOutliner == RunOutliner::AlwaysOutline); + bool AddOutliner = + RunOnAllFunctions || TM->Options.SupportsDefaultOutlining; + if (AddOutliner) { + FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))); + if (CGPBO.RequiresCodeGenSCCOrder) + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + createCGSCCToFunctionPassAdaptor(std::move(FPM)))); + else + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(MachineOutlinerPass(RunOnAllFunctions)); + FPM = FunctionPassManager(); + MFPM = MachineFunctionPassManager(); + } + } + + if (CGPBO.GCEmptyBlocks) + MFPM.addPass(GCEmptyBasicBlocksPass()); + + if (EnableFSDiscriminator) + MFPM.addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast)); + + bool NeedsBBSections = + TM->getBBSectionsType() != llvm::BasicBlockSection::None; + // Machine function splitter uses the basic block sections feature. Both + // cannot be enabled at the same time. We do not apply machine function + // splitter if -basic-block-sections is requested. + if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter || + CGPBO.EnableMachineFunctionSplitter)) { + const std::string ProfileFile = getFSProfileFile(TM, CGPBO); + if (!ProfileFile.empty()) { + if (EnableFSDiscriminator) { + MFPM.addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(TM, CGPBO), + sampleprof::FSDiscriminatorPass::PassLast, nullptr)); + } else { + // Sample profile is given, but FSDiscriminator is not + // enabled, this may result in performance regression. + WithColor::warning() + << "Using AutoFDO without FSDiscriminator for MFS may regress " + "performance.\n"; + } + } + MFPM.addPass(MachineFunctionSplitterPass()); + } + + // We run the BasicBlockSections pass if either we need BB sections or BB + // address map (or both). + if (NeedsBBSections || TM->Options.BBAddrMap) { + if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) + MFPM.addPass(BasicBlockPathCloningPass()); + MFPM.addPass(BasicBlockSectionsPass()); + } + + invokePostBBSectionsEPCallbacks(MFPM); + + if (!CGPBO.DisableCFIFixup && TM->Options.EnableCFIFixup) + MFPM.addPass(CFIFixupPass()); + + MFPM.addPass(StackFrameLayoutAnalysisPass()); + + // Add passes that directly emit MI after all other MI passes. + invokeMIEmitEPCallbacks(MFPM); + + return Error::success(); +} + +Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &TopLevelMPM, + raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, + CodeGenFileType FileType, + MCContext &Ctx) { + if (!TM) + return make_error<StringError>("Need a TargetMachine instance!", + inconvertibleErrorCode()); + + if (CustomCodeGenPipelineBuilderCallback) { + ModulePassManager MPM; + if (auto Err = CustomCodeGenPipelineBuilderCallback(MPM, Out, DwoOut, + FileType, Ctx)) + return Err; + MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); }); + if (auto Err = setStartStop(MPM, PIC)) + return Err; + TopLevelMPM.addPass(std::move(MPM)); + return Error::success(); + } + + CodeGenOptLevel OptLevel = TM->getOptLevel(); + if (auto Err = parseRegAllocOption(CGPBO.RegAlloc)) + return Err; + + bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline(); + bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null; + + ModulePassManager MPM; + FunctionPassManager FPM; + MachineFunctionPassManager MFPM; + + if (!CGPBO.DisableVerify) + MPM.addPass(VerifierPass()); + + // IR part + // TODO: Remove RequireAnalysisPass<MachineModuleAnalysis, Module>() + // when we port AsmPrinter. + MPM.addPass(RequireAnalysisPass<MachineModuleAnalysis, Module>()); + MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); + MPM.addPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>()); + + invokeCodeGenIREarlyEPCallbacks(MPM); + + if (TM->useEmulatedTLS()) + MPM.addPass(LowerEmuTLSPass()); + MPM.addPass(PreISelIntrinsicLoweringPass(TM)); + + // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with + // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. + if (TM->getTargetTriple().isOSBinFormatMachO() && + !CGPBO.DisableAtExitBasedGlobalDtorLowering) + MPM.addPass(LowerGlobalDtorsPass()); + + FPM.addPass(ExpandLargeDivRemPass(TM)); + FPM.addPass(ExpandLargeFpConvertPass(TM)); + + // Run loop strength reduction before anything else. + if (OptLevel != CodeGenOptLevel::None) { + if (!CGPBO.DisableLSR) { + LoopPassManager LPM; + LPM.addPass(LoopStrengthReducePass()); + FPM.addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), + /*UseMemorySSA=*/true)); + } ---------------- optimisan wrote:
`LPM` looks to be unused here? https://github.com/llvm/llvm-project/pull/116913 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
