https://github.com/optimisan updated https://github.com/llvm/llvm-project/pull/115434
>From 17ae43cbf8e8aad79f3cba192079c3841e1425f5 Mon Sep 17 00:00:00 2001 From: Akshat Oke <akshat....@amd.com> Date: Wed, 30 Oct 2024 04:56:54 +0000 Subject: [PATCH 1/4] [CodeGen][NewPM] Port MachineSink to NPM Targets can set the EnableSinkAndFold option in CGPassBuilderOptions for the NPM pipeline in buildCodeGenPipeline(... &Opts, ...) --- llvm/include/llvm/CodeGen/MachineSink.h | 29 ++++ llvm/include/llvm/CodeGen/Passes.h | 2 +- llvm/include/llvm/InitializePasses.h | 2 +- llvm/include/llvm/Passes/CodeGenPassBuilder.h | 3 +- .../llvm/Passes/MachinePassRegistry.def | 9 +- .../include/llvm/Target/CGPassBuilderOption.h | 1 + llvm/lib/CodeGen/CodeGen.cpp | 2 +- llvm/lib/CodeGen/MachineSink.cpp | 136 ++++++++++++------ llvm/lib/CodeGen/TargetPassConfig.cpp | 4 +- llvm/lib/Passes/PassBuilder.cpp | 6 + llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 2 +- llvm/test/CodeGen/AArch64/loop-sink.mir | 1 + .../sink-and-fold-preserve-debugloc.mir | 2 + ...e-sink-temporal-divergence-swdev407790.mir | 2 + .../CodeGen/ARM/machine-sink-multidef.mir | 2 + .../Hexagon/machine-sink-float-usr.mir | 2 + .../PowerPC/sink-down-more-instructions-1.mir | 2 + .../CodeGen/RISCV/MachineSink-implicit-x0.mir | 1 + .../CodeGen/SystemZ/machinesink-dead-cc.mir | 3 + .../CodeGen/X86/machinesink-debug-inv-0.mir | 3 + .../DebugInfo/MIR/X86/sink-leaves-undef.mir | 1 + 21 files changed, 166 insertions(+), 49 deletions(-) create mode 100644 llvm/include/llvm/CodeGen/MachineSink.h diff --git a/llvm/include/llvm/CodeGen/MachineSink.h b/llvm/include/llvm/CodeGen/MachineSink.h new file mode 100644 index 0000000000000..1eee9d7f7e2a4 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineSink.h @@ -0,0 +1,29 @@ +//===- MachineSink.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINESINK_H +#define LLVM_CODEGEN_MACHINESINK_H + +#include "llvm/CodeGen/MachinePassManager.h" + +namespace llvm { + +class MachineSinkingPass : public PassInfoMixin<MachineSinkingPass> { + bool EnableSinkAndFold; + +public: + MachineSinkingPass(bool EnableSinkAndFold = false) + : EnableSinkAndFold(EnableSinkAndFold) {} + + PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &); + + void printPipeline(raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName); +}; + +} // namespace llvm +#endif // LLVM_CODEGEN_MACHINESINK_H diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index b5d2a7e6bf035..581c38e5c1a52 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -353,7 +353,7 @@ namespace llvm { extern char &EarlyMachineLICMID; /// MachineSinking - This pass performs sinking on machine instructions. - extern char &MachineSinkingID; + extern char &MachineSinkingLegacyID; /// MachineCopyPropagation - This pass performs copy propagation on /// machine instructions. diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 30c7402bd6606..5c45a405663b1 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -208,7 +208,7 @@ void initializeMachinePostDominatorTreeWrapperPassPass(PassRegistry &); void initializeMachineRegionInfoPassPass(PassRegistry &); void initializeMachineSanitizerBinaryMetadataPass(PassRegistry &); void initializeMachineSchedulerLegacyPass(PassRegistry &); -void initializeMachineSinkingPass(PassRegistry &); +void initializeMachineSinkingLegacyPass(PassRegistry &); void initializeMachineTraceMetricsWrapperPassPass(PassRegistry &); void initializeMachineUniformityInfoPrinterPassPass(PassRegistry &); void initializeMachineUniformityAnalysisPassPass(PassRegistry &); diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 12781e2b84623..1967a323129c1 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -51,6 +51,7 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/MachineScheduler.h" +#include "llvm/CodeGen/MachineSink.h" #include "llvm/CodeGen/MachineVerifier.h" #include "llvm/CodeGen/OptimizePHIs.h" #include "llvm/CodeGen/PHIElimination.h" @@ -1042,7 +1043,7 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineSSAOptimization( addPass(EarlyMachineLICMPass()); addPass(MachineCSEPass()); - addPass(MachineSinkingPass()); + addPass(MachineSinkingPass(Opt.EnableSinkAndFold)); addPass(PeepholeOptimizerPass()); // Clean-up the dead code that may have been generated by peephole diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 8183ba90ac2a9..1281aa6748b80 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -189,6 +189,13 @@ MACHINE_FUNCTION_PASS("verify<machine-trace-metrics>", MachineTraceMetricsVerifi #define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ PARAMS) #endif +MACHINE_FUNCTION_PASS_WITH_PARAMS( + "machine-sink", "MachineSinkingPass", + [](bool EnableSinkAndFold) { + return MachineSinkingPass(EnableSinkAndFold); + }, + parseMachineSinkingPassOptions, "enable-sink-fold") + MACHINE_FUNCTION_PASS_WITH_PARAMS( "regallocfast", "RegAllocFastPass", [](RegAllocFastPassOptions Opts) { return RegAllocFastPass(Opts); }, @@ -196,6 +203,7 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS( return parseRegAllocFastPassOptions(*PB, Params); }, "filter=reg-filter;no-clear-vregs") + #undef MACHINE_FUNCTION_PASS_WITH_PARAMS // After a pass is converted to new pass manager, its entry should be moved from @@ -249,7 +257,6 @@ DUMMY_MACHINE_FUNCTION_PASS("static-data-splitter", StaticDataSplitter) DUMMY_MACHINE_FUNCTION_PASS("machine-function-splitter", MachineFunctionSplitterPass) DUMMY_MACHINE_FUNCTION_PASS("machine-latecleanup", MachineLateInstrsCleanupPass) DUMMY_MACHINE_FUNCTION_PASS("machine-sanmd", MachineSanitizerBinaryMetadata) -DUMMY_MACHINE_FUNCTION_PASS("machine-sink", MachineSinkingPass) DUMMY_MACHINE_FUNCTION_PASS("machine-uniformity", MachineUniformityInfoWrapperPass) DUMMY_MACHINE_FUNCTION_PASS("machineinstr-printer", MachineFunctionPrinterPass) DUMMY_MACHINE_FUNCTION_PASS("mirfs-discriminators", MIRAddFSDiscriminatorsPass) diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index d3d19c8a7dc9f..81cd1142afa5d 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -33,6 +33,7 @@ struct CGPassBuilderOption { bool EnableBlockPlacementStats = false; bool EnableGlobalMergeFunc = false; bool EnableMachineFunctionSplitter = false; + bool EnableSinkAndFold = false; bool MISchedPostRA = false; bool EarlyLiveIntervals = false; bool GCEmptyBlocks = false; diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp index 4907fd93b2ae5..3726c01bec532 100644 --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -95,7 +95,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeMachinePostDominatorTreeWrapperPassPass(Registry); initializeMachineRegionInfoPassPass(Registry); initializeMachineSchedulerLegacyPass(Registry); - initializeMachineSinkingPass(Registry); + initializeMachineSinkingLegacyPass(Registry); initializeMachineUniformityAnalysisPassPass(Registry); initializeMachineUniformityInfoPrinterPassPass(Registry); initializeMachineVerifierLegacyPassPass(Registry); diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index 82acb780cfb72..18f3740b01298 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -15,6 +15,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/MachineSink.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/MapVector.h" @@ -37,6 +38,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineSizeOpts.h" @@ -47,6 +49,7 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSchedule.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/IR/Analysis.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/LLVMContext.h" @@ -118,7 +121,7 @@ using RegSubRegPair = TargetInstrInfo::RegSubRegPair; namespace { -class MachineSinking : public MachineFunctionPass { +class MachineSinking { const TargetSubtargetInfo *STI = nullptr; const TargetInstrInfo *TII = nullptr; const TargetRegisterInfo *TRI = nullptr; @@ -132,6 +135,8 @@ class MachineSinking : public MachineFunctionPass { AliasAnalysis *AA = nullptr; RegisterClassInfo RegClassInfo; TargetSchedModel SchedModel; + Pass *LegacyPass; + MachineFunctionAnalysisManager *MFAM; // Remember which edges have been considered for breaking. SmallSet<std::pair<MachineBasicBlock *, MachineBasicBlock *>, 8> @@ -189,30 +194,14 @@ class MachineSinking : public MachineFunctionPass { bool EnableSinkAndFold; public: - static char ID; // Pass identification + MachineSinking(Pass *LegacyPass, MachineFunctionAnalysisManager *MFAM, + bool EnableSinkAndFold) + : LegacyPass(LegacyPass), MFAM(MFAM), + EnableSinkAndFold(EnableSinkAndFold) {} - MachineSinking() : MachineFunctionPass(ID) { - initializeMachineSinkingPass(*PassRegistry::getPassRegistry()); - } - - bool runOnMachineFunction(MachineFunction &MF) override; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - MachineFunctionPass::getAnalysisUsage(AU); - AU.addRequired<AAResultsWrapperPass>(); - AU.addRequired<MachineDominatorTreeWrapperPass>(); - AU.addRequired<MachinePostDominatorTreeWrapperPass>(); - AU.addRequired<MachineCycleInfoWrapperPass>(); - AU.addRequired<MachineBranchProbabilityInfoWrapperPass>(); - AU.addPreserved<MachineCycleInfoWrapperPass>(); - AU.addPreserved<MachineLoopInfoWrapperPass>(); - AU.addRequired<ProfileSummaryInfoWrapperPass>(); - if (UseBlockFreqInfo) - AU.addRequired<MachineBlockFrequencyInfoWrapperPass>(); - AU.addRequired<TargetPassConfig>(); - } + bool run(MachineFunction &MF); - void releaseMemory() override { + void releaseMemory() { CEBCandidates.clear(); CEMergeCandidates.clear(); } @@ -290,21 +279,47 @@ class MachineSinking : public MachineFunctionPass { bool registerPressureExceedsLimit(const MachineBasicBlock &MBB); }; +class MachineSinkingLegacy : public MachineFunctionPass { +public: + static char ID; + + MachineSinkingLegacy() : MachineFunctionPass(ID) { + initializeMachineSinkingLegacyPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + MachineFunctionPass::getAnalysisUsage(AU); + AU.addRequired<AAResultsWrapperPass>(); + AU.addRequired<MachineDominatorTreeWrapperPass>(); + AU.addRequired<MachinePostDominatorTreeWrapperPass>(); + AU.addRequired<MachineCycleInfoWrapperPass>(); + AU.addRequired<MachineBranchProbabilityInfoWrapperPass>(); + AU.addPreserved<MachineCycleInfoWrapperPass>(); + AU.addPreserved<MachineLoopInfoWrapperPass>(); + AU.addRequired<ProfileSummaryInfoWrapperPass>(); + if (UseBlockFreqInfo) + AU.addRequired<MachineBlockFrequencyInfoWrapperPass>(); + AU.addRequired<TargetPassConfig>(); + } +}; + } // end anonymous namespace -char MachineSinking::ID = 0; +char MachineSinkingLegacy::ID = 0; -char &llvm::MachineSinkingID = MachineSinking::ID; +char &llvm::MachineSinkingLegacyID = MachineSinkingLegacy::ID; -INITIALIZE_PASS_BEGIN(MachineSinking, DEBUG_TYPE, "Machine code sinking", false, +INITIALIZE_PASS_BEGIN(MachineSinkingLegacy, DEBUG_TYPE, "Machine code sinking", false, false) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) -INITIALIZE_PASS_END(MachineSinking, DEBUG_TYPE, "Machine code sinking", false, - false) +INITIALIZE_PASS_END(MachineSinkingLegacy, DEBUG_TYPE, "Machine code sinking", + false, false) /// Return true if a target defined block prologue instruction interferes /// with a sink candidate. @@ -728,28 +743,66 @@ void MachineSinking::FindCycleSinkCandidates( } } -bool MachineSinking::runOnMachineFunction(MachineFunction &MF) { +PreservedAnalyses +MachineSinkingPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + MachineSinking Impl(nullptr, &MFAM, EnableSinkAndFold); + bool Changed = Impl.run(MF); + if (!Changed) + return PreservedAnalyses::all(); + auto PA = getMachineFunctionPassPreservedAnalyses(); + PA.preserve<MachineCycleAnalysis>(); + PA.preserve<MachineLoopAnalysis>(); + return PA; +} + +void MachineSinkingPass::printPipeline( + raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) { + OS << "machine-sink"; + if (EnableSinkAndFold) + OS << "<enable-sink-fold>"; +} + +bool MachineSinkingLegacy::runOnMachineFunction(MachineFunction &MF) { if (skipFunction(MF.getFunction())) return false; + TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>(); + bool EnableSinkAndFold = PassConfig->getEnableSinkAndFold(); + + MachineSinking Impl(this, nullptr, EnableSinkAndFold); + return Impl.run(MF); +} + +#define GET_ANALYSIS(ANALYSIS, INFIX, GETTER) \ + ((LegacyPass) \ + ? &LegacyPass->getAnalysis<ANALYSIS##INFIX##WrapperPass>().GETTER() \ + : &MFAM->getResult<ANALYSIS##Analysis>(MF)) + +bool MachineSinking::run(MachineFunction &MF) { LLVM_DEBUG(dbgs() << "******** Machine Sinking ********\n"); STI = &MF.getSubtarget(); TII = STI->getInstrInfo(); TRI = STI->getRegisterInfo(); MRI = &MF.getRegInfo(); - DT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree(); - PDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree(); - CI = &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo(); - PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); - MBFI = UseBlockFreqInfo - ? &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI() - : nullptr; - MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); - AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); + DT = GET_ANALYSIS(MachineDominatorTree, , getDomTree); + PDT = GET_ANALYSIS(MachinePostDominatorTree, , getPostDomTree); + CI = GET_ANALYSIS(MachineCycle, Info, getCycleInfo); + PSI = (LegacyPass) + ? &LegacyPass->getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI() + : MFAM->getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF) + .getCachedResult<ProfileSummaryAnalysis>(*MF.getFunction().getParent()); + MBFI = UseBlockFreqInfo ? GET_ANALYSIS(MachineBlockFrequency, Info, getMBFI) + : nullptr; + MBPI = GET_ANALYSIS(MachineBranchProbability, Info, getMBPI); + AA = (LegacyPass) + ? &LegacyPass->getAnalysis<AAResultsWrapperPass>().getAAResults() + : &MFAM->getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF) + .getManager() + .getResult<AAManager>(MF.getFunction()); + RegClassInfo.runOnMachineFunction(MF); - TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>(); - EnableSinkAndFold = PassConfig->getEnableSinkAndFold(); bool EverMadeChange = false; @@ -768,7 +821,7 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) { MachineDomTreeUpdater::UpdateStrategy::Lazy); for (const auto &Pair : ToSplit) { auto NewSucc = - Pair.first->SplitCriticalEdge(Pair.second, *this, nullptr, &MDTU); + Pair.first->SplitCriticalEdge(Pair.second, LegacyPass, MFAM, nullptr, &MDTU); if (NewSucc != nullptr) { LLVM_DEBUG(dbgs() << " *** Splitting critical edge: " << printMBBReference(*Pair.first) << " -- " @@ -858,6 +911,7 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) { MRI->clearKillFlags(I); RegsToClearKillFlags.clear(); + releaseMemory(); return EverMadeChange; } diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index 5d9da9df9092a..ea5e43ff12166 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -324,7 +324,7 @@ static IdentifyingPassPtr overridePass(AnalysisID StandardID, if (StandardID == &MachineLICMID) return applyDisable(TargetID, DisablePostRAMachineLICM); - if (StandardID == &MachineSinkingID) + if (StandardID == &MachineSinkingLegacyID) return applyDisable(TargetID, DisableMachineSink); if (StandardID == &PostRAMachineSinkingID) @@ -1316,7 +1316,7 @@ void TargetPassConfig::addMachineSSAOptimization() { addPass(&EarlyMachineLICMID); addPass(&MachineCSELegacyID); - addPass(&MachineSinkingID); + addPass(&MachineSinkingLegacyID); addPass(&PeepholeOptimizerLegacyID); // Clean-up the dead code that may have been generated by peephole diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 0477aa01e241e..92762cbfe4d70 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -121,6 +121,7 @@ #include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineScheduler.h" +#include "llvm/CodeGen/MachineSink.h" #include "llvm/CodeGen/MachineTraceMetrics.h" #include "llvm/CodeGen/MachineVerifier.h" #include "llvm/CodeGen/OptimizePHIs.h" @@ -1414,6 +1415,11 @@ parseBoundsCheckingOptions(StringRef Params) { return Options; } +Expected<bool> parseMachineSinkingPassOptions(StringRef Params) { + return PassBuilder::parseSinglePassOption(Params, "enable-sink-fold", + "MachineSinkingPass"); +} + } // namespace /// Tests whether a pass name starts with a valid prefix for a default pipeline diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index f2afa6fc20bfa..c65c07dcd9532 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -506,7 +506,7 @@ void NVPTXPassConfig::addMachineSSAOptimization() { addPass(&EarlyMachineLICMID); addPass(&MachineCSELegacyID); - addPass(&MachineSinkingID); + addPass(&MachineSinkingLegacyID); printAndVerify("After Machine LICM, CSE and Sinking passes"); addPass(&PeepholeOptimizerLegacyID); diff --git a/llvm/test/CodeGen/AArch64/loop-sink.mir b/llvm/test/CodeGen/AArch64/loop-sink.mir index 36d39ffbadc29..de81d99369394 100644 --- a/llvm/test/CodeGen/AArch64/loop-sink.mir +++ b/llvm/test/CodeGen/AArch64/loop-sink.mir @@ -1,5 +1,6 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple aarch64 -run-pass=machine-sink -sink-insts-to-avoid-spills -aarch64-enable-sink-fold=true %s -o - 2>&1 | FileCheck %s +# RUN: llc -mtriple aarch64 -passes="machine-sink<enable-sink-fold>" -sink-insts-to-avoid-spills %s -o - 2>&1 | FileCheck %s --- | target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64" diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir b/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir index be5737d297b29..f146c4436bbfa 100644 --- a/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir +++ b/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir @@ -1,4 +1,6 @@ # RUN: llc %s -run-pass=machine-sink --aarch64-enable-sink-fold=true -o - | FileCheck %s + +# RUN: llc %s -passes="machine-sink<enable-sink-fold>" -o - | FileCheck %s --- | target triple = "aarch64-linux" diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir index 329f296712160..0732648f6c7a1 100644 --- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir +++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.mir @@ -1,6 +1,8 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 -run-pass=machine-sink -o - %s | FileCheck %s +# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 -passes="function(machine-function(machine-sink))" -o - %s | FileCheck %s + --- name: machine-sink-temporal-divergence tracksRegLiveness: true diff --git a/llvm/test/CodeGen/ARM/machine-sink-multidef.mir b/llvm/test/CodeGen/ARM/machine-sink-multidef.mir index 737ca824265d6..485e659a53ab6 100644 --- a/llvm/test/CodeGen/ARM/machine-sink-multidef.mir +++ b/llvm/test/CodeGen/ARM/machine-sink-multidef.mir @@ -1,6 +1,8 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc %s -o - -run-pass=machine-sink -mtriple=arm-none-eabi | FileCheck %s +# RUN: llc %s -o - -passes=machine-sink -mtriple=arm-none-eabi | FileCheck %s + --- | target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "arm-none-unknown-eabi" diff --git a/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir b/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir index f994493def0c9..fa05362b8e1e5 100644 --- a/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir +++ b/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir @@ -1,5 +1,7 @@ # RUN: llc -mtriple=hexagon -run-pass machine-sink -o - %s | FileCheck %s +# RUN: llc -march=hexagon -passes machine-sink -o - %s | FileCheck %s + # Test that MachineSink does not sink F2_conv_w2sf. # CHECK: name:{{.*}} main # CHECK: J2_call @feclearexcept diff --git a/llvm/test/CodeGen/PowerPC/sink-down-more-instructions-1.mir b/llvm/test/CodeGen/PowerPC/sink-down-more-instructions-1.mir index fe50d05ab93a5..ee16a8c69afb7 100644 --- a/llvm/test/CodeGen/PowerPC/sink-down-more-instructions-1.mir +++ b/llvm/test/CodeGen/PowerPC/sink-down-more-instructions-1.mir @@ -2,6 +2,8 @@ # RUN: llc -mtriple powerpc64le-unknown-linux-gnu -o - %s -verify-machineinstrs \ # RUN: -run-pass=machine-sink | FileCheck %s +# RUN: llc -mtriple powerpc64le-unknown-linux-gnu -o - %s -verify-machineinstrs \ +# RUN: -passes=machine-sink | FileCheck %s --- | ; ModuleID = 'sink-down-more-instructions-1.ll' source_filename = "sink-down-more-instructions-1.c" diff --git a/llvm/test/CodeGen/RISCV/MachineSink-implicit-x0.mir b/llvm/test/CodeGen/RISCV/MachineSink-implicit-x0.mir index ef006fca60a40..8f5c763e95020 100644 --- a/llvm/test/CodeGen/RISCV/MachineSink-implicit-x0.mir +++ b/llvm/test/CodeGen/RISCV/MachineSink-implicit-x0.mir @@ -1,4 +1,5 @@ # RUN: llc -mtriple=riscv32 %s -run-pass=machine-sink -o - | FileCheck %s +# RUN: llc -mtriple=riscv32 %s -passes=machine-sink -o - | FileCheck %s # Verify that sinking of '%20:gpr = LUI 1, implicit $x0' is not inhibited by # the implicit use of '$x0'. diff --git a/llvm/test/CodeGen/SystemZ/machinesink-dead-cc.mir b/llvm/test/CodeGen/SystemZ/machinesink-dead-cc.mir index 8516cc9114c76..7b62c0b49b816 100644 --- a/llvm/test/CodeGen/SystemZ/machinesink-dead-cc.mir +++ b/llvm/test/CodeGen/SystemZ/machinesink-dead-cc.mir @@ -1,6 +1,9 @@ # RUN: llc -mtriple=s390x-linux-gnu -mcpu=z15 -O3 -run-pass=machine-sink %s -o - \ # RUN: -verify-machineinstrs | FileCheck %s # +# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z15 -O3 -passes=machine-sink %s -o - \ +# RUN: -verify-machineinstrs | FileCheck %s +# # Test that the AGHIK can be sunk into %bb.4. It has a def of CC, but it is dead. --- | diff --git a/llvm/test/CodeGen/X86/machinesink-debug-inv-0.mir b/llvm/test/CodeGen/X86/machinesink-debug-inv-0.mir index 643c557db77f7..20207311e66cd 100644 --- a/llvm/test/CodeGen/X86/machinesink-debug-inv-0.mir +++ b/llvm/test/CodeGen/X86/machinesink-debug-inv-0.mir @@ -2,6 +2,9 @@ # RUN: llc -mtriple=x86_64 -machine-sink-load-instrs-threshold=2 -run-pass=machine-sink %s -o - | FileCheck %s # RUN: llc -mtriple=x86_64 -machine-sink-load-instrs-threshold=2 -run-pass=mir-debugify,machine-sink,mir-strip-debug %s -o - | FileCheck %s +# RUN: llc -mtriple=x86_64 -machine-sink-load-instrs-threshold=2 -passes=machine-sink %s -o - | FileCheck %s +# TODO: add test with mir-debugify once it is ported + # Verify that machine-sink pass is debug invariant wrt to given input. Since # the pass examines MemOperands the IR is required for the original bug to # trigger. diff --git a/llvm/test/DebugInfo/MIR/X86/sink-leaves-undef.mir b/llvm/test/DebugInfo/MIR/X86/sink-leaves-undef.mir index 73a571dbf150f..b7df7983073e8 100644 --- a/llvm/test/DebugInfo/MIR/X86/sink-leaves-undef.mir +++ b/llvm/test/DebugInfo/MIR/X86/sink-leaves-undef.mir @@ -1,4 +1,5 @@ # RUN: llc %s -o - -run-pass=machine-sink -mtriple=x86_64-- | FileCheck %s +# RUN: llc %s -o - -passes=machine-sink -mtriple=x86_64-- | FileCheck %s # This is a copy of test/CodeGen/X86/MachineSink-DbgValue.ll, where we # additionally test that when the MOV32rm defining %0 is sunk, it leaves # an 'undef' DBG_VALUE behind to terminate earlier location ranges. >From fd3300b65e5e615cae51160b42bf2004d1b179ae Mon Sep 17 00:00:00 2001 From: Akshat Oke <akshat....@amd.com> Date: Fri, 21 Feb 2025 09:24:22 +0000 Subject: [PATCH 2/4] remove PM from impl class --- llvm/lib/CodeGen/MachineSink.cpp | 78 ++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index 18f3740b01298..7ff918828a29d 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -27,6 +27,8 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" @@ -44,6 +46,7 @@ #include "llvm/CodeGen/MachineSizeOpts.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/RegisterPressure.h" +#include "llvm/CodeGen/SlotIndexes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -135,8 +138,11 @@ class MachineSinking { AliasAnalysis *AA = nullptr; RegisterClassInfo RegClassInfo; TargetSchedModel SchedModel; - Pass *LegacyPass; - MachineFunctionAnalysisManager *MFAM; + // Required for split critical edge + LiveIntervals *LIS; + SlotIndexes *SI; + LiveVariables *LV; + MachineLoopInfo *MLI; // Remember which edges have been considered for breaking. SmallSet<std::pair<MachineBasicBlock *, MachineBasicBlock *>, 8> @@ -194,9 +200,14 @@ class MachineSinking { bool EnableSinkAndFold; public: - MachineSinking(Pass *LegacyPass, MachineFunctionAnalysisManager *MFAM, - bool EnableSinkAndFold) - : LegacyPass(LegacyPass), MFAM(MFAM), + MachineSinking(bool EnableSinkAndFold, MachineDominatorTree *DT, + MachinePostDominatorTree *PDT, LiveVariables *LV, + MachineLoopInfo *MLI, SlotIndexes *SI, LiveIntervals *LIS, + MachineCycleInfo *CI, ProfileSummaryInfo *PSI, + MachineBlockFrequencyInfo *MBFI, + const MachineBranchProbabilityInfo *MBPI, AliasAnalysis *AA) + : DT(DT), PDT(PDT), CI(CI), PSI(PSI), MBFI(MBFI), MBPI(MBPI), AA(AA), + LIS(LIS), SI(SI), LV(LV), MLI(MLI), EnableSinkAndFold(EnableSinkAndFold) {} bool run(MachineFunction &MF); @@ -746,7 +757,23 @@ void MachineSinking::FindCycleSinkCandidates( PreservedAnalyses MachineSinkingPass::run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) { - MachineSinking Impl(nullptr, &MFAM, EnableSinkAndFold); + auto *DT = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF); + auto *PDT = &MFAM.getResult<MachinePostDominatorTreeAnalysis>(MF); + auto *CI = &MFAM.getResult<MachineCycleAnalysis>(MF); + auto *PSI = MFAM.getCachedResult<ProfileSummaryAnalysis>(MF); + auto *MBFI = UseBlockFreqInfo + ? &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF) + : nullptr; + auto *MBPI = &MFAM.getResult<MachineBranchProbabilityAnalysis>(MF); + auto *AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF) + .getManager() + .getResult<AAManager>(MF.getFunction()); + auto *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF); + auto *SI = &MFAM.getResult<SlotIndexesAnalysis>(MF); + auto *LV = &MFAM.getResult<LiveVariablesAnalysis>(MF); + auto *MLI = &MFAM.getResult<MachineLoopAnalysis>(MF); + MachineSinking Impl(EnableSinkAndFold, DT, PDT, LV, MLI, SI, LIS, CI, PSI, + MBFI, MBPI, AA); bool Changed = Impl.run(MF); if (!Changed) return PreservedAnalyses::all(); @@ -770,7 +797,25 @@ bool MachineSinkingLegacy::runOnMachineFunction(MachineFunction &MF) { TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>(); bool EnableSinkAndFold = PassConfig->getEnableSinkAndFold(); - MachineSinking Impl(this, nullptr, EnableSinkAndFold); + auto *DT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree(); + auto *PDT = + &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree(); + auto *CI = &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo(); + auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); + auto *MBFI = + UseBlockFreqInfo + ? &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI() + : nullptr; + auto *MBPI = + &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); + auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); + auto *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS(); + auto *SI = &getAnalysis<SlotIndexesWrapperPass>().getSI(); + auto *LV = &getAnalysis<LiveVariablesWrapperPass>().getLV(); + auto *MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI(); + + MachineSinking Impl(EnableSinkAndFold, DT, PDT, LV, MLI, SI, LIS, CI, PSI, + MBFI, MBPI, AA); return Impl.run(MF); } @@ -786,21 +831,6 @@ bool MachineSinking::run(MachineFunction &MF) { TII = STI->getInstrInfo(); TRI = STI->getRegisterInfo(); MRI = &MF.getRegInfo(); - DT = GET_ANALYSIS(MachineDominatorTree, , getDomTree); - PDT = GET_ANALYSIS(MachinePostDominatorTree, , getPostDomTree); - CI = GET_ANALYSIS(MachineCycle, Info, getCycleInfo); - PSI = (LegacyPass) - ? &LegacyPass->getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI() - : MFAM->getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF) - .getCachedResult<ProfileSummaryAnalysis>(*MF.getFunction().getParent()); - MBFI = UseBlockFreqInfo ? GET_ANALYSIS(MachineBlockFrequency, Info, getMBFI) - : nullptr; - MBPI = GET_ANALYSIS(MachineBranchProbability, Info, getMBPI); - AA = (LegacyPass) - ? &LegacyPass->getAnalysis<AAResultsWrapperPass>().getAAResults() - : &MFAM->getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF) - .getManager() - .getResult<AAManager>(MF.getFunction()); RegClassInfo.runOnMachineFunction(MF); @@ -820,8 +850,8 @@ bool MachineSinking::run(MachineFunction &MF) { MachineDomTreeUpdater MDTU(DT, PDT, MachineDomTreeUpdater::UpdateStrategy::Lazy); for (const auto &Pair : ToSplit) { - auto NewSucc = - Pair.first->SplitCriticalEdge(Pair.second, LegacyPass, MFAM, nullptr, &MDTU); + auto NewSucc = Pair.first->SplitCriticalEdge( + Pair.second, {LIS, SI, LV, MLI}, nullptr, &MDTU); if (NewSucc != nullptr) { LLVM_DEBUG(dbgs() << " *** Splitting critical edge: " << printMBBReference(*Pair.first) << " -- " >From 4e8d62fa76dcd4e304aa5f0596a438b33d768063 Mon Sep 17 00:00:00 2001 From: Akshat Oke <akshat....@amd.com> Date: Fri, 21 Feb 2025 09:34:05 +0000 Subject: [PATCH 3/4] remove macro --- llvm/lib/CodeGen/MachineSink.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index 7ff918828a29d..c5fbd80267ec5 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -819,11 +819,6 @@ bool MachineSinkingLegacy::runOnMachineFunction(MachineFunction &MF) { return Impl.run(MF); } -#define GET_ANALYSIS(ANALYSIS, INFIX, GETTER) \ - ((LegacyPass) \ - ? &LegacyPass->getAnalysis<ANALYSIS##INFIX##WrapperPass>().GETTER() \ - : &MFAM->getResult<ANALYSIS##Analysis>(MF)) - bool MachineSinking::run(MachineFunction &MF) { LLVM_DEBUG(dbgs() << "******** Machine Sinking ********\n"); >From a7282043cfb98e44f2e7476bfd33c5a0eb6703cd Mon Sep 17 00:00:00 2001 From: Akshat Oke <akshat....@amd.com> Date: Fri, 21 Feb 2025 09:35:53 +0000 Subject: [PATCH 4/4] remove superfluous includes --- llvm/lib/CodeGen/MachineSink.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index c5fbd80267ec5..dccda93a058de 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -40,7 +40,6 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineOperand.h" -#include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineSizeOpts.h" @@ -52,7 +51,6 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSchedule.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/IR/Analysis.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/LLVMContext.h" @@ -785,7 +783,7 @@ MachineSinkingPass::run(MachineFunction &MF, void MachineSinkingPass::printPipeline( raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) { - OS << "machine-sink"; + OS << MapClassName2PassName(name()); // ideally machine-sink if (EnableSinkAndFold) OS << "<enable-sink-fold>"; } _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
