Author: Elvis Wang Date: 2024-11-22T11:32:12+08:00 New Revision: 14b474be36144527a55b5d49954379a3484c5f84
URL: https://github.com/llvm/llvm-project/commit/14b474be36144527a55b5d49954379a3484c5f84 DIFF: https://github.com/llvm/llvm-project/commit/14b474be36144527a55b5d49954379a3484c5f84.diff LOG: Revert "[LV][VPlan] Remove any-of reduction from precomputeCost. NFC (#117109)" This reverts commit ce66b56865426fc1760b5a090ca2748c046094f5. Added: Modified: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp Removed: ################################################################################ diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 5b556058cc762c..d13770a35c108f 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7303,14 +7303,34 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF, // The legacy cost model has special logic to compute the cost of in-loop // reductions, which may be smaller than the sum of all instructions involved - // in the reduction. + // in the reduction. For AnyOf reductions, VPlan codegen may remove the select + // which the legacy cost model uses to assign cost. Pre-compute their costs + // for now. // TODO: Switch to costing based on VPlan once the logic has been ported. for (const auto &[RedPhi, RdxDesc] : Legal->getReductionVars()) { if (ForceTargetInstructionCost.getNumOccurrences()) continue; - if (!CM.isInLoopReduction(RedPhi)) + if (!CM.isInLoopReduction(RedPhi) && + !RecurrenceDescriptor::isAnyOfRecurrenceKind( + RdxDesc.getRecurrenceKind())) + continue; + + // AnyOf reduction codegen may remove the select. To match the legacy cost + // model, pre-compute the cost for AnyOf reductions here. + if (RecurrenceDescriptor::isAnyOfRecurrenceKind( + RdxDesc.getRecurrenceKind())) { + auto *Select = cast<SelectInst>(*find_if( + RedPhi->users(), [](User *U) { return isa<SelectInst>(U); })); + assert(!CostCtx.SkipCostComputation.contains(Select) && + "reduction op visited multiple times"); + CostCtx.SkipCostComputation.insert(Select); + auto ReductionCost = CostCtx.getLegacyCost(Select, VF); + LLVM_DEBUG(dbgs() << "Cost of " << ReductionCost << " for VF " << VF + << ":\n any-of reduction " << *Select << "\n"); + Cost += ReductionCost; continue; + } const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop); SetVector<Instruction *> ChainOpsAndOperands(ChainOps.begin(), _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits