Author: Florian Hahn Date: 2020-11-29T17:24:36Z New Revision: a813090072c0527eb6ed51dd2ea4f54cb6bc72a0
URL: https://github.com/llvm/llvm-project/commit/a813090072c0527eb6ed51dd2ea4f54cb6bc72a0 DIFF: https://github.com/llvm/llvm-project/commit/a813090072c0527eb6ed51dd2ea4f54cb6bc72a0.diff LOG: [VPlan] Manage stored values of interleave groups using VPUser (NFC) Interleave groups also depend on the values they store. Manage the stored values as VPUser operands. This is currently a NFC, but is required to allow VPlan transforms and to manage generated vector values exclusively in VPTransformState. Added: Modified: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp llvm/lib/Transforms/Vectorize/VPlan.h llvm/unittests/Transforms/Vectorize/VPlanTest.cpp Removed: ################################################################################ diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 0f519d136b31..1b716e1051cc 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -550,6 +550,7 @@ class InnerLoopVectorizer { /// values in the vectorized loop. void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group, VPTransformState &State, VPValue *Addr, + ArrayRef<VPValue *> StoredValues, VPValue *BlockInMask = nullptr); /// Vectorize Load and Store instructions with the base address given in \p @@ -2322,7 +2323,7 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) { // store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B void InnerLoopVectorizer::vectorizeInterleaveGroup( const InterleaveGroup<Instruction> *Group, VPTransformState &State, - VPValue *Addr, VPValue *BlockInMask) { + VPValue *Addr, ArrayRef<VPValue *> StoredValues, VPValue *BlockInMask) { Instruction *Instr = Group->getInsertPos(); const DataLayout &DL = Instr->getModule()->getDataLayout(); @@ -2467,8 +2468,8 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup( Instruction *Member = Group->getMember(i); assert(Member && "Fail to get a member from an interleaved store group"); - Value *StoredVec = getOrCreateVectorValue( - cast<StoreInst>(Member)->getValueOperand(), Part); + Value *StoredVec = State.get(StoredValues[i], Part); + if (Group->isReverse()) StoredVec = reverseVector(StoredVec); @@ -7828,7 +7829,13 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( for (auto IG : InterleaveGroups) { auto *Recipe = cast<VPWidenMemoryInstructionRecipe>( RecipeBuilder.getRecipe(IG->getInsertPos())); - (new VPInterleaveRecipe(IG, Recipe->getAddr(), Recipe->getMask())) + SmallVector<VPValue *, 4> StoredValues; + for (unsigned i = 0; i < IG->getFactor(); ++i) + if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i))) + StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0))); + + (new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues, + Recipe->getMask())) ->insertBefore(Recipe); for (unsigned i = 0; i < IG->getFactor(); ++i) @@ -8068,7 +8075,8 @@ void VPBlendRecipe::execute(VPTransformState &State) { void VPInterleaveRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Interleave group being replicated."); - State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getMask()); + State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getStoredValues(), + getMask()); } void VPReductionRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b4d84ed57773..4abb08809da5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1035,16 +1035,24 @@ class VPBlendRecipe : public VPRecipeBase, public VPUser { }; /// VPInterleaveRecipe is a recipe for transforming an interleave group of load -/// or stores into one wide load/store and shuffles. +/// or stores into one wide load/store and shuffles. The first operand of a +/// VPInterleave recipe is the address, followed by the stored values, followed +/// by an optional mask. class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup<Instruction> *IG; + bool HasMask = false; + public: VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr, - VPValue *Mask) + ArrayRef<VPValue *> StoredValues, VPValue *Mask) : VPRecipeBase(VPInterleaveSC), VPUser({Addr}), IG(IG) { - if (Mask) + for (auto *SV : StoredValues) + addOperand(SV); + if (Mask) { + HasMask = true; addOperand(Mask); + } } ~VPInterleaveRecipe() override = default; @@ -1062,7 +1070,16 @@ class VPInterleaveRecipe : public VPRecipeBase, public VPUser { /// by a nullptr. VPValue *getMask() const { // Mask is optional and therefore the last, currently 2nd operand. - return getNumOperands() == 2 ? getOperand(1) : nullptr; + return HasMask ? getOperand(getNumOperands() - 1) : nullptr; + } + + /// Return the VPValues stored by this interleave group. If it is a load + /// interleave group, return an empty ArrayRef. + ArrayRef<VPValue *> getStoredValues() const { + // The first operand is the address, followed by the stored values, followed + // by an optional mask. + return ArrayRef<VPValue *>(op_begin(), getNumOperands()) + .slice(1, getNumOperands() - (HasMask ? 2 : 1)); } /// Generate the wide load or store, and shuffles. diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index d5c11396b303..f89abc8769b4 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -470,7 +470,7 @@ TEST(VPRecipeTest, CastVPInterleaveRecipeToVPUser) { VPValue Addr; VPValue Mask; - VPInterleaveRecipe Recipe(nullptr, &Addr, &Mask); + VPInterleaveRecipe Recipe(nullptr, &Addr, {}, &Mask); EXPECT_TRUE(isa<VPUser>(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa<VPUser>(BaseR)); _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
