================ @@ -20,6 +20,958 @@ using namespace clang; using namespace CodeGen; using namespace llvm; +// The 0th bit simulates the `vta` of RVV +// The 1st bit simulates the `vma` of RVV +static constexpr unsigned RVV_VTA = 0x1; +static constexpr unsigned RVV_VMA = 0x2; + +// RISC-V Vector builtin helper functions are marked NOINLINE to prevent +// excessive inlining in CodeGenFunction::EmitRISCVBuiltinExpr's large switch +// statement, which would significantly increase compilation time. +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVLEFFBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + // Move mask to right before vl. + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + IntrinsicTypes = {ResultType, Ops[4]->getType(), Ops[2]->getType()}; + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops[3]->getType(), Ops[1]->getType()}; + } + Value *NewVL = Ops[2]; + Ops.erase(Ops.begin() + 2); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); + // Store new_vl. + clang::CharUnits Align; + if (IsMasked) + Align = CGM.getNaturalPointeeTypeAlignment( + E->getArg(E->getNumArgs() - 2)->getType()); + else + Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); + llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); + Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); + return V; +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVSSEBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, + // mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVIndexedStoreBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + // Builtin: (mask, ptr, index, value, vl). + // Intrinsic: (value, ptr, index, mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, index, value, vl). + // Intrinsic: (value, ptr, index, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoUnaryBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoVNotBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getAllOnesValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, po2, mask, vl, policy + IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoMaskBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + // op1, vl + IntrinsicTypes = {ResultType, Ops[1]->getType()}; + Ops.insert(Ops.begin() + 1, Ops[0]); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVPseudoVFUnaryBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.insert(Ops.begin() + 2, Ops[1]); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl + IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()}; + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + // op1, po2, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType()}; + Ops.insert(Ops.begin() + 2, Ops[1]); + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoVWCVTBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + auto ElemTy = cast<llvm::VectorType>(Ops[1]->getType())->getElementType(); + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, Ops[1]->getType(), ElemTy, Ops[4]->getType()}; + } else { + // passtru, op1, op2, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), ElemTy, Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVPseudoVNCVTBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (PolicyAttrs & RVV_VTA) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + Ops.insert(Ops.begin() + 2, + llvm::Constant::getNullValue(Ops.back()->getType())); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + // maskedoff, op1, xlen, mask, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[4]->getType(), + Ops[4]->getType()}; + } else { + // passthru, op1, xlen, vl + IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[3]->getType(), + Ops[3]->getType()}; + } + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVlenbBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + LLVMContext &Context = CGM.getLLVMContext(); + llvm::MDBuilder MDHelper(Context); + llvm::Metadata *OpsMD[] = {llvm::MDString::get(Context, "vlenb")}; + llvm::MDNode *RegName = llvm::MDNode::get(Context, OpsMD); + llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); + llvm::Function *F = + CGM.getIntrinsic(llvm::Intrinsic::read_register, {CGF->SizeTy}); + return Builder.CreateCall(F, Metadata); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVsetvliBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes = {ResultType}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value * +emitRVVVSEMaskBuiltin(CodeGenFunction *CGF, const CallExpr *E, + ReturnValueSlot ReturnValue, llvm::Type *ResultType, + Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + if (IsMasked) { + // Builtin: (mask, ptr, value, vl). + // Intrinsic: (value, ptr, mask, vl) + std::swap(Ops[0], Ops[2]); + } else { + // Builtin: (ptr, value, vl). + // Intrinsic: (value, ptr, vl) + std::swap(Ops[0], Ops[1]); + } + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVUnitStridedSegLoadTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + if (IsMasked) + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[0]->getType(), + Ops.back()->getType()}; + else + IntrinsicTypes = {ResultType, Ops[Offset]->getType(), + Ops.back()->getType()}; + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (NoPassthru) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + if (ReturnValue.isNull()) + return LoadValue; + else + return Builder.CreateStore(LoadValue, ReturnValue.getValue()); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVUnitStridedSegStoreTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + // Masked + // Builtin: (mask, ptr, v_tuple, vl) + // Intrinsic: (tuple, ptr, mask, vl, SegInstSEW) + // Unmasked + // Builtin: (ptr, v_tuple, vl) + // Intrinsic: (tuple, ptr, vl, SegInstSEW) + if (IsMasked) + std::swap(Ops[0], Ops[2]); + else + std::swap(Ops[0], Ops[1]); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), + Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + return Builder.CreateCall(F, Ops, ""); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVUnitStridedSegLoadFFTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; + bool NoPassthru = + (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | + (!IsMasked && (PolicyAttrs & RVV_VTA)); + unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; + if (IsMasked) + IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType(), + Ops[0]->getType()}; + else + IntrinsicTypes = {ResultType, Ops.back()->getType(), + Ops[Offset]->getType()}; + if (IsMasked) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (NoPassthru) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) + Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); + Value *NewVL = Ops[2]; + Ops.erase(Ops.begin() + 2); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); + llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); + // Get alignment from the new vl operand + clang::CharUnits Align = + CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); + llvm::Value *ReturnTuple = Builder.CreateExtractValue(LoadValue, 0); + // Store new_vl + llvm::Value *V = Builder.CreateExtractValue(LoadValue, 1); + Builder.CreateStore(V, Address(NewVL, V->getType(), Align)); + if (ReturnValue.isNull()) + return ReturnTuple; + else + return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); +} + +static LLVM_ATTRIBUTE_NOINLINE Value *emitRVVStridedSegLoadTupleBuiltin( + CodeGenFunction *CGF, const CallExpr *E, ReturnValueSlot ReturnValue, + llvm::Type *ResultType, Intrinsic::ID ID, SmallVector<Value *, 4> Ops, + int PolicyAttrs, bool IsMasked, unsigned SegInstSEW) { + auto &Builder = CGF->Builder; + auto &CGM = CGF->CGM; + llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; ---------------- topperc wrote:
2 -> 4 https://github.com/llvm/llvm-project/pull/154906 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits