llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-risc-v Author: Kito Cheng (kito-cheng) <details> <summary>Changes</summary> Extract ManualCodegen blocks from riscv_vector.td to dedicated helper functions in RISCV.cpp to improve compilation times and code organization. This refactoring: - Reduces riscv_vector_builtin_cg.inc from ~70,000 lines to ~30,000 lines - Extracts lots of ManualCodegen blocks into helper functions in RISCV.cpp - Moves complex code generation logic from TableGen to C++ - Marks extracted functions with LLVM_ATTRIBUTE_NOINLINE to prevent excessive inlining in EmitRISCVBuiltinExpr's large switch statement, which would cause compilation time to increase significantly Performance Impact on AMD Ryzen 9 3950X 16-Core with SSD (Release build) with GCC 11: Before: real 0m22.577s, user 0m0.498s, sys 0m0.152s After: real 1m4.560s, user 0m0.529s, sys 0m0.175s Which reduced around 65% of compilation time. During this refactoring, I also found few more opportunities to optimize and simplify the code generation logic, but I think leave to next PR since it already change a lot of code. Fix #<!-- -->88368 --- Patch is 81.36 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/154906.diff 2 Files Affected: - (modified) clang/include/clang/Basic/riscv_vector.td (+77-685) - (modified) clang/lib/CodeGen/TargetBuiltins/RISCV.cpp (+952-4) ``````````diff diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td index c1de2bfe4243d..cc5ab38f8f960 100644 --- a/clang/include/clang/Basic/riscv_vector.td +++ b/clang/include/clang/Basic/riscv_vector.td @@ -56,34 +56,8 @@ multiclass RVVVLEFFBuiltin<list<string> types> { SupportOverloading = false, UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - IntrinsicTypes = {ResultType, Ops[4]->getType(), Ops[2]->getType()}; - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - IntrinsicTypes = {ResultType, Ops[3]->getType(), Ops[1]->getType()}; - } - Value *NewVL = Ops[2]; - Ops.erase(Ops.begin() + 2); - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); - llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); - // Store new_vl. - clang::CharUnits Align; - if (IsMasked) - Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); - else - Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); - Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); - return V; - } + return emitRVVVLEFFBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach type = types in { def : RVVBuiltin<"v", "vPCePz", type>; @@ -139,17 +113,8 @@ multiclass RVVIndexedLoad<string op> { let HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (IsMasked) { - // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) - std::swap(Ops[0], Ops[2]); - } else { - // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) - std::swap(Ops[0], Ops[1]); - } - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; + return emitRVVVSEMaskBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { let Name = "vsm_v"; @@ -177,17 +142,8 @@ multiclass RVVVSSEBuiltin<list<string> types> { HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (IsMasked) { - // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) - std::swap(Ops[0], Ops[3]); - } else { - // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); - } - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[4]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()}; + return emitRVVVSSEBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach type = types in { def : RVVBuiltin<"v", "0Petv", type>; @@ -202,17 +158,8 @@ multiclass RVVIndexedStore<string op> { let HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - if (IsMasked) { - // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) - std::swap(Ops[0], Ops[3]); - } else { - // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); - } - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), Ops[4]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), Ops[3]->getType()}; + return emitRVVIndexedStoreBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach type = TypeList in { foreach eew_list = EEWList[0-2] in { @@ -367,28 +314,8 @@ multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, op2, mask, vl, policy - IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; - } else { - // passthru, op1, op2, vl - IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoUnaryBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"v", "vv", type_range>; } @@ -400,32 +327,8 @@ multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, - llvm::Constant::getAllOnesValue(ElemTy)); - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, po2, mask, vl, policy - IntrinsicTypes = {ResultType, - ElemTy, - Ops[4]->getType()}; - } else { - // passthru, op1, op2, vl - IntrinsicTypes = {ResultType, - ElemTy, - Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoVNotBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"v", "vv", type_range>; def : RVVBuiltin<"Uv", "UvUv", type_range>; @@ -437,13 +340,8 @@ multiclass RVVPseudoMaskBuiltin<string IR, string type_range> { IRName = IR, HasMasked = false, ManualCodegen = [{ - { - // op1, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType()}; - Ops.insert(Ops.begin() + 1, Ops[0]); - break; - } + return emitRVVPseudoMaskBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"m", "mm", type_range>; } @@ -455,28 +353,8 @@ multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.insert(Ops.begin() + 2, Ops[1]); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, op2, mask, vl - IntrinsicTypes = {ResultType, - Ops[2]->getType(), - Ops.back()->getType()}; - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - // op1, po2, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), Ops[2]->getType()}; - Ops.insert(Ops.begin() + 2, Ops[1]); - break; - } - break; - } + return emitRVVPseudoVFUnaryBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { def : RVVBuiltin<"v", "vv", type_range>; } @@ -490,33 +368,8 @@ multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - auto ElemTy = cast<llvm::VectorType>(Ops[1]->getType())->getElementType(); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, op2, mask, vl, policy - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - ElemTy, - Ops[4]->getType()}; - } else { - // passtru, op1, op2, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - ElemTy, - Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoVWCVTBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach s_p = suffixes_prototypes in { def : RVVBuiltin<s_p[0], s_p[1], type_range>; @@ -532,32 +385,8 @@ multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, MaskedIRName = IR # "_mask", UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ - { - if (IsMasked) { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if ((PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } else { - if (PolicyAttrs & RVV_VTA) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - } - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); - if (IsMasked) { - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - // maskedoff, op1, xlen, mask, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - Ops[4]->getType(), - Ops[4]->getType()}; - } else { - // passthru, op1, xlen, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - Ops[3]->getType(), - Ops[3]->getType()}; - } - break; - } + return emitRVVPseudoVNCVTBuiltin(this, E, ReturnValue, ResultType, ID, + Ops, PolicyAttrs, IsMasked, SegInstSEW); }] in { foreach s_p = suffixes_prototypes in { def : RVVBuiltin<s_p[0], s_p[1], type_range>; @@ -575,17 +404,8 @@ let HasBuiltinAlias = false, HasVL = false, HasMasked = false, UnMaskedPolicyScheme = NonePolicy, MaskedPolicyScheme = NonePolicy, Log2LMUL = [0], IRName = "", ManualCodegen = [{ - { - LLVMContext &Context = CGM.getLLVMContext(); - llvm::MDBuilder MDHelper(Context); - - llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "vlenb")}; - llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); - llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName); - llvm::Function *F = - CGM.getIntrinsic(llvm::Intrinsic::read_register, {SizeTy}); - return Builder.CreateCall(F, Metadata); - } + return emitRVVVlenbBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); }] in { def vlenb : RVVBuiltin<"", "u", "i">; @@ -660,7 +480,10 @@ let HasBuiltinAlias = false, HasMasked = false, MaskedPolicyScheme = NonePolicy, Log2LMUL = [0], - ManualCodegen = [{IntrinsicTypes = {ResultType};}] in // Set XLEN type + ManualCodegen = [{ + return emitRVVVsetvliBuiltin(this, E, ReturnValue, ResultType, ID, Ops, + PolicyAttrs, IsMasked, SegInstSEW); + }] in // Set XLEN type { def vsetvli : RVVBuiltin<"", "zzKzKz", "i">; def vsetvlimax : RVVBuiltin<"", "zKzKz", "i">; @@ -720,35 +543,10 @@ multiclass RVVUnitStridedSegLoadTuple<string op> { MaskedIRName = op # nf # "_mask", NF = nf, ManualCodegen = [{ - { - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[0]->getType(), Ops.back()->getType()}; - else - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType()}; - - if (IsMasked) - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (NoPassthru) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - - if (IsMasked) - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - - llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); - if (ReturnValue.isNull()) - return LoadValue; - else - return Builder.CreateStore(LoadValue, ReturnValue.getValue()); - } - }] in { + return emitRVVUnitStridedSegLoadTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, + IsMasked, SegInstSEW); + }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", T # "vPCe", type>; if !not(IsFloat<type>.val) then { @@ -776,29 +574,10 @@ multiclass RVVUnitStridedSegStoreTuple<string op> { NF = nf, HasMaskedOffOperand = false, ManualCodegen = [{ - { - // Masked - // Builtin: (mask, ptr, v_tuple, vl) - // Intrinsic: (tuple, ptr, mask, vl, SegInstSEW) - // Unmasked - // Builtin: (ptr, v_tuple, vl) - // Intrinsic: (tuple, ptr, vl, SegInstSEW) - - if (IsMasked) - std::swap(Ops[0], Ops[2]); - else - std::swap(Ops[0], Ops[1]); - - Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - if (IsMasked) - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), Ops[3]->getType()}; - else - IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()}; - - break; - } - }] in { + return emitRVVUnitStridedSegStoreTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, + IsMasked, SegInstSEW); + }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", "0Pe" # T # "v", type>; if !not(IsFloat<type>.val) then { @@ -825,47 +604,9 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> { MaskedIRName = op # nf # "ff_mask", NF = nf, ManualCodegen = [{ - { - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType(), Ops[0]->getType()}; - else - IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType()}; - - if (IsMasked) - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (NoPassthru) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - - if (IsMasked) - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - Value *NewVL = Ops[2]; - Ops.erase(Ops.begin() + 2); - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - - llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); - // Get alignment from the new vl operand - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType()); - - llvm::Value *ReturnTuple = Builder.CreateExtractValue(LoadValue, 0); - - // Store new_vl - llvm::Value *V = Builder.CreateExtractValue(LoadValue, 1); - Builder.CreateStore(V, Address(NewVL, V->getType(), Align)); - - if (ReturnValue.isNull()) - return ReturnTuple; - else - return Builder.CreateStore(ReturnTuple, ReturnValue.getValue()); - } + return emitRVVUnitStridedSegLoadFFTupleBuiltin( + this, E, ReturnValue, ResultType, ID, Ops, PolicyAttrs, IsMasked, + SegInstSEW); }] in { defvar T = "(Tuple:" # nf # ")"; def : RVVBuiltin<T # "v", T # "vPCePz", type>; @@ -893,34 +634,9 @@ multiclass RVVStridedSegLoadTuple<string op> { MaskedIRName = op # nf # "_mask", NF = nf, ManualCodegen = [{ - { - bool NoPassthru = - (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) | - (!IsMasked && (PolicyAttrs & RVV_VTA)); - unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1; - - if (IsMasked) - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType(), Ops[0]->getType()}; - else - IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType()}; - - if (IsMasked) - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (NoPassthru) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - - if (IsMasked) - Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW)); - - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); - - if (ReturnValue.isNull()) - return LoadValue; - else - return Builder.CreateStore(LoadValue, ReturnValue.getValue()); - } + return emitRVVStridedSegLoadTupleBuiltin( + ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/154906 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
