llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang-codegen Author: Sarah Spall (spall) <details> <summary>Changes</summary> Adds support for elementwise and aggregate splat casting struct types with bitfields. Replacing existing Flattening function which used to produce a list of GEPs representing a flattened object with one that produces a list of LValues representing a flattened object. The LValues can be used by EmitStoreThroughLValue and EmitLoadOfLValue, ensuring bitfields are properly loaded and stored. This also simplifies the code in the elementwise and aggregate splat casting functions. Closes #<!-- -->125986 --- Patch is 37.09 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/161263.diff 11 Files Affected: - (modified) clang/lib/CodeGen/CGExpr.cpp (+66-36) - (modified) clang/lib/CodeGen/CGExprAgg.cpp (+55-91) - (modified) clang/lib/CodeGen/CGExprScalar.cpp (+22-26) - (modified) clang/lib/CodeGen/CodeGenFunction.h (+2-4) - (modified) clang/lib/Sema/SemaHLSL.cpp (-6) - (modified) clang/test/CodeGenHLSL/BasicFeatures/AggregateSplatCast.hlsl (+38) - (modified) clang/test/CodeGenHLSL/BasicFeatures/ArrayElementwiseCast.hlsl (+45-1) - (modified) clang/test/CodeGenHLSL/BasicFeatures/StructElementwiseCast.hlsl (+114-7) - (modified) clang/test/CodeGenHLSL/BasicFeatures/VectorElementwiseCast.hlsl (+42) - (modified) clang/test/SemaHLSL/Language/AggregateSplatCast-errors.hlsl (-6) - (modified) clang/test/SemaHLSL/Language/ElementwiseCast-errors.hlsl (-21) ``````````diff diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index e6e4947882544..04e2b64d2bd7c 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -6784,29 +6784,26 @@ LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) { return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV; } -void CodeGenFunction::FlattenAccessAndType( - Address Addr, QualType AddrType, - SmallVectorImpl<std::pair<Address, llvm::Value *>> &AccessList, - SmallVectorImpl<QualType> &FlatTypes) { - // WorkList is list of type we are processing + the Index List to access - // the field of that type in Addr for use in a GEP - llvm::SmallVector<std::pair<QualType, llvm::SmallVector<llvm::Value *, 4>>, - 16> +void CodeGenFunction::FlattenAccessAndTypeLValue( + LValue Val, SmallVectorImpl<LValue> &AccessList) { + + llvm::SmallVector< + std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16> WorkList; llvm::IntegerType *IdxTy = llvm::IntegerType::get(getLLVMContext(), 32); - // Addr should be a pointer so we need to 'dereference' it - WorkList.push_back({AddrType, {llvm::ConstantInt::get(IdxTy, 0)}}); + WorkList.push_back({Val, Val.getType(), {llvm::ConstantInt::get(IdxTy, 0)}}); while (!WorkList.empty()) { - auto [T, IdxList] = WorkList.pop_back_val(); + auto [LVal, T, IdxList] = WorkList.pop_back_val(); T = T.getCanonicalType().getUnqualifiedType(); assert(!isa<MatrixType>(T) && "Matrix types not yet supported in HLSL"); + if (const auto *CAT = dyn_cast<ConstantArrayType>(T)) { uint64_t Size = CAT->getZExtSize(); for (int64_t I = Size - 1; I > -1; I--) { llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList; IdxListCopy.push_back(llvm::ConstantInt::get(IdxTy, I)); - WorkList.emplace_back(CAT->getElementType(), IdxListCopy); + WorkList.push_back({LVal, CAT->getElementType(), IdxListCopy}); } } else if (const auto *RT = dyn_cast<RecordType>(T)) { const RecordDecl *Record = RT->getOriginalDecl()->getDefinitionOrSelf(); @@ -6814,44 +6811,77 @@ void CodeGenFunction::FlattenAccessAndType( const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Record); - llvm::SmallVector<QualType, 16> FieldTypes; + llvm::SmallVector< + std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16> + ReverseList; if (CXXD && CXXD->isStandardLayout()) Record = CXXD->getStandardLayoutBaseWithFields(); // deal with potential base classes if (CXXD && !CXXD->isStandardLayout()) { - for (auto &Base : CXXD->bases()) - FieldTypes.push_back(Base.getType()); + if (CXXD->getNumBases() > 0) { + assert(CXXD->getNumBases() == 1 && + "HLSL doesn't support multiple inheritance."); + auto Base = CXXD->bases_begin(); + llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList; + IdxListCopy.push_back(llvm::ConstantInt::get( + IdxTy, 0)); // base struct should be at index zero + ReverseList.insert(ReverseList.end(), + {LVal, Base->getType(), IdxListCopy}); + } } - for (auto *FD : Record->fields()) - FieldTypes.push_back(FD->getType()); + const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record); - for (int64_t I = FieldTypes.size() - 1; I > -1; I--) { - llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList; - IdxListCopy.push_back(llvm::ConstantInt::get(IdxTy, I)); - WorkList.insert(WorkList.end(), {FieldTypes[I], IdxListCopy}); + llvm::Type *LLVMT = ConvertTypeForMem(T); + CharUnits Align = getContext().getTypeAlignInChars(T); + LValue RLValue; + bool createdGEP = false; + for (auto *FD : Record->fields()) { + if (FD->isBitField()) { + if (FD->isUnnamedBitField()) + continue; + if (!createdGEP) { + createdGEP = true; + Address GEP = Builder.CreateInBoundsGEP(LVal.getAddress(), IdxList, + LLVMT, Align, "gep"); + RLValue = MakeAddrLValue(GEP, T); + } + LValue FieldLVal = EmitLValueForField(RLValue, FD, true); + ReverseList.insert(ReverseList.end(), {FieldLVal, FD->getType(), {}}); + } else { + llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList; + IdxListCopy.push_back( + llvm::ConstantInt::get(IdxTy, Layout.getLLVMFieldNo(FD))); + ReverseList.insert(ReverseList.end(), + {LVal, FD->getType(), IdxListCopy}); + } } + + std::reverse(ReverseList.begin(), ReverseList.end()); + llvm::append_range(WorkList, ReverseList); } else if (const auto *VT = dyn_cast<VectorType>(T)) { llvm::Type *LLVMT = ConvertTypeForMem(T); CharUnits Align = getContext().getTypeAlignInChars(T); - Address GEP = - Builder.CreateInBoundsGEP(Addr, IdxList, LLVMT, Align, "vector.gep"); + Address GEP = Builder.CreateInBoundsGEP(LVal.getAddress(), IdxList, LLVMT, + Align, "vector.gep"); + LValue Base = MakeAddrLValue(GEP, T); for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) { - llvm::Value *Idx = llvm::ConstantInt::get(IdxTy, I); - // gep on vector fields is not recommended so combine gep with - // extract/insert - AccessList.emplace_back(GEP, Idx); - FlatTypes.push_back(VT->getElementType()); + llvm::Constant *Idx = llvm::ConstantInt::get(IdxTy, I); + LValue LV = + LValue::MakeVectorElt(Base.getAddress(), Idx, VT->getElementType(), + Base.getBaseInfo(), TBAAAccessInfo()); + AccessList.emplace_back(LV); } - } else { - // a scalar/builtin type - llvm::Type *LLVMT = ConvertTypeForMem(T); - CharUnits Align = getContext().getTypeAlignInChars(T); - Address GEP = - Builder.CreateInBoundsGEP(Addr, IdxList, LLVMT, Align, "gep"); - AccessList.emplace_back(GEP, nullptr); - FlatTypes.push_back(T); + } else { // a scalar/builtin type + if (!IdxList.empty()) { + llvm::Type *LLVMT = ConvertTypeForMem(T); + CharUnits Align = getContext().getTypeAlignInChars(T); + Address GEP = Builder.CreateInBoundsGEP(LVal.getAddress(), IdxList, + LLVMT, Align, "gep"); + AccessList.emplace_back(MakeAddrLValue(GEP, T)); + } else // must be a bitfield we already created an lvalue for + AccessList.emplace_back(LVal); } } } diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index b8150a24d45fc..07b9aebe0bbe3 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -488,100 +488,62 @@ static bool isTrivialFiller(Expr *E) { return false; } -static void EmitHLSLAggregateSplatCast(CodeGenFunction &CGF, Address DestVal, - QualType DestTy, llvm::Value *SrcVal, - QualType SrcTy, SourceLocation Loc) { +// emit an elementwise cast where the RHS is a scalar or vector +// or emit an aggregate splat cast +static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF, + LValue DestVal, + llvm::Value *SrcVal, + QualType SrcTy, + SourceLocation Loc) { // Flatten our destination - SmallVector<QualType> DestTypes; // Flattened type - SmallVector<std::pair<Address, llvm::Value *>, 16> StoreGEPList; - // ^^ Flattened accesses to DestVal we want to store into - CGF.FlattenAccessAndType(DestVal, DestTy, StoreGEPList, DestTypes); - - assert(SrcTy->isScalarType() && "Invalid HLSL Aggregate splat cast."); - for (unsigned I = 0, Size = StoreGEPList.size(); I < Size; ++I) { - llvm::Value *Cast = - CGF.EmitScalarConversion(SrcVal, SrcTy, DestTypes[I], Loc); - - // store back - llvm::Value *Idx = StoreGEPList[I].second; - if (Idx) { - llvm::Value *V = - CGF.Builder.CreateLoad(StoreGEPList[I].first, "load.for.insert"); - Cast = CGF.Builder.CreateInsertElement(V, Cast, Idx); - } - CGF.Builder.CreateStore(Cast, StoreGEPList[I].first); - } -} - -// emit a flat cast where the RHS is a scalar, including vector -static void EmitHLSLScalarFlatCast(CodeGenFunction &CGF, Address DestVal, - QualType DestTy, llvm::Value *SrcVal, - QualType SrcTy, SourceLocation Loc) { - // Flatten our destination - SmallVector<QualType, 16> DestTypes; // Flattened type - SmallVector<std::pair<Address, llvm::Value *>, 16> StoreGEPList; - // ^^ Flattened accesses to DestVal we want to store into - CGF.FlattenAccessAndType(DestVal, DestTy, StoreGEPList, DestTypes); - - assert(SrcTy->isVectorType() && "HLSL Flat cast doesn't handle splatting."); - const VectorType *VT = SrcTy->getAs<VectorType>(); - SrcTy = VT->getElementType(); - assert(StoreGEPList.size() <= VT->getNumElements() && - "Cannot perform HLSL flat cast when vector source \ - object has less elements than flattened destination \ - object."); - for (unsigned I = 0, Size = StoreGEPList.size(); I < Size; I++) { - llvm::Value *Load = CGF.Builder.CreateExtractElement(SrcVal, I, "vec.load"); + SmallVector<LValue, 16> StoreList; + CGF.FlattenAccessAndTypeLValue(DestVal, StoreList); + + bool isVector = false; + if (auto *VT = SrcTy->getAs<VectorType>()) { + isVector = true; + SrcTy = VT->getElementType(); + assert(StoreList.size() <= VT->getNumElements() && + "Cannot perform HLSL flat cast when vector source \ + object has less elements than flattened destination \ + object."); + } + + for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) { + LValue DestLVal = StoreList[I]; + llvm::Value *Load = + isVector ? CGF.Builder.CreateExtractElement(SrcVal, I, "vec.load") + : SrcVal; llvm::Value *Cast = - CGF.EmitScalarConversion(Load, SrcTy, DestTypes[I], Loc); - - // store back - llvm::Value *Idx = StoreGEPList[I].second; - if (Idx) { - llvm::Value *V = - CGF.Builder.CreateLoad(StoreGEPList[I].first, "load.for.insert"); - Cast = CGF.Builder.CreateInsertElement(V, Cast, Idx); - } - CGF.Builder.CreateStore(Cast, StoreGEPList[I].first); + CGF.EmitScalarConversion(Load, SrcTy, DestLVal.getType(), Loc); + CGF.EmitStoreThroughLValue(RValue::get(Cast), DestLVal); } } // emit a flat cast where the RHS is an aggregate -static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, Address DestVal, - QualType DestTy, Address SrcVal, - QualType SrcTy, SourceLocation Loc) { +static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal, + LValue SrcVal, SourceLocation Loc) { // Flatten our destination - SmallVector<QualType, 16> DestTypes; // Flattened type - SmallVector<std::pair<Address, llvm::Value *>, 16> StoreGEPList; - // ^^ Flattened accesses to DestVal we want to store into - CGF.FlattenAccessAndType(DestVal, DestTy, StoreGEPList, DestTypes); + SmallVector<LValue, 16> StoreList; + CGF.FlattenAccessAndTypeLValue(DestVal, StoreList); // Flatten our src - SmallVector<QualType, 16> SrcTypes; // Flattened type - SmallVector<std::pair<Address, llvm::Value *>, 16> LoadGEPList; - // ^^ Flattened accesses to SrcVal we want to load from - CGF.FlattenAccessAndType(SrcVal, SrcTy, LoadGEPList, SrcTypes); + SmallVector<LValue, 16> LoadList; + CGF.FlattenAccessAndTypeLValue(SrcVal, LoadList); - assert(StoreGEPList.size() <= LoadGEPList.size() && - "Cannot perform HLSL flat cast when flattened source object \ + assert(StoreList.size() <= LoadList.size() && + "Cannot perform HLSL elementwise cast when flattened source object \ has less elements than flattened destination object."); - // apply casts to what we load from LoadGEPList + // apply casts to what we load from LoadList // and store result in Dest - for (unsigned I = 0, E = StoreGEPList.size(); I < E; I++) { - llvm::Value *Idx = LoadGEPList[I].second; - llvm::Value *Load = CGF.Builder.CreateLoad(LoadGEPList[I].first, "load"); - Load = - Idx ? CGF.Builder.CreateExtractElement(Load, Idx, "vec.extract") : Load; - llvm::Value *Cast = - CGF.EmitScalarConversion(Load, SrcTypes[I], DestTypes[I], Loc); - - // store back - Idx = StoreGEPList[I].second; - if (Idx) { - llvm::Value *V = - CGF.Builder.CreateLoad(StoreGEPList[I].first, "load.for.insert"); - Cast = CGF.Builder.CreateInsertElement(V, Cast, Idx); - } - CGF.Builder.CreateStore(Cast, StoreGEPList[I].first); + for (unsigned I = 0, E = StoreList.size(); I < E; I++) { + LValue DestLVal = StoreList[I]; + LValue SrcLVal = LoadList[I]; + RValue RVal = CGF.EmitLoadOfLValue(SrcLVal, Loc); + assert(RVal.isScalar() && "All flattened source values should be scalars"); + llvm::Value *Val = RVal.getScalarVal(); + llvm::Value *Cast = CGF.EmitScalarConversion(Val, SrcLVal.getType(), + DestLVal.getType(), Loc); + CGF.EmitStoreThroughLValue(RValue::get(Cast), DestLVal); } } @@ -988,31 +950,33 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { Expr *Src = E->getSubExpr(); QualType SrcTy = Src->getType(); RValue RV = CGF.EmitAnyExpr(Src); - QualType DestTy = E->getType(); - Address DestVal = Dest.getAddress(); + LValue DestLVal = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); SourceLocation Loc = E->getExprLoc(); - assert(RV.isScalar() && "RHS of HLSL splat cast must be a scalar."); + assert(RV.isScalar() && SrcTy->isScalarType() && + "RHS of HLSL splat cast must be a scalar."); llvm::Value *SrcVal = RV.getScalarVal(); - EmitHLSLAggregateSplatCast(CGF, DestVal, DestTy, SrcVal, SrcTy, Loc); + EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestLVal, SrcVal, SrcTy, Loc); break; } case CK_HLSLElementwiseCast: { Expr *Src = E->getSubExpr(); QualType SrcTy = Src->getType(); RValue RV = CGF.EmitAnyExpr(Src); - QualType DestTy = E->getType(); - Address DestVal = Dest.getAddress(); + LValue DestLVal = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); SourceLocation Loc = E->getExprLoc(); if (RV.isScalar()) { llvm::Value *SrcVal = RV.getScalarVal(); - EmitHLSLScalarFlatCast(CGF, DestVal, DestTy, SrcVal, SrcTy, Loc); + assert(SrcTy->isVectorType() && + "HLSL Elementwise cast doesn't handle splatting."); + EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestLVal, SrcVal, SrcTy, Loc); } else { assert(RV.isAggregate() && "Can't perform HLSL Aggregate cast on a complex type."); Address SrcVal = RV.getAggregateAddress(); - EmitHLSLElementwiseCast(CGF, DestVal, DestTy, SrcVal, SrcTy, Loc); + EmitHLSLElementwiseCast(CGF, DestLVal, CGF.MakeAddrLValue(SrcVal, SrcTy), + Loc); } break; } diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index 4fa25c5d66669..00ebf41b315bb 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -2392,39 +2392,36 @@ bool CodeGenFunction::ShouldNullCheckClassCastValue(const CastExpr *CE) { } // RHS is an aggregate type -static Value *EmitHLSLElementwiseCast(CodeGenFunction &CGF, Address RHSVal, - QualType RHSTy, QualType LHSTy, - SourceLocation Loc) { - SmallVector<std::pair<Address, llvm::Value *>, 16> LoadGEPList; - SmallVector<QualType, 16> SrcTypes; // Flattened type - CGF.FlattenAccessAndType(RHSVal, RHSTy, LoadGEPList, SrcTypes); - // LHS is either a vector or a builtin? +static Value *EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue SrcVal, + QualType DestTy, SourceLocation Loc) { + SmallVector<LValue, 16> LoadList; + CGF.FlattenAccessAndTypeLValue(SrcVal, LoadList); + // Dest is either a vector or a builtin? // if its a vector create a temp alloca to store into and return that - if (auto *VecTy = LHSTy->getAs<VectorType>()) { - assert(SrcTypes.size() >= VecTy->getNumElements() && + if (auto *VecTy = DestTy->getAs<VectorType>()) { + assert(LoadList.size() >= VecTy->getNumElements() && "Flattened type on RHS must have more elements than vector on LHS."); llvm::Value *V = - CGF.Builder.CreateLoad(CGF.CreateIRTemp(LHSTy, "flatcast.tmp")); + CGF.Builder.CreateLoad(CGF.CreateIRTemp(DestTy, "flatcast.tmp")); // write to V. for (unsigned I = 0, E = VecTy->getNumElements(); I < E; I++) { - llvm::Value *Load = CGF.Builder.CreateLoad(LoadGEPList[I].first, "load"); - llvm::Value *Idx = LoadGEPList[I].second; - Load = Idx ? CGF.Builder.CreateExtractElement(Load, Idx, "vec.extract") - : Load; - llvm::Value *Cast = CGF.EmitScalarConversion( - Load, SrcTypes[I], VecTy->getElementType(), Loc); + RValue RVal = CGF.EmitLoadOfLValue(LoadList[I], Loc); + assert(RVal.isScalar() && + "All flattened source values should be scalars."); + llvm::Value *Cast = + CGF.EmitScalarConversion(RVal.getScalarVal(), LoadList[I].getType(), + VecTy->getElementType(), Loc); V = CGF.Builder.CreateInsertElement(V, Cast, I); } return V; } - // i its a builtin just do an extract element or load. - assert(LHSTy->isBuiltinType() && + // if its a builtin just do an extract element or load. + assert(DestTy->isBuiltinType() && "Destination type must be a vector or builtin type."); - llvm::Value *Load = CGF.Builder.CreateLoad(LoadGEPList[0].first, "load"); - llvm::Value *Idx = LoadGEPList[0].second; - Load = - Idx ? CGF.Builder.CreateExtractElement(Load, Idx, "vec.extract") : Load; - return CGF.EmitScalarConversion(Load, LHSTy, SrcTypes[0], Loc); + RValue RVal = CGF.EmitLoadOfLValue(LoadList[0], Loc); + assert(RVal.isScalar() && "All flattened source values should be scalars."); + return CGF.EmitScalarConversion(RVal.getScalarVal(), LoadList[0].getType(), + DestTy, Loc); } // VisitCastExpr - Emit code for an explicit or implicit cast. Implicit casts @@ -2949,12 +2946,11 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { case CK_HLSLElementwiseCast: { RValue RV = CGF.EmitAnyExpr(E); SourceLocation Loc = CE->getExprLoc(); - QualType SrcTy = E->getType(); assert(RV.isAggregate() && "Not a valid HLSL Elementwise Cast."); // RHS is an aggregate - Address SrcVal = RV.getAggregateAddress(); - return EmitHLSLElementwiseCast(CGF, SrcVal, SrcTy, DestTy, Loc); + LValue SrcVal = CGF.MakeAddrLValue(RV.getAggregateAddress(), E->getType()); + return EmitHLSLElementwiseCast(CGF, SrcVal, DestTy, Loc); } } // end of switch diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 727487b46054f..a36aaca38b0e9 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4463,10 +4463,8 @@ class CodeGenFunction : public CodeGenTypeCache { AggValueSlot slot = AggValueSlot::ignored()); LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e); - void FlattenAccessAndType( - Address Addr, QualType AddrTy, - SmallVectorImpl<std::pair<Address, llvm::Value *>> &AccessList, - SmallVectorImpl<QualType> &FlatTypes); + void FlattenAccessAndTypeLValue(LValue LVal, + SmallVectorImpl<LValue> &AccessList); llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface, ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/161263 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
