https://github.com/bob80905 updated https://github.com/llvm/llvm-project/pull/208128
>From 997234df5400207a5de0c750ed0ec12b70475e0a Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Tue, 7 Jul 2026 18:00:56 -0700 Subject: [PATCH 1/7] first attempt, add interlockedadd resource methods --- clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 68 +++++++++ clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h | 4 + clang/lib/Sema/HLSLExternalSemaSource.cpp | 2 + clang/lib/Sema/SemaHLSL.cpp | 23 +++ .../ByteAddressBuffer-InterlockedAdd.hlsl | 80 ++++++++++ ...teAddressBuffer-InterlockedAdd-errors.hlsl | 63 ++++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 8 + llvm/lib/Target/DirectX/DXIL.td | 11 ++ llvm/lib/Target/DirectX/DXILOpLowering.cpp | 30 ++++ .../lib/Target/DirectX/DXILResourceAccess.cpp | 141 ++++++++++++++++++ .../DirectX/ResourceAtomicBinOp-i64-sm65.ll | 16 ++ .../CodeGen/DirectX/ResourceAtomicBinOp.ll | 59 ++++++++ 12 files changed, 505 insertions(+) create mode 100644 clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll create mode 100644 llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp index f8018729b4644..245487b71875f 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp @@ -1635,6 +1635,38 @@ BuiltinTypeDeclBuilder::addByteAddressBufferStoreMethods() { return *this; } +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { + assert(!Record->isCompleteDefinition() && "record is already complete"); + ASTContext &AST = SemaRef.getASTContext(); + + // Each entry declares two overloads (with and without an out original-value + // parameter). Adding a new atomic here only requires a new line — the shared + // helper takes care of the composition. + addByteAddressBufferInterlockedMethod("InterlockedAdd", AST.UnsignedIntTy, + "__builtin_hlsl_interlocked_add"); + + // 64-bit typed atomics on UAVs require SM 6.6 (DXIL 1.6 introduces the + // int64 overload of the atomicBinOp op). Skip synthesizing the *64 methods + // on older DXIL targets so callers get "no matching member function" from + // overload resolution — this matches DXC and mirrors how other HLSL SM-gated + // features are handled (see hlsl_intrinsics.h `_HLSL_AVAILABILITY` + // annotations). Non-DXIL targets (e.g., SPIR-V) always get the method: their + // 64-bit atomic support is gated by device extensions, not shader model. + const llvm::Triple &TT = AST.getTargetInfo().getTriple(); + bool DXILNeedsSM66 = + TT.getArch() == llvm::Triple::dxil && + AST.getTargetInfo().getPlatformMinVersion() < VersionTuple(6, 6); + if (!DXILNeedsSM66) { + // HLSL's uint64_t is `unsigned long`. + addByteAddressBufferInterlockedMethod("InterlockedAdd64", + AST.UnsignedLongTy, + "__builtin_hlsl_interlocked_add"); + } + + return *this; +} + BuiltinTypeDeclBuilder & BuiltinTypeDeclBuilder::addSampleMethods(ResourceDimension Dim, bool IsArray) { assert(!Record->isCompleteDefinition() && "record is already complete"); @@ -2356,6 +2388,42 @@ BuiltinTypeDeclBuilder::addStoreFunction(DeclarationName &Name, bool IsConst, .finalize(); } +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethod( + StringRef MethodName, QualType ValueTy, StringRef BuiltinName) { + assert(!Record->isCompleteDefinition() && "record is already complete"); + ASTContext &AST = SemaRef.getASTContext(); + using PH = BuiltinTypeMethodBuilder::PlaceHolder; + + // Interlocked atomics operate on a typed slot in the buffer. Compose + // `resource_getpointer_typed` with the scalar `__builtin_hlsl_interlocked_*` + // builtin so backend lowering (DXIL and SPIR-V) can pattern-match a + // resource-pointer atomicrmw. + QualType AddrSpaceElemTy = + AST.getAddrSpaceQualType(ValueTy, LangAS::hlsl_device); + QualType ElemPtrTy = AST.getPointerType(AddrSpaceElemTy); + + auto BuildOverload = [&](bool WithOriginalValue) { + BuiltinTypeMethodBuilder MMB(*this, MethodName, AST.VoidTy); + MMB.addParam("Offset", AST.UnsignedIntTy).addParam("Value", ValueTy); + if (WithOriginalValue) + MMB.addParam("OriginalValue", ValueTy, + HLSLParamModifierAttr::Keyword_out); + MMB.callBuiltin("__builtin_hlsl_resource_getpointer_typed", ElemPtrTy, + PH::Handle, PH::_0, ValueTy) + .dereference(PH::LastStmt); + if (WithOriginalValue) + MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1, PH::_2); + else + MMB.callBuiltin(BuiltinName, AST.VoidTy, PH::LastStmt, PH::_1); + MMB.finalize(); + }; + + BuildOverload(/*WithOriginalValue=*/false); + BuildOverload(/*WithOriginalValue=*/true); + return *this; +} + BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() { using PH = BuiltinTypeMethodBuilder::PlaceHolder; ASTContext &AST = SemaRef.getASTContext(); diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h index 09cf1fceca116..e809ef264198c 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h @@ -106,6 +106,7 @@ class BuiltinTypeDeclBuilder { bool IsArray = false); BuiltinTypeDeclBuilder &addByteAddressBufferLoadMethods(); BuiltinTypeDeclBuilder &addByteAddressBufferStoreMethods(); + BuiltinTypeDeclBuilder &addByteAddressBufferInterlockedMethods(); BuiltinTypeDeclBuilder &addSampleMethods(ResourceDimension Dim, bool IsArray = false); BuiltinTypeDeclBuilder &addSampleBiasMethods(ResourceDimension Dim, @@ -134,6 +135,9 @@ class BuiltinTypeDeclBuilder { QualType ReturnTy = QualType()); BuiltinTypeDeclBuilder &addStoreFunction(DeclarationName &Name, bool IsConst, QualType ValueType); + BuiltinTypeDeclBuilder & + addByteAddressBufferInterlockedMethod(StringRef MethodName, QualType ValueTy, + StringRef BuiltinName); BuiltinTypeDeclBuilder &addAppendMethod(); BuiltinTypeDeclBuilder &addConsumeMethod(); diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp index 7578f20a27f18..605d630489275 100644 --- a/clang/lib/Sema/HLSLExternalSemaSource.cpp +++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp @@ -661,6 +661,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { /*RawBuffer=*/true, /*HasCounter=*/false) .addByteAddressBufferLoadMethods() .addByteAddressBufferStoreMethods() + .addByteAddressBufferInterlockedMethods() .addGetDimensionsMethodForBuffer() .completeDefinition(); }); @@ -670,6 +671,7 @@ void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { onCompletion(Decl, [this](CXXRecordDecl *Decl) { setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, /*IsROV=*/true, /*RawBuffer=*/true, /*HasCounter=*/false) + .addByteAddressBufferInterlockedMethods() .addGetDimensionsMethodForBuffer() .completeDefinition(); }); diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index c333f0dd4c872..4cedfe893d91c 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -4590,6 +4590,29 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; } + // 64-bit interlocked ops require SM 6.6 on DXIL — the DXIL 1.6 int64 + // overloads of atomicBinOp/cmpXchg are what enable them. The synthesized + // wrapper methods (e.g. RWByteAddressBuffer::InterlockedAdd64) that call + // this builtin are themselves only declared when the target supports it + // (see HLSLBuiltinTypeDeclBuilder), so pre-SM6.6 usage is caught by + // overload resolution. This defensive check catches direct + // `__builtin_hlsl_interlocked_add` calls from HLSL code with a 64-bit + // dest on pre-SM6.6 DXIL targets. Skip synthetic invocations (invalid + // source location) built while composing wrapper method bodies. + const TargetInfo &TI = SemaRef.Context.getTargetInfo(); + if (TheCall->getBeginLoc().isValid() && + TI.getTriple().getArch() == llvm::Triple::dxil && + SemaRef.Context.getTypeSize(DestTy) == 64 && + TI.getPlatformMinVersion() < VersionTuple(6, 6)) { + llvm::StringRef PlatformName( + AvailabilityAttr::getPrettyPlatformName(TI.getPlatformName())); + SemaRef.Diag(TheCall->getBeginLoc(), diag::warn_hlsl_availability) + << TheCall->getDirectCallee() << PlatformName + << VersionTuple(6, 6).getAsString() << /*UseEnvironment=*/false + << /*EnvName=*/""; + return true; + } + if (CheckModifiableLValue(&SemaRef, TheCall, 0)) return true; diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl new file mode 100644 index 0000000000000..ae6179dc4bc99 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl @@ -0,0 +1,80 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ +// RUN: dxil-pc-shadermodel6.6-compute %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK + +// Test that the RWByteAddressBuffer::InterlockedAdd and +// RasterizerOrderedByteAddressBuffer::InterlockedAdd member methods lower to +// `dx.resource.getpointer.typed -> dx.interlocked.add`, and that the +// 3-argument overload stores the returned original value through the out +// parameter. + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +export void test_bab_int_2arg(uint off, int v) { + BAB.InterlockedAdd(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: store i32 %[[R]], ptr {{.*}} +export void test_bab_uint_3arg(uint off, uint v, out uint orig) { + BAB.InterlockedAdd(off, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +export void test_rovb_int_2arg(uint off, int v) { + ROVB.InterlockedAdd(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: store i32 %[[R]], ptr {{.*}} +export void test_rovb_uint_3arg(uint off, uint v, out uint orig) { + ROVB.InterlockedAdd(off, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int64_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +export void test_bab_int64_2arg(uint off, int64_t v) { + BAB.InterlockedAdd64(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: store i64 %[[R]], ptr {{.*}} +export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) { + BAB.InterlockedAdd64(off, v, orig); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int64_2arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +export void test_rovb_int64_2arg(uint off, int64_t v) { + ROVB.InterlockedAdd64(off, v); +} + +// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint64_3arg +// DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} +// DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: store i64 %[[R]], ptr {{.*}} +export void test_rovb_uint64_3arg(uint off, uint64_t v, out uint64_t orig) { + ROVB.InterlockedAdd64(off, v, orig); +} diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl new file mode 100644 index 0000000000000..13fc70d700421 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl @@ -0,0 +1,63 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \ +// RUN: -verify-ignore-unexpected=note,warning + +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-compute -DTEST_SM65 %s -fsyntax-only \ +// RUN: -verify -verify-ignore-unexpected=note,warning + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +struct S { int x; }; + +#ifdef TEST_SM65 + +// InterlockedAdd64 is only synthesized on DXIL when the shader model is at +// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so +// the reference must fail with "no member named". +void sm65_no_bab_add64(uint off, int64_t v) { + BAB.InterlockedAdd64(off, v); + // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}} +} + +void sm65_no_rovb_add64(uint off, int64_t v) { + ROVB.InterlockedAdd64(off, v); + // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}} +} + +// 32-bit InterlockedAdd is always available. +void sm65_bab_add32_ok(uint off, int v) { + BAB.InterlockedAdd(off, v); +} + +#else + +void too_few(uint off) { + BAB.InterlockedAdd(off); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +void too_many(uint off, int v, int extra) { + int orig; + BAB.InterlockedAdd(off, v, orig, extra); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +void struct_value(uint off, S v) { + BAB.InterlockedAdd(off, v); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +// Same shape of errors on RasterizerOrderedByteAddressBuffer. +void rovb_too_few(uint off) { + ROVB.InterlockedAdd(off); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +void rovb_struct_value(uint off, S v) { + ROVB.InterlockedAdd(off, v); + // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} +} + +#endif diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index 4dd86270f0d01..f1b3845ea1efa 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -61,6 +61,14 @@ def int_dx_resource_store_rawbuffer : DefaultAttrsIntrinsic< [], [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_any_ty], [IntrWriteMem]>; +// Resource atomic binary op: performs an atomic read-modify-write on a UAV +// resource element and returns the original value. The i32 operation code +// matches DXIL's AtomicBinOpCode enum. +def int_dx_resource_atomicbinop + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, LLVMMatchType<0>], + [IntrArgMemOnly]>; // dx.resource.load.cbufferrow encodes the number of elements returned in the // function name. The total size of the return should always be 128 bits. diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index a268276b07655..7c57c8a7f4aa1 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -1006,6 +1006,17 @@ def BufferStore : DXILOp<69, bufferStore> { let stages = [Stages<DXIL1_0, [all_stages]>]; } +def AtomicBinOp : DXILOp<78, atomicBinOp> { + let Doc = "performs an atomic read-modify-write on a UAV resource " + "element, returning the original value"; + // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue + let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy]; + let result = OverloadTy; + let overloads = [Overloads<DXIL1_0, [Int32Ty]>, + Overloads<DXIL1_6, [Int32Ty, Int64Ty]>]; + let stages = [Stages<DXIL1_0, [all_stages]>]; +} + def UpdateCounter : DXILOp<70, bufferUpdateCounter> { let Doc = "increments/decrements a buffer counter"; let arguments = [HandleTy, Int8Ty]; diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp index 93d5a08a6e0a2..64e0de8e55bdd 100644 --- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp +++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp @@ -849,6 +849,33 @@ class OpLowerer { }); } + [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) { + IRBuilder<> &IRB = OpBuilder.getIRB(); + + return replaceFunction(F, [&](CallInst *CI) -> Error { + IRB.SetInsertPoint(CI); + Value *Handle = + createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType()); + Value *Index = CI->getArgOperand(1); + Value *Offset = CI->getArgOperand(2); + Value *BinOp = CI->getArgOperand(3); + Value *NewValue = CI->getArgOperand(4); + + std::array<Value *, 6> Args{Handle, BinOp, Index, + Offset, IRB.getInt32(0), NewValue}; + + Expected<CallInst *> OpCall = OpBuilder.tryCreateOp( + OpCode::AtomicBinOp, Args, CI->getName(), CI->getType()); + + if (Error E = OpCall.takeError()) + return E; + + CI->replaceAllUsesWith(*OpCall); + CI->eraseFromParent(); + return Error::success(); + }); + } + [[nodiscard]] bool lowerGetDimensionsX(Function &F) { IRBuilder<> &IRB = OpBuilder.getIRB(); Type *Int32Ty = IRB.getInt32Ty(); @@ -1210,6 +1237,9 @@ class OpLowerer { case Intrinsic::dx_resource_updatecounter: HasErrors |= lowerUpdateCounter(F); break; + case Intrinsic::dx_resource_atomicbinop: + HasErrors |= lowerResourceAtomicBinOp(F); + break; case Intrinsic::dx_resource_getdimensions_x: HasErrors |= lowerGetDimensionsX(F); break; diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp index 25d860e615c17..5f5c4660e744b 100644 --- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp +++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp @@ -26,6 +26,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Transforms/Utils/ValueMapper.h" +#include <optional> #define DEBUG_TYPE "dxil-resource-access" @@ -230,6 +231,114 @@ static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI, llvm_unreachable("Unhandled case in switch"); } +static std::optional<unsigned> getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) { + switch (BinOp) { + case AtomicRMWInst::Add: + return 0; + case AtomicRMWInst::And: + return 1; + case AtomicRMWInst::Or: + return 2; + case AtomicRMWInst::Xor: + return 3; + case AtomicRMWInst::Min: + return 4; + case AtomicRMWInst::Max: + return 5; + case AtomicRMWInst::UMin: + return 6; + case AtomicRMWInst::UMax: + return 7; + case AtomicRMWInst::Xchg: + return 8; + case AtomicRMWInst::Sub: + case AtomicRMWInst::Nand: + case AtomicRMWInst::FAdd: + case AtomicRMWInst::FSub: + case AtomicRMWInst::FMax: + case AtomicRMWInst::FMin: + case AtomicRMWInst::FMaximum: + case AtomicRMWInst::FMinimum: + case AtomicRMWInst::FMaximumNum: + case AtomicRMWInst::FMinimumNum: + case AtomicRMWInst::UIncWrap: + case AtomicRMWInst::UDecWrap: + case AtomicRMWInst::USubCond: + case AtomicRMWInst::USubSat: + case AtomicRMWInst::BAD_BINOP: + return std::nullopt; + } + llvm_unreachable("Unhandled atomicrmw operation"); +} + +static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI) { + std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation()); + if (!BinOpCode) { + reportFatalUsageError("DXIL resource atomicrmw operation not implemented"); + return; + } + + const DataLayout &DL = AI->getDataLayout(); + IRBuilder<> Builder(AI); + Value *Index = II->getOperand(1); + + // The offset for the rawbuffer load/store/atomic ops is always in bytes. + uint64_t AccessSize = 1; + Value *Offset = + traverseGEPOffsets(DL, Builder, AI->getPointerOperand(), AccessSize); + + // For raw buffer (ie, HLSL's ByteAddressBuffer), we need to fold the access + // entirely into the index. + if (!RTI.isStruct()) { + auto *ConstantOffset = dyn_cast<ConstantInt>(Offset); + if (!ConstantOffset || !ConstantOffset->isZero()) + Index = Builder.CreateAdd(Index, Offset); + Offset = llvm::PoisonValue::get(Builder.getInt32Ty()); + } + + auto *BinOp = Builder.getInt32(*BinOpCode); + Value *V = Builder.CreateIntrinsic( + AI->getType(), Intrinsic::dx_resource_atomicbinop, + {II->getOperand(0), Index, Offset, BinOp, AI->getValOperand()}); + AI->replaceAllUsesWith(V); +} + +static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI) { + switch (RTI.getResourceKind()) { + case dxil::ResourceKind::TypedBuffer: + case dxil::ResourceKind::RawBuffer: + case dxil::ResourceKind::StructuredBuffer: + return createAtomicBinOp(II, AI, RTI); + case dxil::ResourceKind::Texture1D: + case dxil::ResourceKind::Texture2D: + case dxil::ResourceKind::Texture2DMS: + case dxil::ResourceKind::Texture3D: + case dxil::ResourceKind::TextureCube: + case dxil::ResourceKind::Texture1DArray: + case dxil::ResourceKind::Texture2DArray: + case dxil::ResourceKind::Texture2DMSArray: + case dxil::ResourceKind::TextureCubeArray: + case dxil::ResourceKind::FeedbackTexture2D: + case dxil::ResourceKind::FeedbackTexture2DArray: + reportFatalUsageError( + "DXIL atomicrmw not implemented for texture resources"); + return; + case dxil::ResourceKind::CBuffer: + case dxil::ResourceKind::Sampler: + case dxil::ResourceKind::TBuffer: + reportFatalUsageError( + "DXIL atomicrmw not implemented for this resource type"); + return; + case dxil::ResourceKind::RTAccelerationStructure: + case dxil::ResourceKind::Invalid: + case dxil::ResourceKind::NumEntries: + llvm_unreachable("Invalid resource kind for atomicrmw"); + } + llvm_unreachable("Unhandled case in switch"); +} + static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI) { const DataLayout &DL = LI->getDataLayout(); @@ -550,6 +659,8 @@ static Instruction *getStoreLoadPointerOperand(Instruction *AI) { return dyn_cast<Instruction>(LI->getPointerOperand()); if (auto *SI = dyn_cast<StoreInst>(AI)) return dyn_cast<Instruction>(SI->getPointerOperand()); + if (auto *RMWI = dyn_cast<AtomicRMWInst>(AI)) + return dyn_cast<Instruction>(RMWI->getPointerOperand()); return nullptr; } @@ -786,6 +897,36 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) { } else if (auto *LI = dyn_cast<LoadInst>(U)) { createLoadIntrinsic(II, LI, RTI); DeadInsts.push_back(LI); + } else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) { + createAtomicBinOpIntrinsic(II, AI, RTI); + DeadInsts.push_back(AI); + } else if (auto *CI = dyn_cast<CallInst>(U)) { + // `dx.interlocked.*` intrinsics wrap an atomicrmw and are expanded to + // one by DXILIntrinsicExpansion — but that pass runs after this one, so + // when the source of the pointer is a resource we must expand them here + // (and immediately process the resulting atomicrmw) instead of letting + // the pointer escape. + auto *IntrinCall = dyn_cast<IntrinsicInst>(CI); + std::optional<AtomicRMWInst::BinOp> Op; + if (IntrinCall) { + switch (IntrinCall->getIntrinsicID()) { + case Intrinsic::dx_interlocked_add: + Op = AtomicRMWInst::Add; + break; + default: + break; + } + } + if (!Op) + llvm_unreachable("Unhandled instruction - pointer escaped?"); + IRBuilder<> Builder(IntrinCall); + auto *AI = Builder.CreateAtomicRMW( + *Op, IntrinCall->getArgOperand(0), IntrinCall->getArgOperand(1), + MaybeAlign(), AtomicOrdering::Monotonic); + IntrinCall->replaceAllUsesWith(AI); + createAtomicBinOpIntrinsic(II, AI, RTI); + DeadInsts.push_back(AI); + DeadInsts.push_back(IntrinCall); } else llvm_unreachable("Unhandled instruction - pointer escaped?"); } diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll new file mode 100644 index 0000000000000..6f5a40e0b6c2a --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp-i64-sm65.ll @@ -0,0 +1,16 @@ +; RUN: not opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.5-compute %s 2>&1 | FileCheck %s + +; Verify resource i64 atomicrmw rejects shader models before SM 6.6, where +; dx.op.atomicBinOp gained i64 overload support. + +target triple = "dxil-pc-shadermodel6.5-compute" + +define i64 @atomic_i64(i32 %index, i64 %value) { + %buffer = call target("dx.RawBuffer", i64, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index) + ; CHECK: Cannot create AtomicBinOp operation: Invalid overload type + %old = atomicrmw add ptr %ptr, i64 %value monotonic + ret i64 %old +} diff --git a/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll new file mode 100644 index 0000000000000..bc852c8a2c81f --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ResourceAtomicBinOp.ll @@ -0,0 +1,59 @@ +; RUN: opt -S -dxil-resource-access -dxil-op-lower %s | FileCheck %s --check-prefixes=CHECK,I32 +; RUN: opt -S -dxil-resource-access -dxil-op-lower -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s --check-prefixes=CHECK,I32,I64 + +; Verify atomicrmw through a dx.resource.getpointer is lowered to +; dx.op.atomicBinOp for UAV resources. + +target triple = "dxil-pc-shadermodel6.6-compute" + +; CHECK-LABEL: define i32 @atomic_i32( +define i32 @atomic_i32(i32 %index, i32 %value) { + %buffer = call target("dx.RawBuffer", i32, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i32, 1, 0, 0) %buffer, i32 %index) + + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i32 %value) + %add = atomicrmw add ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 1, i32 %index, i32 0, i32 0, i32 %value) + %and = atomicrmw and ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 2, i32 %index, i32 0, i32 0, i32 %value) + %or = atomicrmw or ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 3, i32 %index, i32 0, i32 0, i32 %value) + %xor = atomicrmw xor ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 4, i32 %index, i32 0, i32 0, i32 %value) + %min = atomicrmw min ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 5, i32 %index, i32 0, i32 0, i32 %value) + %max = atomicrmw max ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 6, i32 %index, i32 0, i32 0, i32 %value) + %umin = atomicrmw umin ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 7, i32 %index, i32 0, i32 0, i32 %value) + %umax = atomicrmw umax ptr %ptr, i32 %value monotonic + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 8, i32 %index, i32 0, i32 0, i32 %value) + %xchg = atomicrmw xchg ptr %ptr, i32 %value monotonic + ret i32 %xchg +} + +; CHECK-LABEL: define i32 @atomic_i32_byteaddress( +define i32 @atomic_i32_byteaddress(i32 %offset, i32 %value) { + %buffer = call target("dx.RawBuffer", i8, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i8, 1, 0, 0) %buffer, i32 %offset) + + ; I32: call i32 @dx.op.atomicBinOp.i32(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %offset, i32 poison, i32 0, i32 %value) + %old = atomicrmw add ptr %ptr, i32 %value monotonic + ret i32 %old +} + +; CHECK-LABEL: define i64 @atomic_i64( +define i64 @atomic_i64(i32 %index, i64 %value) { + %buffer = call target("dx.RawBuffer", i64, 1, 0, 0) + @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, ptr null) + %ptr = call ptr @llvm.dx.resource.getpointer( + target("dx.RawBuffer", i64, 1, 0, 0) %buffer, i32 %index) + + ; I64: call i64 @dx.op.atomicBinOp.i64(i32 78, %dx.types.Handle %{{.*}}, i32 0, i32 %index, i32 0, i32 0, i64 %value) + %old = atomicrmw add ptr %ptr, i64 %value monotonic + ret i64 %old +} >From b34ffa6e2dc6b1354ddfab9719f86d049a628660 Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Wed, 8 Jul 2026 16:16:27 -0700 Subject: [PATCH 2/7] self review: Move op def to appropriate spot --- llvm/lib/Target/DirectX/DXIL.td | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 7c57c8a7f4aa1..3d978c207f104 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -1006,17 +1006,6 @@ def BufferStore : DXILOp<69, bufferStore> { let stages = [Stages<DXIL1_0, [all_stages]>]; } -def AtomicBinOp : DXILOp<78, atomicBinOp> { - let Doc = "performs an atomic read-modify-write on a UAV resource " - "element, returning the original value"; - // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue - let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy]; - let result = OverloadTy; - let overloads = [Overloads<DXIL1_0, [Int32Ty]>, - Overloads<DXIL1_6, [Int32Ty, Int64Ty]>]; - let stages = [Stages<DXIL1_0, [all_stages]>]; -} - def UpdateCounter : DXILOp<70, bufferUpdateCounter> { let Doc = "increments/decrements a buffer counter"; let arguments = [HandleTy, Int8Ty]; @@ -1041,6 +1030,17 @@ def GetDimensions : DXILOp<72, getDimensions> { let stages = [Stages<DXIL1_0, [all_stages]>]; } +def AtomicBinOp : DXILOp<78, atomicBinOp> { + let Doc = "performs an atomic read-modify-write on a UAV resource " + "element, returning the original value"; + // Handle, AtomicBinOpCode, Coord0, Coord1, Coord2, NewValue + let arguments = [HandleTy, Int32Ty, Int32Ty, Int32Ty, Int32Ty, OverloadTy]; + let result = OverloadTy; + let overloads = [Overloads<DXIL1_0, [Int32Ty]>, + Overloads<DXIL1_6, [Int32Ty, Int64Ty]>]; + let stages = [Stages<DXIL1_0, [all_stages]>]; +} + def Barrier : DXILOp<80, barrier> { let Doc = "inserts a memory barrier in the shader"; let intrinsics = [ >From d42ceac6a97b78e22a10b5a74a9ee8ad7ab5cadb Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Wed, 15 Jul 2026 16:11:19 -0700 Subject: [PATCH 3/7] address Farzon --- .../clang/Basic/DiagnosticSemaKinds.td | 3 + clang/lib/CodeGen/CGHLSLBuiltins.cpp | 28 ++++++- clang/lib/CodeGen/CGHLSLRuntime.h | 1 - clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp | 21 ++--- clang/lib/Sema/SemaHLSL.cpp | 21 ++--- .../ByteAddressBuffer-InterlockedAdd.hlsl | 32 +++---- .../CodeGenHLSL/builtins/InterlockedAdd.hlsl | 24 +++--- ...teAddressBuffer-InterlockedAdd-errors.hlsl | 28 ------- ...ressBuffer-InterlockedAdd-sm65-errors.hlsl | 32 +++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 13 --- llvm/include/llvm/IR/IntrinsicsSPIRV.td | 4 - .../Target/DirectX/DXILIntrinsicExpansion.cpp | 4 - llvm/lib/Target/DirectX/DXILOpLowering.cpp | 43 +++------- .../lib/Target/DirectX/DXILResourceAccess.cpp | 83 ++++++++++--------- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 2 - llvm/test/CodeGen/DirectX/InterlockedAdd.ll | 52 ------------ .../SPIRV/hlsl-intrinsics/InterlockedAdd.ll | 36 -------- .../hlsl-intrinsics/InterlockedAdd_spv_i64.ll | 37 --------- 18 files changed, 159 insertions(+), 305 deletions(-) create mode 100644 clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl delete mode 100644 llvm/test/CodeGen/DirectX/InterlockedAdd.ll delete mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll delete mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index f14288dd2967d..fae7a22859425 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -13788,6 +13788,9 @@ def err_hlsl_atomic_arg_addr_space : Error< "%ordinal0 argument to atomic builtin must reference groupshared or device " "memory (was %1)">; +def err_hlsl_builtin_requires_sm : Error< + "%0 requires shader model %1 or newer">; + def err_hlsl_export_not_on_function : Error< "export declaration can only be used on functions">; diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index 1bda113143e07..ef8be15fdfb40 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -310,6 +310,27 @@ static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF, llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture"); } +static Value *handleInterlockedAdd(CodeGenFunction &CGF, const CallExpr *E) { + // Emit `atomicrmw add` directly — no intermediate `*.interlocked.add` + // intrinsic needed on either DXIL or SPIR-V. + LValue DestLV = CGF.EmitLValue(E->getArg(0)); + Address DestAddr = DestLV.getAddress(); + Value *Val = CGF.EmitScalarExpr(E->getArg(1)); + assert(E->getArg(1)->getType()->isIntegerType() && + "Intrinsic InterlockedAdd value operand must be an integer"); + + llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW( + llvm::AtomicRMWInst::Add, DestAddr, Val, llvm::AtomicOrdering::Monotonic); + + // The 3-arg overload writes the old value (the RMW's return value) into + // the `original_value` reference parameter. + if (E->getNumArgs() == 3) { + LValue OrigLV = CGF.EmitLValue(E->getArg(2)); + CGF.EmitStoreThroughLValue(RValue::get(Call), OrigLV); + } + return Call; +} + static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E, Intrinsic::ID ID, const Twine &Name) { // HLSL signatures (synthesized as overloads in HLSLExternalSemaSource): @@ -1457,9 +1478,10 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, "hlsl.wave.active.bit.and"); } case Builtin::BI__builtin_hlsl_interlocked_add: { - return handleInterlockedOp( - *this, E, CGM.getHLSLRuntime().getInterlockedAddIntrinsic(), - "hlsl.interlocked.add"); + // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick + // up the raw instruction (DXIL via DXILResourceAccess for resource + // pointers, SPIR-V via selectAtomicRMW). No intermediate intrinsic. + return handleInterlockedAdd(*this, E); } case Builtin::BI__builtin_hlsl_interlocked_or: { return handleInterlockedOp(*this, E, diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index cf47b1633fd3c..5ca73dc91d75e 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -151,7 +151,6 @@ class CGHLSLRuntime { GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitOr, wave_reduce_or) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitXor, wave_reduce_xor) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveBitAnd, wave_reduce_and) - GENERATE_HLSL_INTRINSIC_FUNCTION(InterlockedAdd, interlocked_add) GENERATE_HLSL_INTRINSIC_FUNCTION(InterlockedOr, interlocked_or) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveMax, wave_reduce_max) GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveUMax, wave_reduce_umax) diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp index 245487b71875f..07cbab6d85766 100644 --- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp +++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp @@ -1640,24 +1640,17 @@ BuiltinTypeDeclBuilder::addByteAddressBufferInterlockedMethods() { assert(!Record->isCompleteDefinition() && "record is already complete"); ASTContext &AST = SemaRef.getASTContext(); - // Each entry declares two overloads (with and without an out original-value - // parameter). Adding a new atomic here only requires a new line — the shared - // helper takes care of the composition. + // This is a helper that declares two overloads with and without an out + // original-value parameter for each entry. addByteAddressBufferInterlockedMethod("InterlockedAdd", AST.UnsignedIntTy, "__builtin_hlsl_interlocked_add"); - // 64-bit typed atomics on UAVs require SM 6.6 (DXIL 1.6 introduces the - // int64 overload of the atomicBinOp op). Skip synthesizing the *64 methods - // on older DXIL targets so callers get "no matching member function" from - // overload resolution — this matches DXC and mirrors how other HLSL SM-gated - // features are handled (see hlsl_intrinsics.h `_HLSL_AVAILABILITY` - // annotations). Non-DXIL targets (e.g., SPIR-V) always get the method: their - // 64-bit atomic support is gated by device extensions, not shader model. + // Skip synthesizing the 64 bit methods on DXIL targets older than SM 6.6. const llvm::Triple &TT = AST.getTargetInfo().getTriple(); - bool DXILNeedsSM66 = - TT.getArch() == llvm::Triple::dxil && - AST.getTargetInfo().getPlatformMinVersion() < VersionTuple(6, 6); - if (!DXILNeedsSM66) { + bool HasInt64AtomicSupport = + TT.getArch() != llvm::Triple::dxil || + AST.getTargetInfo().getPlatformMinVersion() >= VersionTuple(6, 6); + if (HasInt64AtomicSupport) { // HLSL's uint64_t is `unsigned long`. addByteAddressBufferInterlockedMethod("InterlockedAdd64", AST.UnsignedLongTy, diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index 4cedfe893d91c..97913761566b2 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -4590,26 +4590,17 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; } - // 64-bit interlocked ops require SM 6.6 on DXIL — the DXIL 1.6 int64 - // overloads of atomicBinOp/cmpXchg are what enable them. The synthesized - // wrapper methods (e.g. RWByteAddressBuffer::InterlockedAdd64) that call - // this builtin are themselves only declared when the target supports it - // (see HLSLBuiltinTypeDeclBuilder), so pre-SM6.6 usage is caught by - // overload resolution. This defensive check catches direct - // `__builtin_hlsl_interlocked_add` calls from HLSL code with a 64-bit - // dest on pre-SM6.6 DXIL targets. Skip synthetic invocations (invalid - // source location) built while composing wrapper method bodies. + // 64-bit interlocked ops require SM 6.6 on DXIL. The synthesized wrapper + // methods (e.g. RWByteAddressBuffer::InterlockedAdd64) are only declared + // on SM 6.6+, so this defensive check only fires for direct builtin + // calls; skip synthetic invocations (invalid source location). const TargetInfo &TI = SemaRef.Context.getTargetInfo(); if (TheCall->getBeginLoc().isValid() && TI.getTriple().getArch() == llvm::Triple::dxil && SemaRef.Context.getTypeSize(DestTy) == 64 && TI.getPlatformMinVersion() < VersionTuple(6, 6)) { - llvm::StringRef PlatformName( - AvailabilityAttr::getPrettyPlatformName(TI.getPlatformName())); - SemaRef.Diag(TheCall->getBeginLoc(), diag::warn_hlsl_availability) - << TheCall->getDirectCallee() << PlatformName - << VersionTuple(6, 6).getAsString() << /*UseEnvironment=*/false - << /*EnvName=*/""; + SemaRef.Diag(TheCall->getBeginLoc(), diag::err_hlsl_builtin_requires_sm) + << TheCall->getDirectCallee() << VersionTuple(6, 6).getAsString(); return true; } diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl index ae6179dc4bc99..a7306249e21c7 100644 --- a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl +++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl @@ -11,69 +11,69 @@ RWByteAddressBuffer BAB : register(u0); RasterizerOrderedByteAddressBuffer ROVB : register(u1); -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int_2arg +// CHECK-LABEL: define void @{{.*}}test_bab_int_2arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic export void test_bab_int_2arg(uint off, int v) { BAB.InterlockedAdd(off, v); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint_3arg +// CHECK-LABEL: define void @{{.*}}test_bab_uint_3arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic // DXCHECK: store i32 %[[R]], ptr {{.*}} export void test_bab_uint_3arg(uint off, uint v, out uint orig) { BAB.InterlockedAdd(off, v, orig); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int_2arg +// CHECK-LABEL: define void @{{.*}}test_rovb_int_2arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic export void test_rovb_int_2arg(uint off, int v) { ROVB.InterlockedAdd(off, v); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint_3arg +// CHECK-LABEL: define void @{{.*}}test_rovb_uint_3arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p0(ptr %[[PTR]], i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i32 %{{.*}} monotonic // DXCHECK: store i32 %[[R]], ptr {{.*}} export void test_rovb_uint_3arg(uint off, uint v, out uint orig) { ROVB.InterlockedAdd(off, v, orig); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_int64_2arg +// CHECK-LABEL: define void @{{.*}}test_bab_int64_2arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic export void test_bab_int64_2arg(uint off, int64_t v) { BAB.InterlockedAdd64(off, v); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_bab_uint64_3arg +// CHECK-LABEL: define void @{{.*}}test_bab_uint64_3arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 0), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_0t.i32(target("dx.RawBuffer", i8, 1, 0) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic // DXCHECK: store i64 %[[R]], ptr {{.*}} export void test_bab_uint64_3arg(uint off, uint64_t v, out uint64_t orig) { BAB.InterlockedAdd64(off, v, orig); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_int64_2arg +// CHECK-LABEL: define void @{{.*}}test_rovb_int64_2arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic export void test_rovb_int64_2arg(uint off, int64_t v) { ROVB.InterlockedAdd64(off, v); } -// CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_rovb_uint64_3arg +// CHECK-LABEL: define void @{{.*}}test_rovb_uint64_3arg // DXCHECK: %[[HANDLE:.*]] = load target("dx.RawBuffer", i8, 1, 1), ptr {{.*}} // DXCHECK: %[[PTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i8_1_1t.i32(target("dx.RawBuffer", i8, 1, 1) %[[HANDLE]], i32 %{{.*}}) -// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p0(ptr %[[PTR]], i64 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr %[[PTR]], i64 %{{.*}} monotonic // DXCHECK: store i64 %[[R]], ptr {{.*}} export void test_rovb_uint64_3arg(uint off, uint64_t v, out uint64_t orig) { ROVB.InterlockedAdd64(off, v, orig); diff --git a/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl index da53bba3e0d05..6bb4fd25c4a0c 100644 --- a/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl +++ b/clang/test/CodeGenHLSL/builtins/InterlockedAdd.hlsl @@ -14,45 +14,45 @@ groupshared int64_t gs_i64; groupshared uint64_t gs_u64; // CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int_2arg -// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}) -// SPVCHECK: call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}) +// DXCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic +// SPVCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic export void test_int_2arg(int v) { InterlockedAdd(gs_i32, v); } // CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint_2arg -// DXCHECK: call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}) -// SPVCHECK: call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}) +// DXCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic +// SPVCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic export void test_uint_2arg(uint v) { InterlockedAdd(gs_u32, v); } // CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int_3arg -// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}) -// SPVCHECK: %[[R:.*]] = call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic +// SPVCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_i32{{.*}}, i32 %{{.*}} monotonic // CHECK: store i32 %[[R]], ptr {{.*}} export void test_int_3arg(int v, out int orig) { InterlockedAdd(gs_i32, v, orig); } // CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint_3arg -// DXCHECK: %[[R:.*]] = call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}) -// SPVCHECK: %[[R:.*]] = call spir_func i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic +// SPVCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u32{{.*}}, i32 %{{.*}} monotonic // CHECK: store i32 %[[R]], ptr {{.*}} export void test_uint_3arg(uint v, out uint orig) { InterlockedAdd(gs_u32, v, orig); } // CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_int64_2arg -// DXCHECK: call i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}) -// SPVCHECK: call spir_func i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}}) +// DXCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}} monotonic +// SPVCHECK: atomicrmw add ptr addrspace(3) {{.*}}@gs_i64{{.*}}, i64 %{{.*}} monotonic export void test_int64_2arg(int64_t v) { InterlockedAdd(gs_i64, v); } // CHECK-LABEL: define {{(dso_local |hidden |internal |protected |spir_func )*}}void @{{.*}}test_uint64_3arg -// DXCHECK: %[[R:.*]] = call i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}) -// SPVCHECK: %[[R:.*]] = call spir_func i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}}) +// DXCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}} monotonic +// SPVCHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) {{.*}}@gs_u64{{.*}}, i64 %{{.*}} monotonic // CHECK: store i64 %[[R]], ptr {{.*}} export void test_uint64_3arg(uint64_t v, out uint64_t orig) { InterlockedAdd(gs_u64, v, orig); diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl index 13fc70d700421..234932366e406 100644 --- a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl @@ -2,37 +2,11 @@ // RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \ // RUN: -verify-ignore-unexpected=note,warning -// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ -// RUN: -triple dxil-pc-shadermodel6.5-compute -DTEST_SM65 %s -fsyntax-only \ -// RUN: -verify -verify-ignore-unexpected=note,warning - RWByteAddressBuffer BAB : register(u0); RasterizerOrderedByteAddressBuffer ROVB : register(u1); struct S { int x; }; -#ifdef TEST_SM65 - -// InterlockedAdd64 is only synthesized on DXIL when the shader model is at -// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so -// the reference must fail with "no member named". -void sm65_no_bab_add64(uint off, int64_t v) { - BAB.InterlockedAdd64(off, v); - // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}} -} - -void sm65_no_rovb_add64(uint off, int64_t v) { - ROVB.InterlockedAdd64(off, v); - // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}} -} - -// 32-bit InterlockedAdd is always available. -void sm65_bab_add32_ok(uint off, int v) { - BAB.InterlockedAdd(off, v); -} - -#else - void too_few(uint off) { BAB.InterlockedAdd(off); // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} @@ -59,5 +33,3 @@ void rovb_struct_value(uint off, S v) { ROVB.InterlockedAdd(off, v); // expected-error@-1 {{no matching member function for call to 'InterlockedAdd'}} } - -#endif diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl new file mode 100644 index 0000000000000..5e51cbf36b32b --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl @@ -0,0 +1,32 @@ +// RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ +// RUN: -triple dxil-pc-shadermodel6.5-compute %s -fsyntax-only -verify \ +// RUN: -verify-ignore-unexpected=note,warning + +RWByteAddressBuffer BAB : register(u0); +RasterizerOrderedByteAddressBuffer ROVB : register(u1); + +// InterlockedAdd64 is only synthesized on DXIL when the shader model is at +// least 6.6 (matches DXC). On SM 6.5 the member is not declared at all, so +// the reference must fail with "no member named". +void sm65_no_bab_add64(uint off, int64_t v) { + BAB.InterlockedAdd64(off, v); + // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RWByteAddressBuffer'}} +} + +void sm65_no_rovb_add64(uint off, int64_t v) { + ROVB.InterlockedAdd64(off, v); + // expected-error@-1 {{no member named 'InterlockedAdd64' in 'hlsl::RasterizerOrderedByteAddressBuffer'}} +} + +// 32-bit InterlockedAdd is always available. +void sm65_bab_add32_ok(uint off, int v) { + BAB.InterlockedAdd(off, v); +} + +// Direct calls to the 64-bit interlocked builtin must also be rejected with a +// clear source-location error on pre-SM6.6 DXIL targets. +groupshared int64_t gs_i64; +void sm65_direct_builtin(int64_t v) { + __builtin_hlsl_interlocked_add(gs_i64, v); + // expected-error@-1 {{'__builtin_hlsl_interlocked_add' requires shader model 6.6 or newer}} +} diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index f1b3845ea1efa..ec0e8d156b40a 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -61,15 +61,6 @@ def int_dx_resource_store_rawbuffer : DefaultAttrsIntrinsic< [], [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, llvm_any_ty], [IntrWriteMem]>; -// Resource atomic binary op: performs an atomic read-modify-write on a UAV -// resource element and returns the original value. The i32 operation code -// matches DXIL's AtomicBinOpCode enum. -def int_dx_resource_atomicbinop - : DefaultAttrsIntrinsic<[llvm_anyint_ty], - [llvm_any_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, LLVMMatchType<0>], - [IntrArgMemOnly]>; - // dx.resource.load.cbufferrow encodes the number of elements returned in the // function name. The total size of the return should always be 128 bits. def int_dx_resource_load_cbufferrow_8 @@ -265,10 +256,6 @@ def int_dx_wave_getlaneindex : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrCon def int_dx_wave_reduce_or : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; def int_dx_wave_reduce_xor : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; def int_dx_wave_reduce_and : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem, IntrTriviallyScalarizable]>; -def int_dx_interlocked_add : - DefaultAttrsIntrinsic<[llvm_anyint_ty], - [llvm_anyptr_ty, LLVMMatchType<0>], - [IntrArgMemOnly]>; def int_dx_interlocked_or : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyptr_ty, LLVMMatchType<0>], diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index d948ef78b9584..6a2a5a5cbfc42 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -148,10 +148,6 @@ def int_spv_rsqrt : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty] def int_spv_wave_reduce_or : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>; def int_spv_wave_reduce_xor : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>; def int_spv_wave_reduce_and : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrConvergent, IntrNoMem]>; - def int_spv_interlocked_add : - DefaultAttrsIntrinsic<[llvm_anyint_ty], - [llvm_anyptr_ty, LLVMMatchType<0>], - [IntrArgMemOnly]>; def int_spv_interlocked_or : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyptr_ty, LLVMMatchType<0>], diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp index 1025015c09a1f..f6c316f883a54 100644 --- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp +++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp @@ -228,7 +228,6 @@ static bool isIntrinsicExpansion(Function &F) { case Intrinsic::dx_sign: case Intrinsic::dx_step: case Intrinsic::dx_radians: - case Intrinsic::dx_interlocked_add: case Intrinsic::dx_interlocked_or: case Intrinsic::usub_sat: case Intrinsic::vector_reduce_add: @@ -1341,9 +1340,6 @@ static bool expandIntrinsic(Function &F, CallInst *Orig) { case Intrinsic::dx_radians: Result = expandRadiansIntrinsic(Orig); break; - case Intrinsic::dx_interlocked_add: - Result = expandInterlockedIntrinsic(Orig, AtomicRMWInst::Add); - break; case Intrinsic::dx_interlocked_or: Result = expandInterlockedIntrinsic(Orig, AtomicRMWInst::Or); break; diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp index 64e0de8e55bdd..448a7d4e48736 100644 --- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp +++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp @@ -187,6 +187,19 @@ class OpLowerer { SmallVector<CallInst *> ToRemove; SmallVector<Function *> CastFns; + // Also pick up any `dx.resource.casthandle` calls that were introduced + // outside of this pass (e.g. by DXILResourceAccess when it emits DXIL + // ops directly). All such casts must be resolved here. + for (Function &F : M) { + if (!F.isDeclaration() || + F.getIntrinsicID() != Intrinsic::dx_resource_casthandle) + continue; + for (User *U : F.users()) + if (auto *CI = dyn_cast<CallInst>(U)) + if (!llvm::is_contained(CleanupCasts, CI)) + CleanupCasts.push_back(CI); + } + for (CallInst *Cast : CleanupCasts) { // These casts were only put in to ease the move from `target("dx")` types // to `dx.types.Handle in a piecemeal way. At this point, all of the @@ -849,33 +862,6 @@ class OpLowerer { }); } - [[nodiscard]] bool lowerResourceAtomicBinOp(Function &F) { - IRBuilder<> &IRB = OpBuilder.getIRB(); - - return replaceFunction(F, [&](CallInst *CI) -> Error { - IRB.SetInsertPoint(CI); - Value *Handle = - createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType()); - Value *Index = CI->getArgOperand(1); - Value *Offset = CI->getArgOperand(2); - Value *BinOp = CI->getArgOperand(3); - Value *NewValue = CI->getArgOperand(4); - - std::array<Value *, 6> Args{Handle, BinOp, Index, - Offset, IRB.getInt32(0), NewValue}; - - Expected<CallInst *> OpCall = OpBuilder.tryCreateOp( - OpCode::AtomicBinOp, Args, CI->getName(), CI->getType()); - - if (Error E = OpCall.takeError()) - return E; - - CI->replaceAllUsesWith(*OpCall); - CI->eraseFromParent(); - return Error::success(); - }); - } - [[nodiscard]] bool lowerGetDimensionsX(Function &F) { IRBuilder<> &IRB = OpBuilder.getIRB(); Type *Int32Ty = IRB.getInt32Ty(); @@ -1237,9 +1223,6 @@ class OpLowerer { case Intrinsic::dx_resource_updatecounter: HasErrors |= lowerUpdateCounter(F); break; - case Intrinsic::dx_resource_atomicbinop: - HasErrors |= lowerResourceAtomicBinOp(F); - break; case Intrinsic::dx_resource_getdimensions_x: HasErrors |= lowerGetDimensionsX(F); break; diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp index 5f5c4660e744b..f9d7e8f85b198 100644 --- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp +++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "DXILResourceAccess.h" +#include "DXILOpBuilder.h" #include "DirectX.h" #include "llvm/ADT/SetVector.h" #include "llvm/Analysis/DXILResource.h" @@ -272,13 +273,21 @@ static std::optional<unsigned> getAtomicBinOpCode(AtomicRMWInst::BinOp BinOp) { } static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, - dxil::ResourceTypeInfo &RTI) { + dxil::ResourceTypeInfo &RTI, + std::optional<dxil::DXILOpBuilder> &OpBuilder) { std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation()); if (!BinOpCode) { + // TODO(#nnn): DXIL only defines atomic ops for Add/And/Or/Xor/ + // Min/Max/UMin/UMax/Xchg; the remaining atomicrmw ops (FSub, Nand, + // FAdd, FMin/Max variants, UIncWrap, etc.) have no direct DXIL + // equivalent and need explicit expansion (e.g. compare-and-swap loop). reportFatalUsageError("DXIL resource atomicrmw operation not implemented"); return; } + if (!OpBuilder) + OpBuilder.emplace(*AI->getModule()); + const DataLayout &DL = AI->getDataLayout(); IRBuilder<> Builder(AI); Value *Index = II->getOperand(1); @@ -298,19 +307,37 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, } auto *BinOp = Builder.getInt32(*BinOpCode); - Value *V = Builder.CreateIntrinsic( - AI->getType(), Intrinsic::dx_resource_atomicbinop, - {II->getOperand(0), Index, Offset, BinOp, AI->getValOperand()}); - AI->replaceAllUsesWith(V); + + // Cast the target-extension typed handle to `%dx.types.Handle` so we can + // emit the DXIL op directly. DXILOpLowering::cleanupHandleCasts will + // reconcile this cast once the handle-defining intrinsic has been lowered. + Value *Handle = Builder.CreateIntrinsic(OpBuilder->getHandleType(), + Intrinsic::dx_resource_casthandle, + {II->getOperand(0)}); + + std::array<Value *, 6> Args{ + Handle, BinOp, Index, Offset, Builder.getInt32(0), AI->getValOperand()}; + + OpBuilder->getIRB().SetInsertPoint(AI); + Expected<CallInst *> OpCall = OpBuilder->tryCreateOp( + dxil::OpCode::AtomicBinOp, Args, AI->getName(), AI->getType()); + if (Error E = OpCall.takeError()) { + AI->getContext().emitError(AI, toString(std::move(E))); + return; + } + + AI->replaceAllUsesWith(*OpCall); } -static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, - dxil::ResourceTypeInfo &RTI) { +static void +createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, + dxil::ResourceTypeInfo &RTI, + std::optional<dxil::DXILOpBuilder> &OpBuilder) { switch (RTI.getResourceKind()) { case dxil::ResourceKind::TypedBuffer: case dxil::ResourceKind::RawBuffer: case dxil::ResourceKind::StructuredBuffer: - return createAtomicBinOp(II, AI, RTI); + return createAtomicBinOp(II, AI, RTI, OpBuilder); case dxil::ResourceKind::Texture1D: case dxil::ResourceKind::Texture2D: case dxil::ResourceKind::Texture2DMS: @@ -322,12 +349,15 @@ static void createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, case dxil::ResourceKind::TextureCubeArray: case dxil::ResourceKind::FeedbackTexture2D: case dxil::ResourceKind::FeedbackTexture2DArray: + // TODO(#nnn): lower atomicrmw on texture UAVs to dx.op.textureAtomic. reportFatalUsageError( "DXIL atomicrmw not implemented for texture resources"); return; case dxil::ResourceKind::CBuffer: case dxil::ResourceKind::Sampler: case dxil::ResourceKind::TBuffer: + // TODO(#nnn): decide whether these resource kinds should be diagnosed + // in the frontend instead of reaching backend lowering. reportFatalUsageError( "DXIL atomicrmw not implemented for this resource type"); return; @@ -874,7 +904,8 @@ static bool legalizeResourceHandles(Function &F, DXILResourceTypeMap &DRTM) { return MadeChanges; } -static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) { +static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI, + std::optional<dxil::DXILOpBuilder> &OpBuilder) { SmallVector<User *> Worklist; for (User *U : II->users()) Worklist.push_back(U); @@ -898,35 +929,8 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) { createLoadIntrinsic(II, LI, RTI); DeadInsts.push_back(LI); } else if (auto *AI = dyn_cast<AtomicRMWInst>(U)) { - createAtomicBinOpIntrinsic(II, AI, RTI); - DeadInsts.push_back(AI); - } else if (auto *CI = dyn_cast<CallInst>(U)) { - // `dx.interlocked.*` intrinsics wrap an atomicrmw and are expanded to - // one by DXILIntrinsicExpansion — but that pass runs after this one, so - // when the source of the pointer is a resource we must expand them here - // (and immediately process the resulting atomicrmw) instead of letting - // the pointer escape. - auto *IntrinCall = dyn_cast<IntrinsicInst>(CI); - std::optional<AtomicRMWInst::BinOp> Op; - if (IntrinCall) { - switch (IntrinCall->getIntrinsicID()) { - case Intrinsic::dx_interlocked_add: - Op = AtomicRMWInst::Add; - break; - default: - break; - } - } - if (!Op) - llvm_unreachable("Unhandled instruction - pointer escaped?"); - IRBuilder<> Builder(IntrinCall); - auto *AI = Builder.CreateAtomicRMW( - *Op, IntrinCall->getArgOperand(0), IntrinCall->getArgOperand(1), - MaybeAlign(), AtomicOrdering::Monotonic); - IntrinCall->replaceAllUsesWith(AI); - createAtomicBinOpIntrinsic(II, AI, RTI); + createAtomicBinOpIntrinsic(II, AI, RTI, OpBuilder); DeadInsts.push_back(AI); - DeadInsts.push_back(IntrinCall); } else llvm_unreachable("Unhandled instruction - pointer escaped?"); } @@ -938,6 +942,9 @@ static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) { } static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) { + // Constructed lazily on the first atomicrmw so that non-atomic resource + // access still works on triples without a DXIL version. + std::optional<dxil::DXILOpBuilder> OpBuilder; SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources; for (BasicBlock &BB : make_early_inc_range(F)) for (Instruction &I : BB) @@ -953,7 +960,7 @@ static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) { } for (auto &[II, RI] : Resources) - replaceAccess(II, RI); + replaceAccess(II, RI, OpBuilder); return !Resources.empty(); } diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 11128eadef95f..bd50f03489255 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -5440,8 +5440,6 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, case Intrinsic::spv_wave_reduce_and: return selectWaveReduceOp(ResVReg, ResType, I, SPIRV::OpGroupNonUniformBitwiseAnd); - case Intrinsic::spv_interlocked_add: - return selectInterlockedOp(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); case Intrinsic::spv_interlocked_or: return selectInterlockedOp(ResVReg, ResType, I, SPIRV::OpAtomicOr); case Intrinsic::spv_wave_reduce_umax: diff --git a/llvm/test/CodeGen/DirectX/InterlockedAdd.ll b/llvm/test/CodeGen/DirectX/InterlockedAdd.ll deleted file mode 100644 index b12490b59cb79..0000000000000 --- a/llvm/test/CodeGen/DirectX/InterlockedAdd.ll +++ /dev/null @@ -1,52 +0,0 @@ -; RUN: opt -S -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.6-compute %s | FileCheck %s - -; Verify llvm.dx.interlocked.add expands to atomicrmw add monotonic. - -; Groupshared (addrspace 3) memory tests. -@gs_i32 = internal addrspace(3) global i32 zeroinitializer -@gs_i64 = internal addrspace(3) global i64 zeroinitializer - -define i32 @test_i32(i32 %v) { -entry: -; CHECK-LABEL: @test_i32 -; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) @gs_i32, i32 %v monotonic -; CHECK: ret i32 %[[R]] - %r = call i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3) @gs_i32, i32 %v) - ret i32 %r -} - -define i64 @test_i64(i64 %v) { -entry: -; CHECK-LABEL: @test_i64 -; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(3) @gs_i64, i64 %v monotonic -; CHECK: ret i64 %[[R]] - %r = call i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3) @gs_i64, i64 %v) - ret i64 %r -} - -; Device (addrspace 1) memory tests. -@dev_i32 = external addrspace(1) global i32 -@dev_i64 = external addrspace(1) global i64 - -define i32 @test_device_i32(i32 %v) { -entry: -; CHECK-LABEL: @test_device_i32 -; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(1) @dev_i32, i32 %v monotonic -; CHECK: ret i32 %[[R]] - %r = call i32 @llvm.dx.interlocked.add.i32.p1(ptr addrspace(1) @dev_i32, i32 %v) - ret i32 %r -} - -define i64 @test_device_i64(i64 %v) { -entry: -; CHECK-LABEL: @test_device_i64 -; CHECK: %[[R:.*]] = atomicrmw add ptr addrspace(1) @dev_i64, i64 %v monotonic -; CHECK: ret i64 %[[R]] - %r = call i64 @llvm.dx.interlocked.add.i64.p1(ptr addrspace(1) @dev_i64, i64 %v) - ret i64 %r -} - -declare i32 @llvm.dx.interlocked.add.i32.p3(ptr addrspace(3), i32) -declare i64 @llvm.dx.interlocked.add.i64.p3(ptr addrspace(3), i64) -declare i32 @llvm.dx.interlocked.add.i32.p1(ptr addrspace(1), i32) -declare i64 @llvm.dx.interlocked.add.i64.p1(ptr addrspace(1), i64) diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll deleted file mode 100644 index 44ff6cff6ee5f..0000000000000 --- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd.ll +++ /dev/null @@ -1,36 +0,0 @@ -; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - | FileCheck %s -; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %} - -; Test lowering of llvm.spv.interlocked.add to OpAtomicIAdd. - -; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0 -; CHECK-DAG: %[[#scope_wg:]] = OpConstant %[[#uint]] 2 -; CHECK-DAG: %[[#scope_dev:]] = OpConstant %[[#uint]] 1 -; CHECK-DAG: %[[#mem_wg:]] = OpConstant %[[#uint]] 256 -; CHECK-DAG: %[[#mem_uniform:]] = OpConstant %[[#uint]] 64 - -@gs_i32 = internal addrspace(3) global i32 zeroinitializer -@dev_i32 = external addrspace(11) global i32 - -; Workgroup (addrspace 3) memory tests. - -; CHECK-LABEL: Begin function test_i32 -define i32 @test_i32(i32 %v) { -entry: -; CHECK: %[[#R:]] = OpAtomicIAdd %[[#uint]] %[[#]] %[[#scope_wg]] %[[#mem_wg]] %[[#]] - %r = call i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3) @gs_i32, i32 %v) - ret i32 %r -} - -; Device / StorageBuffer (addrspace 11) memory tests. - -; CHECK-LABEL: Begin function test_device_i32 -define i32 @test_device_i32(i32 %v) { -entry: -; CHECK: %[[#R:]] = OpAtomicIAdd %[[#uint]] %[[#]] %[[#scope_dev]] %[[#mem_uniform]] %[[#]] - %r = call i32 @llvm.spv.interlocked.add.i32.p11(ptr addrspace(11) @dev_i32, i32 %v) - ret i32 %r -} - -declare i32 @llvm.spv.interlocked.add.i32.p3(ptr addrspace(3), i32) -declare i32 @llvm.spv.interlocked.add.i32.p11(ptr addrspace(11), i32) diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll deleted file mode 100644 index 794c0a925862a..0000000000000 --- a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/InterlockedAdd_spv_i64.ll +++ /dev/null @@ -1,37 +0,0 @@ -; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - | FileCheck %s -; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %} - -; Test lowering of llvm.spv.interlocked.add with i64 to OpAtomicIAdd. - -; CHECK-DAG: %[[#ulong:]] = OpTypeInt 64 0 -; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0 -; CHECK-DAG: %[[#scope_wg:]] = OpConstant %[[#uint]] 2 -; CHECK-DAG: %[[#scope_dev:]] = OpConstant %[[#uint]] 1 -; CHECK-DAG: %[[#mem_wg:]] = OpConstant %[[#uint]] 256 -; CHECK-DAG: %[[#mem_uniform:]] = OpConstant %[[#uint]] 64 - -@gs_i64 = internal addrspace(3) global i64 zeroinitializer -@dev_i64 = external addrspace(11) global i64 - -; Workgroup (addrspace 3) memory test. - -; CHECK-LABEL: Begin function test_i64 -define i64 @test_i64(i64 %v) { -entry: -; CHECK: %[[#R:]] = OpAtomicIAdd %[[#ulong]] %[[#]] %[[#scope_wg]] %[[#mem_wg]] %[[#]] - %r = call i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3) @gs_i64, i64 %v) - ret i64 %r -} - -; Device / StorageBuffer (addrspace 11) memory test. - -; CHECK-LABEL: Begin function test_device_i64 -define i64 @test_device_i64(i64 %v) { -entry: -; CHECK: %[[#R:]] = OpAtomicIAdd %[[#ulong]] %[[#]] %[[#scope_dev]] %[[#mem_uniform]] %[[#]] - %r = call i64 @llvm.spv.interlocked.add.i64.p11(ptr addrspace(11) @dev_i64, i64 %v) - ret i64 %r -} - -declare i64 @llvm.spv.interlocked.add.i64.p3(ptr addrspace(3), i64) -declare i64 @llvm.spv.interlocked.add.i64.p11(ptr addrspace(11), i64) >From 4e2b382d310f132bb512aa1b976f6ad7df91554c Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Thu, 16 Jul 2026 11:34:00 -0700 Subject: [PATCH 4/7] fix failing tests --- .../ByteAddressBuffer-InterlockedAdd-errors.hlsl | 9 ++++++++- .../ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl index 234932366e406..f63e64b9f5837 100644 --- a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-errors.hlsl @@ -1,7 +1,14 @@ // RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ -// RUN: -triple dxil-pc-shadermodel6.6-compute %s -fsyntax-only -verify \ +// RUN: -triple dxil-pc-shadermodel6.6-library %s -fsyntax-only -verify \ // RUN: -verify-ignore-unexpected=note,warning +// notes that are ignored are strictly ones of the form: +// (frontend): candidate function not viable: requires X arguments, but Y was provided +// or +// (frontend): candidate function not viable: no known conversion from X to Y for Nth argument +// which is in line with expectations, but is difficult to exactly match since +// the notes are not tied to explicit source lines, but just (frontend). + RWByteAddressBuffer BAB : register(u0); RasterizerOrderedByteAddressBuffer ROVB : register(u1); diff --git a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl index 5e51cbf36b32b..e6210ba36e357 100644 --- a/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ByteAddressBuffer-InterlockedAdd-sm65-errors.hlsl @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -std=hlsl202x -finclude-default-header \ -// RUN: -triple dxil-pc-shadermodel6.5-compute %s -fsyntax-only -verify \ -// RUN: -verify-ignore-unexpected=note,warning +// RUN: -triple dxil-pc-shadermodel6.5-library %s -fsyntax-only -verify \ +// RUN: -verify-ignore-unexpected=warning RWByteAddressBuffer BAB : register(u0); RasterizerOrderedByteAddressBuffer ROVB : register(u1); >From 02619f419b9d7d95cd16901f4d76a15696b9f69e Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Thu, 16 Jul 2026 12:50:48 -0700 Subject: [PATCH 5/7] update target profile --- .../CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl index a7306249e21c7..cf939a7507dbf 100644 --- a/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl +++ b/clang/test/CodeGenHLSL/builtins/ByteAddressBuffer-InterlockedAdd.hlsl @@ -1,5 +1,5 @@ // RUN: %clang_cc1 -std=hlsl202x -finclude-default-header -triple \ -// RUN: dxil-pc-shadermodel6.6-compute %s -emit-llvm -disable-llvm-passes -o - | \ +// RUN: dxil-pc-shadermodel6.6-library %s -emit-llvm -disable-llvm-passes -o - | \ // RUN: FileCheck %s --check-prefixes=CHECK,DXCHECK // Test that the RWByteAddressBuffer::InterlockedAdd and >From 1e13e596e976c4fe1cafb5979e09aeba6934c71a Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Thu, 16 Jul 2026 14:31:58 -0700 Subject: [PATCH 6/7] try different error delivery to fix not opt --- llvm/lib/Target/DirectX/DXILResourceAccess.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp index f9d7e8f85b198..056c0f8cd5a5a 100644 --- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp +++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp @@ -322,7 +322,12 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, Expected<CallInst *> OpCall = OpBuilder->tryCreateOp( dxil::OpCode::AtomicBinOp, Args, AI->getName(), AI->getType()); if (Error E = OpCall.takeError()) { - AI->getContext().emitError(AI, toString(std::move(E))); + std::string Message(toString(std::move(E))); + AI->getContext().diagnose(DiagnosticInfoUnsupported( + *AI->getFunction(), Message, AI->getDebugLoc())); + // RAUW with poison so the caller's subsequent eraseFromParent() doesn't + // leave dangling uses of the AtomicRMWInst. + AI->replaceAllUsesWith(PoisonValue::get(AI->getType())); return; } >From c4f6704f3c7c2841fd670537a2d04a8925013518 Mon Sep 17 00:00:00 2001 From: Joshua Batista <[email protected]> Date: Fri, 17 Jul 2026 14:51:53 -0700 Subject: [PATCH 7/7] self review --- clang/lib/CodeGen/CGHLSLBuiltins.cpp | 13 +++++++------ llvm/lib/Target/DirectX/DXILResourceAccess.cpp | 7 ------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp index ef8be15fdfb40..12a3f61c734d0 100644 --- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp +++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp @@ -310,17 +310,18 @@ static Value *handleElementwiseF32ToF16(CodeGenFunction &CGF, llvm_unreachable("Intrinsic F32ToF16 not supported by target architecture"); } -static Value *handleInterlockedAdd(CodeGenFunction &CGF, const CallExpr *E) { - // Emit `atomicrmw add` directly — no intermediate `*.interlocked.add` - // intrinsic needed on either DXIL or SPIR-V. +static Value *handleInterlockedOp(CodeGenFunction &CGF, const CallExpr *E, + llvm::AtomicRMWInst::BinOp Op) { + // Emit `atomicrmw <op>` directly — no intermediate intrinsic needed on + // either DXIL or SPIR-V. LValue DestLV = CGF.EmitLValue(E->getArg(0)); Address DestAddr = DestLV.getAddress(); Value *Val = CGF.EmitScalarExpr(E->getArg(1)); assert(E->getArg(1)->getType()->isIntegerType() && - "Intrinsic InterlockedAdd value operand must be an integer"); + "Intrinsic InterlockedOp value operand must be an integer"); llvm::AtomicRMWInst *Call = CGF.Builder.CreateAtomicRMW( - llvm::AtomicRMWInst::Add, DestAddr, Val, llvm::AtomicOrdering::Monotonic); + Op, DestAddr, Val, llvm::AtomicOrdering::Monotonic); // The 3-arg overload writes the old value (the RMW's return value) into // the `original_value` reference parameter. @@ -1481,7 +1482,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, // Emit `atomicrmw` directly for both DXIL and SPIR-V — the backends pick // up the raw instruction (DXIL via DXILResourceAccess for resource // pointers, SPIR-V via selectAtomicRMW). No intermediate intrinsic. - return handleInterlockedAdd(*this, E); + return handleInterlockedOp(*this, E, llvm::AtomicRMWInst::Add); } case Builtin::BI__builtin_hlsl_interlocked_or: { return handleInterlockedOp(*this, E, diff --git a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp index 056c0f8cd5a5a..ae3e0b5f6c503 100644 --- a/llvm/lib/Target/DirectX/DXILResourceAccess.cpp +++ b/llvm/lib/Target/DirectX/DXILResourceAccess.cpp @@ -277,10 +277,6 @@ static void createAtomicBinOp(IntrinsicInst *II, AtomicRMWInst *AI, std::optional<dxil::DXILOpBuilder> &OpBuilder) { std::optional<unsigned> BinOpCode = getAtomicBinOpCode(AI->getOperation()); if (!BinOpCode) { - // TODO(#nnn): DXIL only defines atomic ops for Add/And/Or/Xor/ - // Min/Max/UMin/UMax/Xchg; the remaining atomicrmw ops (FSub, Nand, - // FAdd, FMin/Max variants, UIncWrap, etc.) have no direct DXIL - // equivalent and need explicit expansion (e.g. compare-and-swap loop). reportFatalUsageError("DXIL resource atomicrmw operation not implemented"); return; } @@ -354,15 +350,12 @@ createAtomicBinOpIntrinsic(IntrinsicInst *II, AtomicRMWInst *AI, case dxil::ResourceKind::TextureCubeArray: case dxil::ResourceKind::FeedbackTexture2D: case dxil::ResourceKind::FeedbackTexture2DArray: - // TODO(#nnn): lower atomicrmw on texture UAVs to dx.op.textureAtomic. reportFatalUsageError( "DXIL atomicrmw not implemented for texture resources"); return; case dxil::ResourceKind::CBuffer: case dxil::ResourceKind::Sampler: case dxil::ResourceKind::TBuffer: - // TODO(#nnn): decide whether these resource kinds should be diagnosed - // in the frontend instead of reaching backend lowering. reportFatalUsageError( "DXIL atomicrmw not implemented for this resource type"); return; _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
