https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/101699
>From 66a2fe24e7ee4754d720cd7b12060bf44f981e38 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <matthew.arsena...@amd.com>
Date: Wed, 26 Jun 2024 14:13:31 +0200
Subject: [PATCH] AMDGPU: Stop handling legacy amdgpu-unsafe-fp-atomics
attribute
This is now autoupgraded to annotate atomicrmw instructions in
old bitcode.
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 9 +-
.../AtomicExpand/AMDGPU/expand-atomic-mmra.ll | 7 +-
.../AMDGPU/expand-atomic-rmw-fadd.ll | 169 +++++++++---------
.../AMDGPU/inline-amdgpu-unsafe-fp-atomics.ll | 99 ----------
4 files changed, 88 insertions(+), 196 deletions(-)
delete mode 100644
llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-unsafe-fp-atomics.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 1cf9fb7a3724b7..5bcf158f4b62a4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16165,14 +16165,7 @@ static bool globalMemoryFPAtomicIsLegal(const
GCNSubtarget &Subtarget,
} else if (Subtarget.supportsAgentScopeFineGrainedRemoteMemoryAtomics())
return true;
- if (RMW->hasMetadata("amdgpu.no.fine.grained.memory"))
- return true;
-
- // TODO: Auto-upgrade this attribute to the metadata in function body and
stop
- // checking it.
- return RMW->getFunction()
- ->getFnAttribute("amdgpu-unsafe-fp-atomics")
- .getValueAsBool();
+ return RMW->hasMetadata("amdgpu.no.fine.grained.memory");
}
/// \return Action to perform on AtomicRMWInsts for integer operations.
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-mmra.ll
b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-mmra.ll
index 78969839efcb8a..f6a07d9d923c35 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-mmra.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-mmra.ll
@@ -124,7 +124,7 @@ define i16 @test_cmpxchg_i16_global_agent_align4(ptr
addrspace(1) %out, i16 %in,
ret i16 %extract
}
-define void @syncscope_workgroup_nortn(ptr %addr, float %val) #0 {
+define void @syncscope_workgroup_nortn(ptr %addr, float %val) {
; GFX90A-LABEL: define void @syncscope_workgroup_nortn(
; GFX90A-SAME: ptr [[ADDR:%.*]], float [[VAL:%.*]]) #[[ATTR1:[0-9]+]] {
; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr
[[ADDR]])
@@ -157,7 +157,7 @@ define void @syncscope_workgroup_nortn(ptr %addr, float
%val) #0 {
; GFX1100-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[VAL]]
syncscope("workgroup") seq_cst, align 4, !mmra [[META0]]
; GFX1100-NEXT: ret void
;
- %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst,
!mmra !2
+ %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst,
!mmra !2, !amdgpu.no.fine.grained.memory !3, !amdgpu.ignore.denormal.mode !3
ret void
}
@@ -186,11 +186,10 @@ define i32 @atomic_load_global_align1(ptr addrspace(1)
%ptr) {
ret i32 %val
}
-attributes #0 = { "amdgpu-unsafe-fp-atomics"="true" }
-
!0 = !{!"foo", !"bar"}
!1 = !{!"bux", !"baz"}
!2 = !{!0, !1}
+!3 = !{}
;.
; GFX90A: [[META0]] = !{[[META1:![0-9]+]], [[META2:![0-9]+]]}
; GFX90A: [[META1]] = !{!"foo", !"bar"}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
index 0a091bd0fc9ada..888e3cd29b9a32 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
@@ -273,7 +273,7 @@ define void
@test_atomicrmw_fadd_f32_as999_no_use_unsafe(ptr addrspace(999) %ptr
ret void
}
-define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr,
float %value) #0 {
+define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr,
float %value) #3 {
; CI-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(
; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -323,22 +323,22 @@ define float @test_atomicrmw_fadd_f32_global_unsafe(ptr
addrspace(1) %ptr, float
; GFX908-NEXT: ret float [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
; GFX90A-NEXT: ret float [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
; GFX940-NEXT: ret float [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(
-; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
; GFX11-NEXT: ret float [[RES]]
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value
syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value
syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
ret float %res
}
-define float @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(ptr addrspace(7)
%ptr, float %value) #0 {
+define float @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(ptr addrspace(7)
%ptr, float %value) #3 {
; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(
; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -388,22 +388,22 @@ define float
@test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(ptr addrspace(7) %pt
; GFX908-NEXT: ret float [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret float [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret float [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(
-; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX11-NEXT: ret float [[RES]]
;
- %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value
syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value
syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
ret float %res
}
-define float @test_atomicrmw_fadd_f32_as999_unsafe(ptr addrspace(999) %ptr,
float %value) #0 {
+define float @test_atomicrmw_fadd_f32_as999_unsafe(ptr addrspace(999) %ptr,
float %value) #3 {
; CI-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(
; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align
4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -453,22 +453,22 @@ define float @test_atomicrmw_fadd_f32_as999_unsafe(ptr
addrspace(999) %ptr, floa
; GFX908-NEXT: ret float [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret float [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret float [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(
-; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX11-NEXT: ret float [[RES]]
;
- %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value
syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value
syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
ret float %res
}
-define double @test_atomicrmw_fadd_f64_global_unsafe(ptr addrspace(1) %ptr,
double %value) #0 {
+define double @test_atomicrmw_fadd_f64_global_unsafe(ptr addrspace(1) %ptr,
double %value) #3 {
; CI-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(
; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -518,11 +518,11 @@ define double @test_atomicrmw_fadd_f64_global_unsafe(ptr
addrspace(1) %ptr, doub
; GFX908-NEXT: ret double [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret double [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret double [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(
@@ -541,11 +541,11 @@ define double @test_atomicrmw_fadd_f64_global_unsafe(ptr
addrspace(1) %ptr, doub
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret double [[TMP5]]
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value
syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value
syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
ret double %res
}
-define float @test_atomicrmw_fadd_f32_flat_unsafe(ptr %ptr, float %value) #0 {
+define float @test_atomicrmw_fadd_f32_flat_unsafe(ptr %ptr, float %value) #3 {
; CI-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(
; CI-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -599,7 +599,7 @@ define float @test_atomicrmw_fadd_f32_flat_unsafe(ptr %ptr,
float %value) #0 {
; GFX90A-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label
[[ATOMICRMW_CHECK_PRIVATE:%.*]]
; GFX90A: atomicrmw.shared:
; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)
-; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: br label [[ATOMICRMW_PHI:%.*]]
; GFX90A: atomicrmw.check.private:
; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr
[[PTR]])
@@ -612,7 +612,7 @@ define float @test_atomicrmw_fadd_f32_flat_unsafe(ptr %ptr,
float %value) #0 {
; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]
; GFX90A: atomicrmw.global:
; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
-; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]],
float [[VALUE]] syncscope("wavefront") monotonic, align 4
+; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]],
float [[VALUE]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]
; GFX90A: atomicrmw.phi:
; GFX90A-NEXT: [[RES:%.*]] = phi float [ [[TMP2]], [[ATOMICRMW_SHARED]] ],
[ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], [[ATOMICRMW_GLOBAL]]
]
@@ -621,18 +621,18 @@ define float @test_atomicrmw_fadd_f32_flat_unsafe(ptr
%ptr, float %value) #0 {
; GFX90A-NEXT: ret float [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], float
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], float
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret float [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(
-; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], float
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], float
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX11-NEXT: ret float [[RES]]
;
- %res = atomicrmw fadd ptr %ptr, float %value syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr %ptr, float %value syncscope("wavefront")
monotonic, !amdgpu.no.fine.grained.memory !0
ret float %res
}
-define double @test_atomicrmw_fadd_f64_flat_unsafe(ptr %ptr, double %value) #0
{
+define double @test_atomicrmw_fadd_f64_flat_unsafe(ptr %ptr, double %value) #3
{
; CI-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(
; CI-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -682,11 +682,11 @@ define double @test_atomicrmw_fadd_f64_flat_unsafe(ptr
%ptr, double %value) #0 {
; GFX908-NEXT: ret double [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], double
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 8
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], double
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 8,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret double [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], double
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 8
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], double
[[VALUE:%.*]] syncscope("wavefront") monotonic, align 8,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret double [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(
@@ -705,7 +705,7 @@ define double @test_atomicrmw_fadd_f64_flat_unsafe(ptr
%ptr, double %value) #0 {
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret double [[TMP5]]
;
- %res = atomicrmw fadd ptr %ptr, double %value syncscope("wavefront")
monotonic
+ %res = atomicrmw fadd ptr %ptr, double %value syncscope("wavefront")
monotonic, !amdgpu.no.fine.grained.memory !0
ret double %res
}
@@ -814,7 +814,7 @@ define void @test_atomicrmw_fadd_f32_global_no_use_ieee(ptr
addrspace(1) %ptr, f
ret void
}
-define void @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(ptr
addrspace(1) %ptr, float %value) #0 {
+define void @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(ptr
addrspace(1) %ptr, float %value) #4 {
; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(
; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -856,14 +856,14 @@ define void
@test_atomicrmw_fadd_f32_global_no_use_denorm_flush(ptr addrspace(1)
; GFX90A-NEXT: ret void
;
; GFX940-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret void
;
; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(
-; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] seq_cst, align 4
+; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
; GFX11-NEXT: ret void
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value seq_cst
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value seq_cst,
!amdgpu.no.fine.grained.memory !0
ret void
}
@@ -1270,13 +1270,13 @@ define float @test_atomicrmw_fadd_f32_global_one_as(ptr
addrspace(1) %ptr, float
ret float %res
}
-define void @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(ptr
addrspace(1) %ptr, float %value) #1 {
-; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(
+define void @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(ptr
addrspace(1) %ptr, float %value) #1 {
+; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(
; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; CI-NEXT: [[NEW:%.*]] = call float
@llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10:[0-9]+]]
+; CI-NEXT: [[NEW:%.*]] = call float
@llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8:[0-9]+]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]],
i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4
@@ -1287,12 +1287,12 @@ define void
@test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(ptr addrspace
; CI: atomicrmw.end:
; CI-NEXT: ret void
;
-; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(
; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align
4
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; GFX9-NEXT: [[NEW:%.*]] = call float
@llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10:[0-9]+]]
+; GFX9-NEXT: [[NEW:%.*]] = call float
@llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8:[0-9]+]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]],
i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4
@@ -1303,23 +1303,23 @@ define void
@test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(ptr addrspace
; GFX9: atomicrmw.end:
; GFX9-NEXT: ret void
;
-; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(
-; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(
+; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
; GFX908-NEXT: ret void
;
-; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret void
;
-; GFX940-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX940-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret void
;
-; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_structfp(
-; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4
+; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(
+; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX11-NEXT: ret void
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value
syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value
syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
ret void
}
@@ -1329,7 +1329,7 @@ define double
@test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; CI-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR10]]
+; CI-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR8]]
; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]],
i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8
@@ -1345,7 +1345,7 @@ define double
@test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %
; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX9: atomicrmw.start:
; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; GFX9-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR10]]
+; GFX9-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR8]]
; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]],
i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8
@@ -1361,7 +1361,7 @@ define double
@test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %
; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX908: atomicrmw.start:
; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; GFX908-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR10:[0-9]+]]
+; GFX908-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR8:[0-9]+]]
; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64
[[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8
@@ -1373,11 +1373,11 @@ define double
@test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %
; GFX908-NEXT: ret double [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret double [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret double [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(
@@ -1385,7 +1385,7 @@ define double
@test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %
; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]
; GFX11: atomicrmw.start:
; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; GFX11-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR10:[0-9]+]]
+; GFX11-NEXT: [[NEW:%.*]] = call double
@llvm.experimental.constrained.fadd.f64(double [[LOADED]], double
[[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict")
#[[ATTR8:[0-9]+]]
; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64
[[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8
@@ -1396,7 +1396,7 @@ define double
@test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret double [[TMP5]]
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value
syncscope("wavefront") monotonic
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value
syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0
ret double %res
}
@@ -1406,7 +1406,7 @@ define float @test_atomicrmw_fadd_f32_local_strictfp(ptr
addrspace(3) %ptr, floa
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
; CI: atomicrmw.start:
; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [
[[TMP5:%.*]], [[ATOMICRMW_START]] ]
-; CI-NEXT: [[NEW:%.*]] = call float
@llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CI-NEXT: [[NEW:%.*]] = call float
@llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]],
i32 [[TMP2]] seq_cst seq_cst, align 4
@@ -1628,7 +1628,7 @@ define bfloat
@test_atomicrmw_fadd_bf16_global_system_align4(ptr addrspace(1) %p
define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr,
bfloat %value) #2 {
; ALL-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp(
-; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3)
@llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR10:[0-9]+]]
+; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3)
@llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR8:[0-9]+]]
; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32
; ALL-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
; ALL-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
@@ -1641,7 +1641,7 @@ define bfloat
@test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bf
; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]
; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat
-; ALL-NEXT: [[NEW:%.*]] = call bfloat
@llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; ALL-NEXT: [[NEW:%.*]] = call bfloat
@llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]],
metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16
; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32
; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]
@@ -2099,11 +2099,11 @@ define void
@test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(pt
; GFX9-NEXT: ret void
;
; GFX908-LABEL:
@test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(
-; GFX908-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3)
[[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4,
!amdgpu.ignore.denormal.mode [[META0:![0-9]+]]
+; GFX908-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3)
[[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4,
!amdgpu.ignore.denormal.mode [[META0]]
; GFX908-NEXT: ret void
;
; GFX90A-LABEL:
@test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(
-; GFX90A-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3)
[[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4,
!amdgpu.ignore.denormal.mode [[META0:![0-9]+]]
+; GFX90A-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3)
[[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4,
!amdgpu.ignore.denormal.mode [[META0]]
; GFX90A-NEXT: ret void
;
; GFX940-LABEL:
@test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(
@@ -2111,7 +2111,7 @@ define void
@test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(pt
; GFX940-NEXT: ret void
;
; GFX11-LABEL:
@test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(
-; GFX11-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]],
float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode
[[META0:![0-9]+]]
+; GFX11-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]],
float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]
; GFX11-NEXT: ret void
;
%unused = atomicrmw fadd ptr addrspace(3) %ptr, float %value monotonic,
!amdgpu.ignore.denormal.mode !0
@@ -3787,7 +3787,7 @@ define void
@test_atomicrmw_fadd_v2bf16_flat_local_noret(ptr addrspace(3) %ptr,
ret void
}
-define <2 x half> @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(ptr %ptr, <2 x
half> %value) #6 {
+define <2 x half> @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(ptr %ptr, <2 x
half> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -3876,7 +3876,7 @@ define <2 x half>
@test_atomicrmw_fadd_v2f16_flat_agent__unsafe(ptr %ptr, <2 x h
ret <2 x half> %res
}
-define void @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(ptr %ptr, <2 x
half> %value) #6 {
+define void @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(ptr %ptr, <2 x
half> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -3965,7 +3965,7 @@ define void
@test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(ptr %ptr, <2 x h
ret void
}
-define <2 x half> @test_atomicrmw_fadd_v2f16_global_agent__unsafe(ptr
addrspace(1) %ptr, <2 x half> %value) #6 {
+define <2 x half> @test_atomicrmw_fadd_v2f16_global_agent__unsafe(ptr
addrspace(1) %ptr, <2 x half> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]],
align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -4015,11 +4015,11 @@ define <2 x half>
@test_atomicrmw_fadd_v2f16_global_agent__unsafe(ptr addrspace(
; GFX908-NEXT: ret <2 x half> [[TMP5]]
;
; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret <2 x half> [[RES]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret <2 x half> [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(
@@ -4038,11 +4038,11 @@ define <2 x half>
@test_atomicrmw_fadd_v2f16_global_agent__unsafe(ptr addrspace(
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret <2 x half> [[TMP5]]
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value
syncscope("agent") seq_cst
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value
syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x half> %res
}
-define void @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(ptr
addrspace(1) %ptr, <2 x half> %value) #6 {
+define void @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(ptr
addrspace(1) %ptr, <2 x half> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]],
align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -4080,11 +4080,11 @@ define void
@test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(ptr addrspace(
; GFX908-NEXT: ret void
;
; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(
-; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX90A-NEXT: ret void
;
; GFX940-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret void
;
; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(
@@ -4103,11 +4103,11 @@ define void
@test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(ptr addrspace(
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret void
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value
syncscope("agent") seq_cst
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value
syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(ptr %ptr,
<2 x bfloat> %value) #6 {
+define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(ptr %ptr,
<2 x bfloat> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -4173,7 +4173,7 @@ define <2 x bfloat>
@test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(ptr %ptr, <2
; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat>
[[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat>
[[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret <2 x bfloat> [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(
@@ -4192,11 +4192,11 @@ define <2 x bfloat>
@test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(ptr %ptr, <2
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]
;
- %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent")
seq_cst
+ %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent")
seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %res
}
-define void @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(ptr %ptr, <2
x bfloat> %value) #6 {
+define void @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(ptr %ptr, <2
x bfloat> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -4262,7 +4262,7 @@ define void
@test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(ptr %ptr, <2 x
; GFX90A-NEXT: ret void
;
; GFX940-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat>
[[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat>
[[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret void
;
; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(
@@ -4281,11 +4281,11 @@ define void
@test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(ptr %ptr, <2 x
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret void
;
- %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent")
seq_cst
+ %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent")
seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(ptr
addrspace(1) %ptr, <2 x bfloat> %value) #6 {
+define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(ptr
addrspace(1) %ptr, <2 x bfloat> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]],
align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -4351,7 +4351,7 @@ define <2 x bfloat>
@test_atomicrmw_fadd_v2bf16_global_agent__unsafe(ptr addrspa
; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]
;
; GFX940-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret <2 x bfloat> [[RES]]
;
; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(
@@ -4370,11 +4370,11 @@ define <2 x bfloat>
@test_atomicrmw_fadd_v2bf16_global_agent__unsafe(ptr addrspa
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value
syncscope("agent") seq_cst
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value
syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret <2 x bfloat> %res
}
-define void @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(ptr
addrspace(1) %ptr, <2 x bfloat> %value) #6 {
+define void @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(ptr
addrspace(1) %ptr, <2 x bfloat> %value) {
; CI-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(
; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]],
align 4
; CI-NEXT: br label [[ATOMICRMW_START:%.*]]
@@ -4440,7 +4440,7 @@ define void
@test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(ptr addrspace
; GFX90A-NEXT: ret void
;
; GFX940-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(
-; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]],
<2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4,
!amdgpu.no.fine.grained.memory [[META0]]
; GFX940-NEXT: ret void
;
; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(
@@ -4459,16 +4459,15 @@ define void
@test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(ptr addrspace
; GFX11: atomicrmw.end:
; GFX11-NEXT: ret void
;
- %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value
syncscope("agent") seq_cst
+ %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value
syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
-attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign"
"amdgpu-unsafe-fp-atomics"="true" }
-attributes #1 = { strictfp
"denormal-fp-math-f32"="preserve-sign,preserve-sign"
"amdgpu-unsafe-fp-atomics"="true" }
+attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #1 = { strictfp
"denormal-fp-math-f32"="preserve-sign,preserve-sign" }
attributes #2 = { strictfp }
attributes #3 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
attributes #4 = { "denormal-fp-math-f32"="dynamic,dynamic" }
attributes #5 = { "denormal-fp-math"="dynamic,dynamic" }
-attributes #6 = { "amdgpu-unsafe-fp-atomics"="true" }
!0 = !{}
diff --git
a/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-unsafe-fp-atomics.ll
b/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-unsafe-fp-atomics.ll
deleted file mode 100644
index b867f942a84c45..00000000000000
--- a/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-unsafe-fp-atomics.ll
+++ /dev/null
@@ -1,99 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
UTC_ARGS: --function-signature --check-globals
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=inline < %s | FileCheck %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='cgscc(inline)' < %s |
FileCheck %s
-
-define i32 @func_default() #0 {
-; CHECK-LABEL: define {{[^@]+}}@func_default
-; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: ret i32 0
-;
- ret i32 0
-}
-
-define i32 @func_unsafe_fp_atomic_enabled() #1 {
-; CHECK-LABEL: define {{[^@]+}}@func_unsafe_fp_atomic_enabled
-; CHECK-SAME: () #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: ret i32 0
-;
- ret i32 0
-}
-
-define i32 @func_unsafe_fp_atomic_disabled() #2 {
-; CHECK-LABEL: define {{[^@]+}}@func_unsafe_fp_atomic_disabled
-; CHECK-SAME: () #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: ret i32 0
-;
- ret i32 0
-}
-
-define i32 @default_call_default() #0 {
-; CHECK-LABEL: define {{[^@]+}}@default_call_default
-; CHECK-SAME: () #[[ATTR0]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_default()
- ret i32 %call
-}
-
-define i32 @unsafe_fp_atomic_enabled_call_default() #1 {
-; CHECK-LABEL: define {{[^@]+}}@unsafe_fp_atomic_enabled_call_default
-; CHECK-SAME: () #[[ATTR1]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_default()
- ret i32 %call
-}
-
-define i32 @unsafe_fp_atomic_enabled_call_unsafe_fp_atomic_enabled() #1 {
-; CHECK-LABEL: define
{{[^@]+}}@unsafe_fp_atomic_enabled_call_unsafe_fp_atomic_enabled
-; CHECK-SAME: () #[[ATTR1]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_unsafe_fp_atomic_enabled()
- ret i32 %call
-}
-
-define i32 @unsafe_fp_atomic_enabled_call_unsafe_fp_atomic_disabled() #1 {
-; CHECK-LABEL: define
{{[^@]+}}@unsafe_fp_atomic_enabled_call_unsafe_fp_atomic_disabled
-; CHECK-SAME: () #[[ATTR1]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_unsafe_fp_atomic_disabled()
- ret i32 %call
-}
-
-define i32 @unsafe_fp_atomic_disabled_call_default() #2 {
-; CHECK-LABEL: define {{[^@]+}}@unsafe_fp_atomic_disabled_call_default
-; CHECK-SAME: () #[[ATTR2]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_default()
- ret i32 %call
-}
-
-define i32 @unsafe_fp_atomic_disabled_call_unsafe_fp_atomic_enabled() #2 {
-; CHECK-LABEL: define
{{[^@]+}}@unsafe_fp_atomic_disabled_call_unsafe_fp_atomic_enabled
-; CHECK-SAME: () #[[ATTR2]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_unsafe_fp_atomic_enabled()
- ret i32 %call
-}
-
-define i32 @unsafe_fp_atomic_disabled_call_unsafe_fp_atomic_disabled() #2 {
-; CHECK-LABEL: define
{{[^@]+}}@unsafe_fp_atomic_disabled_call_unsafe_fp_atomic_disabled
-; CHECK-SAME: () #[[ATTR2]] {
-; CHECK-NEXT: ret i32 0
-;
- %call = call i32 @func_unsafe_fp_atomic_disabled()
- ret i32 %call
-}
-
-attributes #0 = { nounwind }
-attributes #1 = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
-attributes #2 = { nounwind "amdgpu-unsafe-fp-atomics"="false" }
-;.
-; CHECK: attributes #[[ATTR0]] = { nounwind }
-; CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-unsafe-fp-atomics"="true" }
-; CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-unsafe-fp-atomics"="false"
}
-;.
_______________________________________________
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
