================
@@ -0,0 +1,898 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
UTC_ARGS: --version 6
+; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s
-check-prefixes=GFX900
+; RUN: llc -march=amdgcn -mcpu=gfx942 < %s | FileCheck %s
-check-prefixes=GFX942
+; RUN: llc -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s
-check-prefixes=GFX1010
+
+; Test async mark/wait with global_load_lds and global loads
+; This version uses wave barriers to enforce program order so that unrelated
vmem
+; instructions do not get reordered before reaching this point.
+
+define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr
addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) {
+; GFX900-LABEL: interleaved_with_wave_barrier:
+; GFX900: ; %bb.0: ; %entry
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: s_movk_i32 s4, 0x54
+; GFX900-NEXT: v_add_u32_e32 v11, 0x54, v2
+; GFX900-NEXT: v_add_co_u32_e32 v7, vcc, s4, v0
+; GFX900-NEXT: v_readfirstlane_b32 s4, v11
+; GFX900-NEXT: v_addc_co_u32_e32 v8, vcc, 0, v1, vcc
+; GFX900-NEXT: s_mov_b32 m0, s4
+; GFX900-NEXT: global_load_dword v9, v[3:4], off offset:44
+; GFX900-NEXT: global_load_dword v10, v[0:1], off offset:4
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: s_movk_i32 s4, 0x58
+; GFX900-NEXT: global_load_dword v[7:8], off glc lds
+; GFX900-NEXT: v_add_u32_e32 v8, 0x58, v2
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: global_load_dword v7, v[0:1], off offset:8
+; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, s4, v3
+; GFX900-NEXT: v_readfirstlane_b32 s4, v8
+; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v4, vcc
+; GFX900-NEXT: s_mov_b32 m0, s4
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: s_nop 0
+; GFX900-NEXT: global_load_dword v[0:1], off glc slc lds
+; GFX900-NEXT: ; wave barrier
+; GFX900-NEXT: global_load_dword v0, v[3:4], off offset:48
+; GFX900-NEXT: ; asyncmark
+; GFX900-NEXT: ; wait_asyncmark(1)
+; GFX900-NEXT: s_waitcnt vmcnt(3)
+; GFX900-NEXT: ds_read_b32 v1, v2 offset:84
+; GFX900-NEXT: ; wait_asyncmark(0)
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: ds_read_b32 v2, v2 offset:88
+; GFX900-NEXT: v_add_u32_e32 v3, v10, v9
+; GFX900-NEXT: s_waitcnt lgkmcnt(1)
+; GFX900-NEXT: v_add3_u32 v1, v3, v1, v7
+; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_add3_u32 v0, v1, v0, v2
+; GFX900-NEXT: global_store_dword v[5:6], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: interleaved_with_wave_barrier:
+; GFX942: ; %bb.0: ; %entry
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_add_u32_e32 v11, 0x54, v2
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x54
+; GFX942-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_readfirstlane_b32 s0, v11
+; GFX942-NEXT: v_mov_b32_e32 v8, v3
+; GFX942-NEXT: s_mov_b32 m0, s0
+; GFX942-NEXT: global_load_dword v3, v[8:9], off offset:44
+; GFX942-NEXT: global_load_dword v10, v[0:1], off offset:4
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x58
+; GFX942-NEXT: global_load_lds_dword v[4:5], off sc0
+; GFX942-NEXT: v_add_u32_e32 v5, 0x58, v2
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:8
+; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[8:9], 0, s[0:1]
+; GFX942-NEXT: v_readfirstlane_b32 s0, v5
+; GFX942-NEXT: s_mov_b32 m0, s0
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: s_waitcnt vmcnt(2)
+; GFX942-NEXT: v_add_u32_e32 v3, v10, v3
+; GFX942-NEXT: global_load_lds_dword v[0:1], off sc0 nt
+; GFX942-NEXT: ; wave barrier
+; GFX942-NEXT: global_load_dword v0, v[8:9], off offset:48
+; GFX942-NEXT: ; asyncmark
+; GFX942-NEXT: ; wait_asyncmark(1)
+; GFX942-NEXT: s_waitcnt vmcnt(3)
+; GFX942-NEXT: ds_read_b32 v1, v2 offset:84
+; GFX942-NEXT: ; wait_asyncmark(0)
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: ds_read_b32 v2, v2 offset:88
+; GFX942-NEXT: s_waitcnt lgkmcnt(1)
+; GFX942-NEXT: v_add3_u32 v1, v3, v1, v4
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_add3_u32 v0, v1, v0, v2
+; GFX942-NEXT: global_store_dword v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1010-LABEL: interleaved_with_wave_barrier:
+; GFX1010: ; %bb.0: ; %entry
+; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_add_nc_u32_e32 v7, 0x54, v2
+; GFX1010-NEXT: v_add_nc_u32_e32 v11, 0x58, v2
+; GFX1010-NEXT: global_load_dword v9, v[3:4], off offset:44
+; GFX1010-NEXT: global_load_dword v10, v[0:1], off offset:4
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: v_readfirstlane_b32 s4, v7
+; GFX1010-NEXT: v_add_co_u32 v7, vcc_lo, 0x54, v0
+; GFX1010-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v1, vcc_lo
+; GFX1010-NEXT: s_mov_b32 m0, s4
+; GFX1010-NEXT: v_readfirstlane_b32 s4, v11
+; GFX1010-NEXT: global_load_dword v[7:8], off glc lds
+; GFX1010-NEXT: v_add_co_u32 v7, vcc_lo, 0x58, v3
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v4, vcc_lo
+; GFX1010-NEXT: global_load_dword v0, v[0:1], off offset:8
+; GFX1010-NEXT: s_mov_b32 m0, s4
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: global_load_dword v[7:8], off glc slc lds
+; GFX1010-NEXT: ; wave barrier
+; GFX1010-NEXT: global_load_dword v1, v[3:4], off offset:48
+; GFX1010-NEXT: ; asyncmark
+; GFX1010-NEXT: ; wait_asyncmark(1)
+; GFX1010-NEXT: s_waitcnt vmcnt(3)
+; GFX1010-NEXT: ds_read_b32 v3, v2 offset:84
+; GFX1010-NEXT: ; wait_asyncmark(0)
+; GFX1010-NEXT: s_waitcnt vmcnt(1)
+; GFX1010-NEXT: ds_read_b32 v2, v2 offset:88
+; GFX1010-NEXT: v_add_nc_u32_e32 v4, v10, v9
+; GFX1010-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1010-NEXT: v_add3_u32 v0, v4, v3, v0
+; GFX1010-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX1010-NEXT: v_add3_u32 v0, v0, v1, v2
+; GFX1010-NEXT: global_store_dword v[5:6], v0, off
+; GFX1010-NEXT: s_setpc_b64 s[30:31]
+entry:
+ ; First batch: global load, global load, async global-to-LDS
+ %bar_gep11 = getelementptr i32, ptr addrspace(1) %bar, i32 11
+ %bar_v11 = load i32, ptr addrspace(1) %bar_gep11
+ %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
+ %foo_v1 = load i32, ptr addrspace(1) %foo_gep1
+ %lds_gep21 = getelementptr i32, ptr addrspace(3) %lds, i32 21
+ %bar_gep21 = getelementptr i32, ptr addrspace(1) %foo, i32 21
+ call void @llvm.amdgcn.wave.barrier()
+ call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %bar_gep21, ptr
addrspace(3) %lds_gep21, i32 4, i32 0, i32 u0x21)
+ call void @llvm.amdgcn.wave.barrier()
+ call void @llvm.amdgcn.asyncmark()
----------------
nhaehnle wrote:
What's the logic behind having wave.barriers here (and elsewhere)? They are
unexpected to me.
(I briefly wondered about `convergent` in all this, but IMO the way to go is to
conceptually make the asyncmarks a thread-level concept in LLVM IR. They are
turned into wave-level concepts in assembly, but the worst that can happen is
more conservative waits.)
https://github.com/llvm/llvm-project/pull/173259
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
