https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/116678
None >From f3682aa080aebde46106fa11176442973ff62c26 Mon Sep 17 00:00:00 2001 From: Pravin Jagtap <pravin.jag...@amd.com> Date: Mon, 5 Feb 2024 04:29:01 -0500 Subject: [PATCH] AMDGPU: Add V_CVT_PK_BF16_F32 for gfx950 --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 + llvm/lib/Target/AMDGPU/SIInstrInfo.td | 1 + llvm/lib/Target/AMDGPU/VOP3Instructions.td | 25 ++ llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 395 ++++++++---------- llvm/test/MC/AMDGPU/gfx950_asm_vop3.s | 26 ++ .../Disassembler/AMDGPU/gfx950_dasm_vop3.txt | 19 + 6 files changed, 255 insertions(+), 217 deletions(-) create mode 100644 llvm/test/MC/AMDGPU/gfx950_asm_vop3.s create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 1e261f4256c93b..ad89812558d25c 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -889,6 +889,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::MUL, MVT::i1, Promote); + if (Subtarget->hasBF16ConversionInsts()) { + setOperationAction(ISD::FP_ROUND, MVT::v2bf16, Legal); + setOperationAction(ISD::FP_ROUND, MVT::bf16, Legal); + setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal); + } + setTargetDAGCombine({ISD::ADD, ISD::UADDO_CARRY, ISD::SUB, diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 882e147dc231fa..7df9be5c6f7a0b 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -2787,6 +2787,7 @@ def VOP_I32_I32_I32 : VOPProfile <[i32, i32, i32, untyped]>; def VOP_I32_I32_I32_ARITH : VOPProfile <[i32, i32, i32, untyped], /*EnableClamp=*/1>; def VOP_V2F16_F32_F32 : VOPProfile <[v2f16, f32, f32, untyped]>; def VOP_F32_F16_F16_F16 : VOPProfile <[f32, f16, f16, f16]>; +def VOP_V2BF16_F32_F32 : VOPProfile <[v2bf16, f32, f32, untyped]>; def VOP_I64_I64_I32 : VOPProfile <[i64, i64, i32, untyped]>; def VOP_I64_I32_I64 : VOPProfile <[i64, i32, i64, untyped]>; diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 551e8b3a679202..917e1b3974b46a 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -944,6 +944,30 @@ let SubtargetPredicate = isGFX11Plus in { defm V_CVT_PK_U16_F32 : VOP3Inst<"v_cvt_pk_u16_f32", VOP3_Profile<VOP_V2I16_F32_F32>>; } // End SubtargetPredicate = isGFX11Plus +// FIXME: GlobalISel cannot distinguish f16 and bf16 and may start using bf16 patterns +// instead of less complex f16. Disable GlobalISel for these for now. +def bf16_fpround : PatFrag <(ops node:$src0), (fpround $src0), [{ return true; }]> { + let GISelPredicateCode = [{return false;}]; +} + +let SubtargetPredicate = HasBF16ConversionInsts in { + let ReadsModeReg = 0 in { + defm V_CVT_PK_BF16_F32 : VOP3Inst<"v_cvt_pk_bf16_f32", VOP3_Profile<VOP_V2BF16_F32_F32>>; + } + def : GCNPat<(v2bf16 (bf16_fpround v2f32:$src)), + (V_CVT_PK_BF16_F32_e64 0, (EXTRACT_SUBREG VReg_64:$src, sub0), 0, (EXTRACT_SUBREG VReg_64:$src, sub1))>; + def : GCNPat<(v2bf16 (bf16_fpround v2f64:$src)), + (V_CVT_PK_BF16_F32_e64 0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub0_sub1)), + 0, (V_CVT_F32_F64_e64 0, (EXTRACT_SUBREG VReg_128:$src, sub2_sub3)))>; + def : GCNPat<(v2bf16 (build_vector (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), + (bf16 (bf16_fpround (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)))))), + (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1)>; + def : GCNPat<(bf16 (bf16_fpround (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), + (V_CVT_PK_BF16_F32_e64 $src0_modifiers, $src0, 0, (f32 (IMPLICIT_DEF)))>; + def : GCNPat<(bf16 (bf16_fpround (f64 (VOP3Mods f64:$src0, i32:$src0_modifiers)))), + (V_CVT_PK_BF16_F32_e64 0, (f32 (V_CVT_F32_F64_e64 $src0_modifiers, $src0)), 0, (f32 (IMPLICIT_DEF)))>; +} + let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in { defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>; defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>; @@ -1721,5 +1745,6 @@ defm V_LSHL_ADD_U64 : VOP3_Real_vi <0x208>; defm V_CVT_PK_FP8_F32 : VOP3OpSel_Real_gfx9 <0x2a2>; defm V_CVT_PK_BF8_F32 : VOP3OpSel_Real_gfx9 <0x2a3>; +defm V_CVT_PK_BF16_F32: VOP3OpSel_Real_gfx9 <0x268>; defm V_CVT_SR_FP8_F32 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x2a4>; defm V_CVT_SR_BF8_F32 : VOP3OpSel_Real_gfx9_forced_opsel2 <0x2a5>; diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll index 425fc5884cec7f..135efceb31fdda 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll @@ -24,139 +24,168 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) { } define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_v(<2 x float> %src) { -; GCN-LABEL: v_test_cvt_v2f32_v2bf16_v: -; GCN: ; %bb.0: -; GCN-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GCN-NEXT: s_movk_i32 s0, 0x7fff -; GCN-NEXT: v_add3_u32 v2, v2, v0, s0 -; GCN-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GCN-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; GCN-NEXT: s_nop 1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc -; GCN-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GCN-NEXT: v_add3_u32 v2, v2, v1, s0 -; GCN-NEXT: v_or_b32_e32 v3, 0x400000, v1 -; GCN-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; GCN-NEXT: s_mov_b32 s0, 0x7060302 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc -; GCN-NEXT: v_perm_b32 v0, v1, v0, s0 -; GCN-NEXT: ; return to shader part epilog +; GFX-940-LABEL: v_test_cvt_v2f32_v2bf16_v: +; GFX-940: ; %bb.0: +; GFX-940-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX-940-NEXT: s_movk_i32 s0, 0x7fff +; GFX-940-NEXT: v_add3_u32 v2, v2, v0, s0 +; GFX-940-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX-940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 +; GFX-940-NEXT: s_nop 1 +; GFX-940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc +; GFX-940-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX-940-NEXT: v_add3_u32 v2, v2, v1, s0 +; GFX-940-NEXT: v_or_b32_e32 v3, 0x400000, v1 +; GFX-940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 +; GFX-940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX-940-NEXT: s_nop 0 +; GFX-940-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc +; GFX-940-NEXT: v_perm_b32 v0, v1, v0, s0 +; GFX-940-NEXT: ; return to shader part epilog +; +; GFX-950-LABEL: v_test_cvt_v2f32_v2bf16_v: +; GFX-950: ; %bb.0: +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX-950-NEXT: ; return to shader part epilog %res = fptrunc <2 x float> %src to <2 x bfloat> %cast = bitcast <2 x bfloat> %res to float ret float %cast } define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) { -; GCN-LABEL: v_test_cvt_v2f32_v2bf16_s: -; GCN: ; %bb.0: -; GCN-NEXT: s_bfe_u32 s2, s1, 0x10010 -; GCN-NEXT: s_add_i32 s2, s2, s1 -; GCN-NEXT: s_or_b32 s4, s1, 0x400000 -; GCN-NEXT: s_add_i32 s5, s2, 0x7fff -; GCN-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s1 -; GCN-NEXT: s_and_b64 s[2:3], s[2:3], exec -; GCN-NEXT: s_cselect_b32 s1, s4, s5 -; GCN-NEXT: s_lshr_b32 s2, s1, 16 -; GCN-NEXT: s_bfe_u32 s1, s0, 0x10010 -; GCN-NEXT: s_add_i32 s1, s1, s0 -; GCN-NEXT: s_or_b32 s3, s0, 0x400000 -; GCN-NEXT: s_add_i32 s4, s1, 0x7fff -; GCN-NEXT: v_cmp_u_f32_e64 s[0:1], s0, s0 -; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GCN-NEXT: s_cselect_b32 s0, s3, s4 -; GCN-NEXT: s_lshr_b32 s0, s0, 16 -; GCN-NEXT: s_pack_ll_b32_b16 s0, s0, s2 -; GCN-NEXT: v_mov_b32_e32 v0, s0 -; GCN-NEXT: ; return to shader part epilog +; GFX-940-LABEL: v_test_cvt_v2f32_v2bf16_s: +; GFX-940: ; %bb.0: +; GFX-940-NEXT: s_bfe_u32 s2, s1, 0x10010 +; GFX-940-NEXT: s_add_i32 s2, s2, s1 +; GFX-940-NEXT: s_or_b32 s4, s1, 0x400000 +; GFX-940-NEXT: s_add_i32 s5, s2, 0x7fff +; GFX-940-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s1 +; GFX-940-NEXT: s_and_b64 s[2:3], s[2:3], exec +; GFX-940-NEXT: s_cselect_b32 s1, s4, s5 +; GFX-940-NEXT: s_lshr_b32 s2, s1, 16 +; GFX-940-NEXT: s_bfe_u32 s1, s0, 0x10010 +; GFX-940-NEXT: s_add_i32 s1, s1, s0 +; GFX-940-NEXT: s_or_b32 s3, s0, 0x400000 +; GFX-940-NEXT: s_add_i32 s4, s1, 0x7fff +; GFX-940-NEXT: v_cmp_u_f32_e64 s[0:1], s0, s0 +; GFX-940-NEXT: s_and_b64 s[0:1], s[0:1], exec +; GFX-940-NEXT: s_cselect_b32 s0, s3, s4 +; GFX-940-NEXT: s_lshr_b32 s0, s0, 16 +; GFX-940-NEXT: s_pack_ll_b32_b16 s0, s0, s2 +; GFX-940-NEXT: v_mov_b32_e32 v0, s0 +; GFX-940-NEXT: ; return to shader part epilog +; +; GFX-950-LABEL: v_test_cvt_v2f32_v2bf16_s: +; GFX-950: ; %bb.0: +; GFX-950-NEXT: v_mov_b32_e32 v0, s1 +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, s0, v0 +; GFX-950-NEXT: ; return to shader part epilog %res = fptrunc <2 x float> %src to <2 x bfloat> %cast = bitcast <2 x bfloat> %res to float ret float %cast } define amdgpu_ps float @v_test_cvt_f32_bf16_v(float %src) { -; GCN-LABEL: v_test_cvt_f32_bf16_v: -; GCN: ; %bb.0: -; GCN-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GCN-NEXT: s_movk_i32 s0, 0x7fff -; GCN-NEXT: v_add3_u32 v1, v1, v0, s0 -; GCN-NEXT: v_or_b32_e32 v2, 0x400000, v0 -; GCN-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; GCN-NEXT: s_nop 1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc -; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GCN-NEXT: ; return to shader part epilog +; GFX-940-LABEL: v_test_cvt_f32_bf16_v: +; GFX-940: ; %bb.0: +; GFX-940-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX-940-NEXT: s_movk_i32 s0, 0x7fff +; GFX-940-NEXT: v_add3_u32 v1, v1, v0, s0 +; GFX-940-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX-940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 +; GFX-940-NEXT: s_nop 1 +; GFX-940-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc +; GFX-940-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX-940-NEXT: ; return to shader part epilog +; +; GFX-950-LABEL: v_test_cvt_f32_bf16_v: +; GFX-950: ; %bb.0: +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX-950-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX-950-NEXT: ; return to shader part epilog %trunc = fptrunc float %src to bfloat %ext = fpext bfloat %trunc to float ret float %ext } define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) { -; GCN-LABEL: v_test_cvt_v2f64_v2bf16_v: -; GCN: ; %bb.0: -; GCN-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]| -; GCN-NEXT: v_cvt_f64_f32_e32 v[4:5], v6 -; GCN-NEXT: v_and_b32_e32 v7, 1, v6 -; GCN-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5] -; GCN-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5] -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 -; GCN-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3] -; GCN-NEXT: v_add_u32_e32 v4, v6, v4 -; GCN-NEXT: s_or_b64 vcc, s[0:1], vcc -; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc -; GCN-NEXT: s_brev_b32 s4, 1 -; GCN-NEXT: v_and_or_b32 v5, v1, s4, v4 -; GCN-NEXT: v_bfe_u32 v4, v4, 16, 1 -; GCN-NEXT: s_movk_i32 s5, 0x7fff -; GCN-NEXT: v_add3_u32 v4, v4, v5, s5 -; GCN-NEXT: v_or_b32_e32 v5, 0x400000, v5 -; GCN-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1] -; GCN-NEXT: s_nop 1 -; GCN-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc -; GCN-NEXT: v_cvt_f32_f64_e64 v5, |v[2:3]| -; GCN-NEXT: v_cvt_f64_f32_e32 v[0:1], v5 -; GCN-NEXT: v_and_b32_e32 v6, 1, v5 -; GCN-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[2:3]|, v[0:1] -; GCN-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[2:3]|, v[0:1] -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 -; GCN-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3] -; GCN-NEXT: v_add_u32_e32 v0, v5, v0 -; GCN-NEXT: s_or_b64 vcc, s[0:1], vcc -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc -; GCN-NEXT: v_and_or_b32 v1, v3, s4, v0 -; GCN-NEXT: v_bfe_u32 v0, v0, 16, 1 -; GCN-NEXT: v_add3_u32 v0, v0, v1, s5 -; GCN-NEXT: v_or_b32_e32 v1, 0x400000, v1 -; GCN-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[2:3] -; GCN-NEXT: s_mov_b32 s0, 0x7060302 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GCN-NEXT: v_perm_b32 v0, v0, v4, s0 -; GCN-NEXT: ; return to shader part epilog +; GFX-940-LABEL: v_test_cvt_v2f64_v2bf16_v: +; GFX-940: ; %bb.0: +; GFX-940-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]| +; GFX-940-NEXT: v_cvt_f64_f32_e32 v[4:5], v6 +; GFX-940-NEXT: v_and_b32_e32 v7, 1, v6 +; GFX-940-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5] +; GFX-940-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5] +; GFX-940-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 +; GFX-940-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3] +; GFX-940-NEXT: v_add_u32_e32 v4, v6, v4 +; GFX-940-NEXT: s_or_b64 vcc, s[0:1], vcc +; GFX-940-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX-940-NEXT: s_brev_b32 s4, 1 +; GFX-940-NEXT: v_and_or_b32 v5, v1, s4, v4 +; GFX-940-NEXT: v_bfe_u32 v4, v4, 16, 1 +; GFX-940-NEXT: s_movk_i32 s5, 0x7fff +; GFX-940-NEXT: v_add3_u32 v4, v4, v5, s5 +; GFX-940-NEXT: v_or_b32_e32 v5, 0x400000, v5 +; GFX-940-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1] +; GFX-940-NEXT: s_nop 1 +; GFX-940-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc +; GFX-940-NEXT: v_cvt_f32_f64_e64 v5, |v[2:3]| +; GFX-940-NEXT: v_cvt_f64_f32_e32 v[0:1], v5 +; GFX-940-NEXT: v_and_b32_e32 v6, 1, v5 +; GFX-940-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[2:3]|, v[0:1] +; GFX-940-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[2:3]|, v[0:1] +; GFX-940-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 +; GFX-940-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3] +; GFX-940-NEXT: v_add_u32_e32 v0, v5, v0 +; GFX-940-NEXT: s_or_b64 vcc, s[0:1], vcc +; GFX-940-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc +; GFX-940-NEXT: v_and_or_b32 v1, v3, s4, v0 +; GFX-940-NEXT: v_bfe_u32 v0, v0, 16, 1 +; GFX-940-NEXT: v_add3_u32 v0, v0, v1, s5 +; GFX-940-NEXT: v_or_b32_e32 v1, 0x400000, v1 +; GFX-940-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[2:3] +; GFX-940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX-940-NEXT: s_nop 0 +; GFX-940-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX-940-NEXT: v_perm_b32 v0, v0, v4, s0 +; GFX-940-NEXT: ; return to shader part epilog +; +; GFX-950-LABEL: v_test_cvt_v2f64_v2bf16_v: +; GFX-950: ; %bb.0: +; GFX-950-NEXT: v_cvt_f32_f64_e32 v2, v[2:3] +; GFX-950-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2 +; GFX-950-NEXT: ; return to shader part epilog %res = fptrunc <2 x double> %src to <2 x bfloat> %cast = bitcast <2 x bfloat> %res to float ret float %cast } define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16(float %a, float %b) { -; GCN-LABEL: fptrunc_f32_f32_to_v2bf16: -; GCN: ; %bb.0: ; %entry -; GCN-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GCN-NEXT: s_movk_i32 s0, 0x7fff -; GCN-NEXT: v_add3_u32 v2, v2, v0, s0 -; GCN-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GCN-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; GCN-NEXT: s_nop 1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc -; GCN-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GCN-NEXT: v_add3_u32 v2, v2, v1, s0 -; GCN-NEXT: v_or_b32_e32 v3, 0x400000, v1 -; GCN-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; GCN-NEXT: s_mov_b32 s0, 0x7060302 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc -; GCN-NEXT: v_perm_b32 v0, v1, v0, s0 -; GCN-NEXT: ; return to shader part epilog +; GFX-940-LABEL: fptrunc_f32_f32_to_v2bf16: +; GFX-940: ; %bb.0: ; %entry +; GFX-940-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX-940-NEXT: s_movk_i32 s0, 0x7fff +; GFX-940-NEXT: v_add3_u32 v2, v2, v0, s0 +; GFX-940-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX-940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 +; GFX-940-NEXT: s_nop 1 +; GFX-940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc +; GFX-940-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX-940-NEXT: v_add3_u32 v2, v2, v1, s0 +; GFX-940-NEXT: v_or_b32_e32 v3, 0x400000, v1 +; GFX-940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 +; GFX-940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX-940-NEXT: s_nop 0 +; GFX-940-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc +; GFX-940-NEXT: v_perm_b32 v0, v1, v0, s0 +; GFX-940-NEXT: ; return to shader part epilog +; +; GFX-950-LABEL: fptrunc_f32_f32_to_v2bf16: +; GFX-950: ; %bb.0: ; %entry +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX-950-NEXT: ; return to shader part epilog entry: %a.cvt = fptrunc float %a to bfloat %b.cvt = fptrunc float %b to bfloat @@ -167,26 +196,31 @@ entry: } define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16_mods(float %a, float %b) { -; GCN-LABEL: fptrunc_f32_f32_to_v2bf16_mods: -; GCN: ; %bb.0: ; %entry -; GCN-NEXT: v_xor_b32_e32 v2, 0x80000000, v0 -; GCN-NEXT: v_bfe_u32 v3, v2, 16, 1 -; GCN-NEXT: s_movk_i32 s0, 0x7fff -; GCN-NEXT: v_add3_u32 v3, v3, v2, s0 -; GCN-NEXT: v_or_b32_e32 v2, 0x400000, v2 -; GCN-NEXT: v_cmp_u_f32_e64 vcc, -v0, -v0 -; GCN-NEXT: s_nop 1 -; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc -; GCN-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1 -; GCN-NEXT: v_bfe_u32 v3, v2, 16, 1 -; GCN-NEXT: v_add3_u32 v3, v3, v2, s0 -; GCN-NEXT: v_or_b32_e32 v2, 0x400000, v2 -; GCN-NEXT: v_cmp_u_f32_e64 vcc, |v1|, |v1| -; GCN-NEXT: s_mov_b32 s0, 0x7060302 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc -; GCN-NEXT: v_perm_b32 v0, v1, v0, s0 -; GCN-NEXT: ; return to shader part epilog +; GFX-940-LABEL: fptrunc_f32_f32_to_v2bf16_mods: +; GFX-940: ; %bb.0: ; %entry +; GFX-940-NEXT: v_xor_b32_e32 v2, 0x80000000, v0 +; GFX-940-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX-940-NEXT: s_movk_i32 s0, 0x7fff +; GFX-940-NEXT: v_add3_u32 v3, v3, v2, s0 +; GFX-940-NEXT: v_or_b32_e32 v2, 0x400000, v2 +; GFX-940-NEXT: v_cmp_u_f32_e64 vcc, -v0, -v0 +; GFX-940-NEXT: s_nop 1 +; GFX-940-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc +; GFX-940-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1 +; GFX-940-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX-940-NEXT: v_add3_u32 v3, v3, v2, s0 +; GFX-940-NEXT: v_or_b32_e32 v2, 0x400000, v2 +; GFX-940-NEXT: v_cmp_u_f32_e64 vcc, |v1|, |v1| +; GFX-940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX-940-NEXT: s_nop 0 +; GFX-940-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc +; GFX-940-NEXT: v_perm_b32 v0, v1, v0, s0 +; GFX-940-NEXT: ; return to shader part epilog +; +; GFX-950-LABEL: fptrunc_f32_f32_to_v2bf16_mods: +; GFX-950: ; %bb.0: ; %entry +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, -v0, |v1| +; GFX-950-NEXT: ; return to shader part epilog entry: %a.neg = fneg float %a %a.cvt = fptrunc float %a.neg to bfloat @@ -217,14 +251,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16(float %a, ptr %out) { ; GFX-950: ; %bb.0: ; %entry ; GFX-950-NEXT: v_mov_b32_e32 v3, v2 ; GFX-950-NEXT: v_mov_b32_e32 v2, v1 -; GFX-950-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GFX-950-NEXT: s_movk_i32 s0, 0x7fff -; GFX-950-NEXT: v_add3_u32 v1, v1, v0, s0 -; GFX-950-NEXT: v_or_b32_e32 v4, 0x400000, v0 -; GFX-950-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; GFX-950-NEXT: s_nop 1 -; GFX-950-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc -; GFX-950-NEXT: flat_store_short_d16_hi v[2:3], v0 +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX-950-NEXT: flat_store_short v[2:3], v0 ; GFX-950-NEXT: s_endpgm entry: %a.cvt = fptrunc float %a to bfloat @@ -252,15 +280,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_abs(float %a, ptr %out) { ; GFX-950: ; %bb.0: ; %entry ; GFX-950-NEXT: v_mov_b32_e32 v3, v2 ; GFX-950-NEXT: v_mov_b32_e32 v2, v1 -; GFX-950-NEXT: v_and_b32_e32 v1, 0x7fffffff, v0 -; GFX-950-NEXT: v_bfe_u32 v4, v1, 16, 1 -; GFX-950-NEXT: s_movk_i32 s0, 0x7fff -; GFX-950-NEXT: v_add3_u32 v4, v4, v1, s0 -; GFX-950-NEXT: v_or_b32_e32 v1, 0x400000, v1 -; GFX-950-NEXT: v_cmp_u_f32_e64 vcc, |v0|, |v0| -; GFX-950-NEXT: s_nop 1 -; GFX-950-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc -; GFX-950-NEXT: flat_store_short_d16_hi v[2:3], v0 +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, |v0|, s0 +; GFX-950-NEXT: flat_store_short v[2:3], v0 ; GFX-950-NEXT: s_endpgm entry: %a.abs = call float @llvm.fabs.f32(float %a) @@ -289,15 +310,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_neg(float %a, ptr %out) { ; GFX-950: ; %bb.0: ; %entry ; GFX-950-NEXT: v_mov_b32_e32 v3, v2 ; GFX-950-NEXT: v_mov_b32_e32 v2, v1 -; GFX-950-NEXT: v_xor_b32_e32 v1, 0x80000000, v0 -; GFX-950-NEXT: v_bfe_u32 v4, v1, 16, 1 -; GFX-950-NEXT: s_movk_i32 s0, 0x7fff -; GFX-950-NEXT: v_add3_u32 v4, v4, v1, s0 -; GFX-950-NEXT: v_or_b32_e32 v1, 0x400000, v1 -; GFX-950-NEXT: v_cmp_u_f32_e64 vcc, -v0, -v0 -; GFX-950-NEXT: s_nop 1 -; GFX-950-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc -; GFX-950-NEXT: flat_store_short_d16_hi v[2:3], v0 +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, -v0, s0 +; GFX-950-NEXT: flat_store_short v[2:3], v0 ; GFX-950-NEXT: s_endpgm entry: %a.neg = fneg float %a @@ -333,26 +347,9 @@ define amdgpu_ps void @fptrunc_f64_to_bf16(double %a, ptr %out) { ; ; GFX-950-LABEL: fptrunc_f64_to_bf16: ; GFX-950: ; %bb.0: ; %entry -; GFX-950-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]| -; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v6 -; GFX-950-NEXT: v_and_b32_e32 v7, 1, v6 -; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5] -; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5] -; GFX-950-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 -; GFX-950-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3] -; GFX-950-NEXT: v_add_u32_e32 v4, v6, v4 -; GFX-950-NEXT: s_or_b64 vcc, s[0:1], vcc -; GFX-950-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc -; GFX-950-NEXT: s_brev_b32 s0, 1 -; GFX-950-NEXT: v_and_or_b32 v5, v1, s0, v4 -; GFX-950-NEXT: v_bfe_u32 v4, v4, 16, 1 -; GFX-950-NEXT: s_movk_i32 s0, 0x7fff -; GFX-950-NEXT: v_add3_u32 v4, v4, v5, s0 -; GFX-950-NEXT: v_or_b32_e32 v5, 0x400000, v5 -; GFX-950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1] -; GFX-950-NEXT: s_nop 1 -; GFX-950-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc -; GFX-950-NEXT: flat_store_short_d16_hi v[2:3], v0 +; GFX-950-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX-950-NEXT: flat_store_short v[2:3], v0 ; GFX-950-NEXT: s_endpgm entry: %a.cvt = fptrunc double %a to bfloat @@ -388,27 +385,9 @@ define amdgpu_ps void @fptrunc_f64_to_bf16_neg(double %a, ptr %out) { ; ; GFX-950-LABEL: fptrunc_f64_to_bf16_neg: ; GFX-950: ; %bb.0: ; %entry -; GFX-950-NEXT: v_cvt_f32_f64_e64 v7, |v[0:1]| -; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v7 -; GFX-950-NEXT: v_and_b32_e32 v8, 1, v7 -; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5] -; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5] -; GFX-950-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8 -; GFX-950-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3] -; GFX-950-NEXT: v_add_u32_e32 v4, v7, v4 -; GFX-950-NEXT: s_or_b64 vcc, s[0:1], vcc -; GFX-950-NEXT: s_brev_b32 s4, 1 -; GFX-950-NEXT: v_xor_b32_e32 v6, 0x80000000, v1 -; GFX-950-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc -; GFX-950-NEXT: v_and_or_b32 v5, v6, s4, v4 -; GFX-950-NEXT: v_bfe_u32 v4, v4, 16, 1 -; GFX-950-NEXT: s_movk_i32 s0, 0x7fff -; GFX-950-NEXT: v_add3_u32 v4, v4, v5, s0 -; GFX-950-NEXT: v_or_b32_e32 v5, 0x400000, v5 -; GFX-950-NEXT: v_cmp_u_f64_e64 vcc, -v[0:1], -v[0:1] -; GFX-950-NEXT: s_nop 1 -; GFX-950-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc -; GFX-950-NEXT: flat_store_short_d16_hi v[2:3], v0 +; GFX-950-NEXT: v_cvt_f32_f64_e64 v0, -v[0:1] +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX-950-NEXT: flat_store_short v[2:3], v0 ; GFX-950-NEXT: s_endpgm entry: %a.neg = fneg double %a @@ -445,27 +424,9 @@ define amdgpu_ps void @fptrunc_f64_to_bf16_abs(double %a, ptr %out) { ; ; GFX-950-LABEL: fptrunc_f64_to_bf16_abs: ; GFX-950: ; %bb.0: ; %entry -; GFX-950-NEXT: v_cvt_f32_f64_e64 v7, |v[0:1]| -; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v7 -; GFX-950-NEXT: v_and_b32_e32 v8, 1, v7 -; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, v[4:5] -; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5] -; GFX-950-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8 -; GFX-950-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3] -; GFX-950-NEXT: v_add_u32_e32 v4, v7, v4 -; GFX-950-NEXT: s_or_b64 vcc, s[0:1], vcc -; GFX-950-NEXT: v_and_b32_e32 v6, 0x7fffffff, v1 -; GFX-950-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc -; GFX-950-NEXT: s_brev_b32 s0, 1 -; GFX-950-NEXT: v_and_or_b32 v5, v6, s0, v4 -; GFX-950-NEXT: v_bfe_u32 v4, v4, 16, 1 -; GFX-950-NEXT: s_movk_i32 s0, 0x7fff -; GFX-950-NEXT: v_add3_u32 v4, v4, v5, s0 -; GFX-950-NEXT: v_or_b32_e32 v5, 0x400000, v5 -; GFX-950-NEXT: v_cmp_u_f64_e64 vcc, |v[0:1]|, |v[0:1]| -; GFX-950-NEXT: s_nop 1 -; GFX-950-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc -; GFX-950-NEXT: flat_store_short_d16_hi v[2:3], v0 +; GFX-950-NEXT: v_cvt_f32_f64_e64 v0, |v[0:1]| +; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX-950-NEXT: flat_store_short v[2:3], v0 ; GFX-950-NEXT: s_endpgm entry: %a.abs = call double @llvm.fabs.f64(double %a) diff --git a/llvm/test/MC/AMDGPU/gfx950_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx950_asm_vop3.s new file mode 100644 index 00000000000000..c9980f420b9552 --- /dev/null +++ b/llvm/test/MC/AMDGPU/gfx950_asm_vop3.s @@ -0,0 +1,26 @@ +// RUN: llvm-mc -arch=amdgcn -mcpu=gfx950 -show-encoding %s | FileCheck --check-prefix=GFX950 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx940 %s 2>&1 | FileCheck -check-prefix=GFX940-ERR --strict-whitespace %s + +v_cvt_pk_bf16_f32 v5, v1, v2 +// GFX950: v_cvt_pk_bf16_f32 v5, v1, v2 ; encoding: [0x05,0x00,0x68,0xd2,0x01,0x05,0x02,0x00] +// GFX940-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU + +v_cvt_pk_bf16_f32 v5, v255, v255 +// GFX950: v_cvt_pk_bf16_f32 v5, v255, v255 ; encoding: [0x05,0x00,0x68,0xd2,0xff,0xff,0x03,0x00] +// GFX940-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU + +v_cvt_pk_bf16_f32 v5, v1, s2 +// GFX950: v_cvt_pk_bf16_f32 v5, v1, s2 ; encoding: [0x05,0x00,0x68,0xd2,0x01,0x05,0x00,0x00] +// GFX940-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU + +v_cvt_pk_bf16_f32 v5, m0, 0.5 +// GFX950: v_cvt_pk_bf16_f32 v5, m0, 0.5 ; encoding: [0x05,0x00,0x68,0xd2,0x7c,0xe0,0x01,0x00] +// GFX940-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU + +v_cvt_pk_bf16_f32 v5, -1, exec_hi +// GFX950: v_cvt_pk_bf16_f32 v5, -1, exec_hi ; encoding: [0x05,0x00,0x68,0xd2,0xc1,0xfe,0x00,0x00] +// GFX940-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU + +v_cvt_pk_bf16_f32 v5, 0.5, m0 mul:2 +// GFX950: v_cvt_pk_bf16_f32 v5, 0.5, m0 mul:2 ; encoding: [0x05,0x00,0x68,0xd2,0xf0,0xf8,0x00,0x08] +// GFX940-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt new file mode 100644 index 00000000000000..909743c2babf59 --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx950_dasm_vop3.txt @@ -0,0 +1,19 @@ +# RUN: llvm-mc -arch=amdgcn -mcpu=gfx950 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX950 %s + +# GFX950: v_cvt_pk_bf16_f32 v5, v1, v2 ; encoding: [0x05,0x00,0x68,0xd2,0x01,0x05,0x02,0x00] +0x05,0x00,0x68,0xd2,0x01,0x05,0x02,0x00 + +# GFX950: v_cvt_pk_bf16_f32 v5, v255, v255 ; encoding: [0x05,0x00,0x68,0xd2,0xff,0xff,0x03,0x00] +0x05,0x00,0x68,0xd2,0xff,0xff,0x03,0x00 + +# GFX950: v_cvt_pk_bf16_f32 v5, v1, s2 ; encoding: [0x05,0x00,0x68,0xd2,0x01,0x05,0x00,0x00] +0x05,0x00,0x68,0xd2,0x01,0x05,0x00,0x00 + +# GFX950: v_cvt_pk_bf16_f32 v5, m0, 0.5 ; encoding: [0x05,0x00,0x68,0xd2,0x7c,0xe0,0x01,0x00] +0x05,0x00,0x68,0xd2,0x7c,0xe0,0x01,0x00 + +# GFX950: v_cvt_pk_bf16_f32 v5, -1, exec_hi ; encoding: [0x05,0x00,0x68,0xd2,0xc1,0xfe,0x00,0x00] +0x05,0x00,0x68,0xd2,0xc1,0xfe,0x00,0x00 + +# GFX950: v_cvt_pk_bf16_f32 v5, 0.5, m0 mul:2 ; encoding: [0x05,0x00,0x68,0xd2,0xf0,0xf8,0x00,0x08] +0x05,0x00,0x68,0xd2,0xf0,0xf8,0x00,0x08 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits