llvm/test/CodeGen/AMDGPU/trans-bf16-omod.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefix=SDAG-FAKE16 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefix=GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-REAL16 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefix=GISEL-REAL16 %s

; FIXME: GlobalISel does not work with bf16

define amdgpu_ps void @v_cos_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_cos_bf16_e64 v0, v0 mul:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_cos_bf16_e64 v0.l, v0.l mul:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat %cos, 2.0
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_exp_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_exp_bf16_mul4:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_exp_bf16_e64 v0, v0 mul:4
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_exp_bf16_mul4:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_exp_bf16_e64 v0.l, v0.l mul:4
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %in)
  %mul4 = fmul nsz bfloat %exp, 4.0
  store bfloat %mul4, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_log_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_log_bf16_div2:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_log_bf16_e64 v0, v0 div:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_log_bf16_div2:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_log_bf16_e64 v0.l, v0.l div:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %in)
  %div2 = fmul nsz bfloat %log, 0.5
  store bfloat %div2, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_cos_bf16_mul2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_imm_first:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_cos_bf16_e64 v0, v0 mul:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2_imm_first:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_cos_bf16_e64 v0.l, v0.l mul:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat 2.0, %cos
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_exp_bf16_mul4_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_exp_bf16_mul4_imm_first:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_exp_bf16_e64 v0, v0 mul:4
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_exp_bf16_mul4_imm_first:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_exp_bf16_e64 v0.l, v0.l mul:4
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %in)
  %mul4 = fmul nsz bfloat 4.0, %exp
  store bfloat %mul4, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_log_bf16_div2_imm_first(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_log_bf16_div2_imm_first:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_log_bf16_e64 v0, v0 div:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_log_bf16_div2_imm_first:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_log_bf16_e64 v0.l, v0.l div:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %in)
  %div2 = fmul nsz bfloat 0.5, %log
  store bfloat %div2, ptr addrspace(1) %out
  ret void
}


define amdgpu_ps void @v_rcp_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_rcp_bf16_mul2:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_rcp_bf16_e64 v0, v0 mul:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_rcp_bf16_mul2:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_rcp_bf16_e64 v0.l, v0.l mul:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %rcp = call bfloat @llvm.amdgcn.rcp.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat %rcp, 2.0
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_rsq_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_rsq_bf16_mul4:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_rsq_bf16_e64 v0, v0 mul:4
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_rsq_bf16_mul4:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_rsq_bf16_e64 v0.l, v0.l mul:4
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat %in)
  %mul4 = fmul nsz bfloat %rsq, 4.0
  store bfloat %mul4, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_sin_bf16_div2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_sin_bf16_div2:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_sin_bf16_e64 v0, v0 div:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_sin_bf16_div2:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_sin_bf16_e64 v0.l, v0.l div:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %sin = call bfloat @llvm.amdgcn.sin.bf16(bfloat %in)
  %div2 = fmul nsz bfloat %sin, 0.5
  store bfloat %div2, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_sqrt_bf16_mul2(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_sqrt_bf16_mul2:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_sqrt_bf16_e64 v0, v0 mul:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_sqrt_bf16_mul2:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_sqrt_bf16_e64 v0.l, v0.l mul:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %sqrt = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat %sqrt, 2.0
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_tanh_bf16_mul4(bfloat %in, ptr addrspace(1) %out) #0 {
; SDAG-FAKE16-LABEL: v_tanh_bf16_mul4:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_tanh_bf16_e64 v0, v0 mul:4
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_tanh_bf16_mul4:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_tanh_bf16_e64 v0.l, v0.l mul:4
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %tanh = call bfloat @llvm.amdgcn.tanh.bf16(bfloat %in)
  %mul4 = fmul nsz bfloat %tanh, 4.0
  store bfloat %mul4, ptr addrspace(1) %out
  ret void
}

; Output denormals enabled for every type: no omod folding.
define amdgpu_ps void @v_cos_bf16_mul2_denorm_ieee(bfloat %in, ptr addrspace(1) %out) #1 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_denorm_ieee:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_cos_bf16_e32 v0, v0
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
; SDAG-FAKE16-NEXT:    v_pk_add_bf16 v0, v0, v0
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2_denorm_ieee:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_cos_bf16_e32 v0.l, v0.l
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
; SDAG-REAL16-NEXT:    v_pk_add_bf16 v0, v0, v0
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat %cos, 2.0
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

; Only f32 output denormals are flushed. bf16 follows the default mode, which
; is IEEE here, so there is still no omod folding.
define amdgpu_ps void @v_cos_bf16_mul2_f32_denorm_flush(bfloat %in, ptr addrspace(1) %out) #2 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_f32_denorm_flush:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_cos_bf16_e32 v0, v0
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
; SDAG-FAKE16-NEXT:    v_pk_add_bf16 v0, v0, v0
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2_f32_denorm_flush:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_cos_bf16_e32 v0.l, v0.l
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
; SDAG-REAL16-NEXT:    v_pk_add_bf16 v0, v0, v0
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat %cos, 2.0
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

; The default mode flushes output denormals, so bf16 does too, even though f32
; is IEEE: omod folding applies.
define amdgpu_ps void @v_cos_bf16_mul2_default_denorm_flush(bfloat %in, ptr addrspace(1) %out) #3 {
; SDAG-FAKE16-LABEL: v_cos_bf16_mul2_default_denorm_flush:
; SDAG-FAKE16:       ; %bb.0:
; SDAG-FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-FAKE16-NEXT:    v_nop
; SDAG-FAKE16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-FAKE16-NEXT:    v_cos_bf16_e64 v0, v0 mul:2
; SDAG-FAKE16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-FAKE16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-FAKE16-NEXT:    s_endpgm
;
; SDAG-REAL16-LABEL: v_cos_bf16_mul2_default_denorm_flush:
; SDAG-REAL16:       ; %bb.0:
; SDAG-REAL16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-REAL16-NEXT:    v_nop
; SDAG-REAL16-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-REAL16-NEXT:    v_cos_bf16_e64 v0.l, v0.l mul:2
; SDAG-REAL16-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; SDAG-REAL16-NEXT:    global_store_b16 v[2:3], v0, off
; SDAG-REAL16-NEXT:    s_endpgm
  %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %in)
  %mul2 = fmul nsz bfloat %cos, 2.0
  store bfloat %mul2, ptr addrspace(1) %out
  ret void
}

attributes #0 = { nounwind denormal_fpenv(preservesign) }
attributes #1 = { nounwind denormal_fpenv(ieee) }
attributes #2 = { nounwind denormal_fpenv(ieee, float: preservesign) }
attributes #3 = { nounwind denormal_fpenv(preservesign, float: ieee) }