Google Git
Sign in
llvm/llvm-project/e14b11b74dcafea1e2a71b9788b85b34ab4cf1e6/./llvm/test/Transforms/SLPVectorizer/AMDGPU/notriviallyvectorizableintrinsicoperands.ll
blob: 533f8bce9ed18107bd5d01244291476c02441d0c [file]
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu12.50-amd-amdhsa -o - %s | FileCheck %s --check-prefix GCN
define amdgpu_kernel void @test_with_wmma( ptr addrspace(1) %input, ptr addrspace(1) %output, float %scaled_max, <16 x i32> %A, <16 x i32> %B, i32 %scale_idx) {
; GCN-LABEL: define amdgpu_kernel void @test_with_wmma(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[OUTPUT:%.*]], float [[SCALED_MAX:%.*]], <16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]], i32 [[SCALE_IDX:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[IN0:%.*]] = load float, ptr addrspace(1) [[INPUT]], align 4
; GCN-NEXT: [[PTR1:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 1
; GCN-NEXT: [[IN1:%.*]] = load float, ptr addrspace(1) [[PTR1]], align 4
; GCN-NEXT: [[MUL0:%.*]] = fmul contract float [[IN0]], f0x3E0293EE
; GCN-NEXT: [[MUL1:%.*]] = fmul contract float [[IN1]], f0x3E0293EE
; GCN-NEXT: [[SUB0:%.*]] = fsub contract float [[MUL0]], [[SCALED_MAX]]
; GCN-NEXT: [[SUB1:%.*]] = fsub contract float [[MUL1]], [[SCALED_MAX]]
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB0]])
; GCN-NEXT: [[EXP1:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB1]])
; GCN-NEXT: [[VEC0:%.*]] = insertelement <2 x float> poison, float [[EXP0]], i64 0
; GCN-NEXT: [[VEC1:%.*]] = insertelement <2 x float> [[VEC0]], float [[EXP1]], i64 1
; GCN-NEXT: [[VEC_I32:%.*]] = bitcast <2 x float> [[VEC1]] to <2 x i32>
; GCN-NEXT: [[SCALE0:%.*]] = extractelement <2 x i32> [[VEC_I32]], i64 0
; GCN-NEXT: [[SCALE1:%.*]] = extractelement <2 x i32> [[VEC_I32]], i64 1
; GCN-NEXT: [[WMMA0:%.*]] = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> [[A]], i32 0, <16 x i32> [[B]], i16 0, <8 x float> zeroinitializer, i32 0, i32 0, i32 [[SCALE0]], i32 0, i32 0, i32 [[SCALE_IDX]], i1 false, i1 false)
; GCN-NEXT: [[WMMA1:%.*]] = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> [[A]], i32 0, <16 x i32> [[B]], i16 0, <8 x float> [[WMMA0]], i32 0, i32 0, i32 [[SCALE1]], i32 0, i32 0, i32 [[SCALE_IDX]], i1 false, i1 false)
; GCN-NEXT: store <8 x float> [[WMMA1]], ptr addrspace(1) [[OUTPUT]], align 32
; GCN-NEXT: ret void
;
entry:
%in0 = load float, ptr addrspace(1) %input, align 4
%ptr1 = getelementptr float, ptr addrspace(1) %input, i64 1
%in1 = load float, ptr addrspace(1) %ptr1, align 4
%mul0 = fmul contract float %in0, 0x3FC0527DC0000000
%mul1 = fmul contract float %in1, 0x3FC0527DC0000000
%sub0 = fsub contract float %mul0, %scaled_max
%sub1 = fsub contract float %mul1, %scaled_max
%exp0 = tail call float @llvm.amdgcn.exp2.f32(float %sub0)
%exp1 = tail call float @llvm.amdgcn.exp2.f32(float %sub1)
%vec0 = insertelement <2 x float> poison, float %exp0, i64 0
%vec1 = insertelement <2 x float> %vec0, float %exp1, i64 1
%vec_i32 = bitcast <2 x float> %vec1 to <2 x i32>
%scale0 = extractelement <2 x i32> %vec_i32, i64 0
%scale1 = extractelement <2 x i32> %vec_i32, i64 1
%wmma0 = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(
i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> zeroinitializer,
i32 0, i32 0, i32 %scale0, i32 0, i32 0, i32 %scale_idx, i1 false, i1 false)
%wmma1 = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(
i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %wmma0,
i32 0, i32 0, i32 %scale1, i32 0, i32 0, i32 %scale_idx, i1 false, i1 false)
store <8 x float> %wmma1, ptr addrspace(1) %output, align 32
ret void
}
define amdgpu_kernel void @test_amdgcn_exp_log(ptr addrspace(1) %input, ptr addrspace(1) %scales, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @test_amdgcn_exp_log(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[INPUT]], align 4
; GCN-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[SCALES]], align 4
; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x float> [[TMP0]], splat (float f0x3E0293EE)
; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x float> [[TMP2]], [[TMP1]]
; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB0]])
; GCN-NEXT: [[LOG0:%.*]] = tail call float @llvm.amdgcn.log.f32(float [[EXP0]])
; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x float> [[TMP3]], i64 1
; GCN-NEXT: [[EXP1:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB1]])
; GCN-NEXT: [[LOG1:%.*]] = tail call float @llvm.amdgcn.log.f32(float [[EXP1]])
; GCN-NEXT: [[SUM:%.*]] = fadd fast float [[LOG0]], [[LOG1]]
; GCN-NEXT: store float [[SUM]], ptr addrspace(1) [[OUTPUT]], align 4
; GCN-NEXT: ret void
;
entry:
%in0 = load float, ptr addrspace(1) %input, align 4
%ptr1 = getelementptr float, ptr addrspace(1) %input, i64 1
%in1 = load float, ptr addrspace(1) %ptr1, align 4
%scale0 = load float, ptr addrspace(1) %scales, align 4
%sptr1 = getelementptr float, ptr addrspace(1) %scales, i64 1
%scale1 = load float, ptr addrspace(1) %sptr1, align 4
%mul0 = fmul contract float %in0, 0x3FC0527DC0000000
%mul1 = fmul contract float %in1, 0x3FC0527DC0000000
%sub0 = fsub contract float %mul0, %scale0
%sub1 = fsub contract float %mul1, %scale1
%exp0 = tail call float @llvm.amdgcn.exp2.f32(float %sub0)
%log0 = tail call float @llvm.amdgcn.log.f32(float %exp0)
%exp1 = tail call float @llvm.amdgcn.exp2.f32(float %sub1)
%log1 = tail call float @llvm.amdgcn.log.f32(float %exp1)
%sum = fadd fast float %log0, %log1
store float %sum, ptr addrspace(1) %output, align 4
ret void
}
define amdgpu_kernel void @test_amdgcn_exp_f16(ptr addrspace(1) %input, ptr addrspace(1) %scales, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @test_amdgcn_exp_f16(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x half> [[TMP0]], splat (half 1.519530e+00)
; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x half> [[TMP2]], [[TMP1]]
; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x half> [[TMP3]], i64 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB0]])
; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x half> [[TMP3]], i64 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB1]])
; GCN-NEXT: [[SUM:%.*]] = fadd fast half [[EXP0]], [[EXP1]]
; GCN-NEXT: store half [[SUM]], ptr addrspace(1) [[OUTPUT]], align 2
; GCN-NEXT: ret void
;
entry:
%in0 = load half, ptr addrspace(1) %input, align 2
%ptr1 = getelementptr half, ptr addrspace(1) %input, i64 1
%in1 = load half, ptr addrspace(1) %ptr1, align 2
%scale0 = load half, ptr addrspace(1) %scales, align 2
%sptr1 = getelementptr half, ptr addrspace(1) %scales, i64 1
%scale1 = load half, ptr addrspace(1) %sptr1, align 2
%mul0 = fmul contract half %in0, 0xH3E14
%mul1 = fmul contract half %in1, 0xH3E14
%sub0 = fsub contract half %mul0, %scale0
%sub1 = fsub contract half %mul1, %scale1
%exp0 = tail call half @llvm.amdgcn.exp2.f16(half %sub0)
%exp1 = tail call half @llvm.amdgcn.exp2.f16(half %sub1)
%sum = fadd fast half %exp0, %exp1
store half %sum, ptr addrspace(1) %output, align 2
ret void
}
define amdgpu_kernel void @kernel_f16(ptr addrspace(1) %input, ptr addrspace(1) %scales, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @kernel_f16(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x half> [[TMP0]], splat (half 1.519530e+00)
; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x half> [[TMP2]], [[TMP1]]
; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x half> [[TMP3]], i64 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB0]])
; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x half> [[TMP3]], i64 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB1]])
; GCN-NEXT: [[LOG0:%.*]] = tail call half @llvm.amdgcn.log.f16(half [[EXP0]])
; GCN-NEXT: [[LOG1:%.*]] = tail call half @llvm.amdgcn.log.f16(half [[EXP1]])
; GCN-NEXT: [[SUM:%.*]] = fadd fast half [[LOG0]], [[LOG1]]
; GCN-NEXT: store half [[SUM]], ptr addrspace(1) [[OUTPUT]], align 2
; GCN-NEXT: ret void
;
entry:
%in0 = load half, ptr addrspace(1) %input, align 2
%ptr1 = getelementptr half, ptr addrspace(1) %input, i64 1
%in1 = load half, ptr addrspace(1) %ptr1, align 2
%scale0 = load half, ptr addrspace(1) %scales, align 2
%sptr1 = getelementptr half, ptr addrspace(1) %scales, i64 1
%scale1 = load half, ptr addrspace(1) %sptr1, align 2
%mul0 = fmul contract half %in0, 0xH3E14
%mul1 = fmul contract half %in1, 0xH3E14
%sub0 = fsub contract half %mul0, %scale0
%sub1 = fsub contract half %mul1, %scale1
%exp0 = tail call half @llvm.amdgcn.exp2.f16(half %sub0)
%exp1 = tail call half @llvm.amdgcn.exp2.f16(half %sub1)
%log0 = tail call half @llvm.amdgcn.log.f16(half %exp0)
%log1 = tail call half @llvm.amdgcn.log.f16(half %exp1)
%sum = fadd fast half %log0, %log1
store half %sum, ptr addrspace(1) %output, align 2
ret void
}
define amdgpu_kernel void @look_through_reuse_shuffle(
; GCN-LABEL: define amdgpu_kernel void @look_through_reuse_shuffle(
; GCN-SAME: ptr addrspace(1) noalias [[INPUT:%.*]], ptr addrspace(1) noalias [[SCALES:%.*]], ptr addrspace(1) noalias [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[TMP4:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
; GCN-NEXT: [[TMP5:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
; GCN-NEXT: [[TMP6:%.*]] = fadd contract <2 x half> [[TMP4]], splat (half 1.519530e+00)
; GCN-NEXT: [[TMP3:%.*]] = fmul contract <2 x half> [[TMP6]], [[TMP5]]
; GCN-NEXT: [[MUL0:%.*]] = extractelement <2 x half> [[TMP3]], i64 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[MUL0]])
; GCN-NEXT: [[MUL1:%.*]] = extractelement <2 x half> [[TMP3]], i64 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[MUL1]])
; GCN-NEXT: [[TMP7:%.*]] = insertelement <4 x half> poison, half [[EXP0]], i64 0
; GCN-NEXT: [[TMP1:%.*]] = insertelement <4 x half> [[TMP7]], half [[EXP1]], i64 1
; GCN-NEXT: [[TMP2:%.*]] = shufflevector <4 x half> [[TMP1]], <4 x half> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
; GCN-NEXT: store <4 x half> [[TMP2]], ptr addrspace(1) [[OUTPUT]], align 2
; GCN-NEXT: ret void
;
ptr addrspace(1) noalias %input, ptr addrspace(1) noalias %scales,
ptr addrspace(1) noalias %output) {
entry:
%iptr1 = getelementptr half, ptr addrspace(1) %input, i64 1
%sptr1 = getelementptr half, ptr addrspace(1) %scales, i64 1
%optr1 = getelementptr half, ptr addrspace(1) %output, i64 1
%optr2 = getelementptr half, ptr addrspace(1) %output, i64 2
%optr3 = getelementptr half, ptr addrspace(1) %output, i64 3
%in0 = load half, ptr addrspace(1) %input, align 2
%in1 = load half, ptr addrspace(1) %iptr1, align 2
%s0 = load half, ptr addrspace(1) %scales, align 2
%s1 = load half, ptr addrspace(1) %sptr1, align 2
%add0 = fadd contract half %in0, 0xH3E14
%add1 = fadd contract half %in1, 0xH3E14
%mul0 = fmul contract half %add0, %s0
%mul1 = fmul contract half %add1, %s1
%exp0 = tail call half @llvm.amdgcn.exp2.f16(half %mul0)
%exp1 = tail call half @llvm.amdgcn.exp2.f16(half %mul1)
store half %exp0, ptr addrspace(1) %output, align 2
store half %exp1, ptr addrspace(1) %optr1, align 2
store half %exp1, ptr addrspace(1) %optr2, align 2
store half %exp1, ptr addrspace(1) %optr3, align 2
ret void
}
define amdgpu_kernel void @wider_exp2_f32(ptr addrspace(1) %input, ptr addrspace(1) %scales, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @wider_exp2_f32(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[PTR2:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 2
; GCN-NEXT: [[SPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[SCALES]], i64 2
; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[INPUT]], align 4
; GCN-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[SCALES]], align 4
; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x float> [[TMP0]], splat (float f0x3E0293EE)
; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x float> [[TMP2]], [[TMP1]]
; GCN-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[PTR2]], align 4
; GCN-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[SPTR2]], align 4
; GCN-NEXT: [[TMP6:%.*]] = fmul contract <2 x float> [[TMP4]], splat (float f0x3E0293EE)
; GCN-NEXT: [[TMP7:%.*]] = fsub contract <2 x float> [[TMP6]], [[TMP5]]
; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB0]])
; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x float> [[TMP3]], i64 1
; GCN-NEXT: [[EXP1:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB1]])
; GCN-NEXT: [[SUB2:%.*]] = extractelement <2 x float> [[TMP7]], i64 0
; GCN-NEXT: [[EXP2:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB2]])
; GCN-NEXT: [[SUB3:%.*]] = extractelement <2 x float> [[TMP7]], i64 1
; GCN-NEXT: [[EXP3:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[SUB3]])
; GCN-NEXT: [[SUM01:%.*]] = fadd fast float [[EXP0]], [[EXP1]]
; GCN-NEXT: [[SUM23:%.*]] = fadd fast float [[EXP2]], [[EXP3]]
; GCN-NEXT: [[SUM:%.*]] = fadd fast float [[SUM01]], [[SUM23]]
; GCN-NEXT: store float [[SUM]], ptr addrspace(1) [[OUTPUT]], align 4
; GCN-NEXT: ret void
;
entry:
%in0 = load float, ptr addrspace(1) %input, align 4
%ptr1 = getelementptr float, ptr addrspace(1) %input, i64 1
%in1 = load float, ptr addrspace(1) %ptr1, align 4
%ptr2 = getelementptr float, ptr addrspace(1) %input, i64 2
%in2 = load float, ptr addrspace(1) %ptr2, align 4
%ptr3 = getelementptr float, ptr addrspace(1) %input, i64 3
%in3 = load float, ptr addrspace(1) %ptr3, align 4
%scale0 = load float, ptr addrspace(1) %scales, align 4
%sptr1 = getelementptr float, ptr addrspace(1) %scales, i64 1
%scale1 = load float, ptr addrspace(1) %sptr1, align 4
%sptr2 = getelementptr float, ptr addrspace(1) %scales, i64 2
%scale2 = load float, ptr addrspace(1) %sptr2, align 4
%sptr3 = getelementptr float, ptr addrspace(1) %scales, i64 3
%scale3 = load float, ptr addrspace(1) %sptr3, align 4
%mul0 = fmul contract float %in0, 0x3FC0527DC0000000
%mul1 = fmul contract float %in1, 0x3FC0527DC0000000
%mul2 = fmul contract float %in2, 0x3FC0527DC0000000
%mul3 = fmul contract float %in3, 0x3FC0527DC0000000
%sub0 = fsub contract float %mul0, %scale0
%sub1 = fsub contract float %mul1, %scale1
%sub2 = fsub contract float %mul2, %scale2
%sub3 = fsub contract float %mul3, %scale3
%exp0 = tail call float @llvm.amdgcn.exp2.f32(float %sub0)
%exp1 = tail call float @llvm.amdgcn.exp2.f32(float %sub1)
%exp2 = tail call float @llvm.amdgcn.exp2.f32(float %sub2)
%exp3 = tail call float @llvm.amdgcn.exp2.f32(float %sub3)
%sum01 = fadd fast float %exp0, %exp1
%sum23 = fadd fast float %exp2, %exp3
%sum = fadd fast float %sum01, %sum23
store float %sum, ptr addrspace(1) %output, align 4
ret void
}
define amdgpu_kernel void @wider_exp2_half(ptr addrspace(1) %input, ptr addrspace(1) %scales, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @wider_exp2_half(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[SCALES:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[PTR2:%.*]] = getelementptr half, ptr addrspace(1) [[INPUT]], i64 2
; GCN-NEXT: [[SPTR2:%.*]] = getelementptr half, ptr addrspace(1) [[SCALES]], i64 2
; GCN-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[INPUT]], align 2
; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[SCALES]], align 2
; GCN-NEXT: [[TMP2:%.*]] = fmul contract <2 x half> [[TMP0]], splat (half 1.519530e+00)
; GCN-NEXT: [[TMP3:%.*]] = fsub contract <2 x half> [[TMP2]], [[TMP1]]
; GCN-NEXT: [[TMP4:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR2]], align 2
; GCN-NEXT: [[TMP5:%.*]] = load <2 x half>, ptr addrspace(1) [[SPTR2]], align 2
; GCN-NEXT: [[TMP6:%.*]] = fmul contract <2 x half> [[TMP4]], splat (half 1.519530e+00)
; GCN-NEXT: [[TMP7:%.*]] = fsub contract <2 x half> [[TMP6]], [[TMP5]]
; GCN-NEXT: [[SUB0:%.*]] = extractelement <2 x half> [[TMP3]], i64 0
; GCN-NEXT: [[EXP0:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB0]])
; GCN-NEXT: [[SUB1:%.*]] = extractelement <2 x half> [[TMP3]], i64 1
; GCN-NEXT: [[EXP1:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB1]])
; GCN-NEXT: [[SUB2:%.*]] = extractelement <2 x half> [[TMP7]], i64 0
; GCN-NEXT: [[EXP2:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB2]])
; GCN-NEXT: [[SUB3:%.*]] = extractelement <2 x half> [[TMP7]], i64 1
; GCN-NEXT: [[EXP3:%.*]] = tail call half @llvm.amdgcn.exp2.f16(half [[SUB3]])
; GCN-NEXT: [[SUM01:%.*]] = fadd fast half [[EXP0]], [[EXP1]]
; GCN-NEXT: [[SUM23:%.*]] = fadd fast half [[EXP2]], [[EXP3]]
; GCN-NEXT: [[SUM:%.*]] = fadd fast half [[SUM01]], [[SUM23]]
; GCN-NEXT: store half [[SUM]], ptr addrspace(1) [[OUTPUT]], align 2
; GCN-NEXT: ret void
;
entry:
;; Same 4-wide pattern as wider_exp2_f32. Cost model gives half higher
;; LookThrough cost, so we get 2-wide vectorization (or none on stricter targets).
%in0 = load half, ptr addrspace(1) %input, align 2
%ptr1 = getelementptr half, ptr addrspace(1) %input, i64 1
%in1 = load half, ptr addrspace(1) %ptr1, align 2
%ptr2 = getelementptr half, ptr addrspace(1) %input, i64 2
%in2 = load half, ptr addrspace(1) %ptr2, align 2
%ptr3 = getelementptr half, ptr addrspace(1) %input, i64 3
%in3 = load half, ptr addrspace(1) %ptr3, align 2
%scale0 = load half, ptr addrspace(1) %scales, align 2
%sptr1 = getelementptr half, ptr addrspace(1) %scales, i64 1
%scale1 = load half, ptr addrspace(1) %sptr1, align 2
%sptr2 = getelementptr half, ptr addrspace(1) %scales, i64 2
%scale2 = load half, ptr addrspace(1) %sptr2, align 2
%sptr3 = getelementptr half, ptr addrspace(1) %scales, i64 3
%scale3 = load half, ptr addrspace(1) %sptr3, align 2
%mul0 = fmul contract half %in0, 0xH3E14
%mul1 = fmul contract half %in1, 0xH3E14
%mul2 = fmul contract half %in2, 0xH3E14
%mul3 = fmul contract half %in3, 0xH3E14
%sub0 = fsub contract half %mul0, %scale0
%sub1 = fsub contract half %mul1, %scale1
%sub2 = fsub contract half %mul2, %scale2
%sub3 = fsub contract half %mul3, %scale3
%exp0 = tail call half @llvm.amdgcn.exp2.f16(half %sub0)
%exp1 = tail call half @llvm.amdgcn.exp2.f16(half %sub1)
%exp2 = tail call half @llvm.amdgcn.exp2.f16(half %sub2)
%exp3 = tail call half @llvm.amdgcn.exp2.f16(half %sub3)
%sum01 = fadd fast half %exp0, %exp1
%sum23 = fadd fast half %exp2, %exp3
%sum = fadd fast half %sum01, %sum23
store half %sum, ptr addrspace(1) %output, align 2
ret void
}
define amdgpu_kernel void @kernel_div_scale(ptr addrspace(1) %num, ptr addrspace(1) %den, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @kernel_div_scale(
; GCN-SAME: ptr addrspace(1) [[NUM:%.*]], ptr addrspace(1) [[DEN:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[NPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[NUM]], i64 2
; GCN-NEXT: [[N2:%.*]] = load float, ptr addrspace(1) [[NPTR2]], align 4
; GCN-NEXT: [[DPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[DEN]], i64 2
; GCN-NEXT: [[D2:%.*]] = load float, ptr addrspace(1) [[DPTR2]], align 4
; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[NUM]], align 4
; GCN-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[TMP0]], splat (float 2.000000e+00)
; GCN-NEXT: [[MUL_N2:%.*]] = fmul float [[N2]], 2.000000e+00
; GCN-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(1) [[DEN]], align 4
; GCN-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], splat (float 4.000000e+00)
; GCN-NEXT: [[MUL_D2:%.*]] = fmul float [[D2]], 4.000000e+00
; GCN-NEXT: [[MUL_N0:%.*]] = extractelement <2 x float> [[TMP1]], i64 0
; GCN-NEXT: [[MUL_D0:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
; GCN-NEXT: [[DS0:%.*]] = call { float, i1 } @llvm.amdgcn.div.scale.f32(float [[MUL_N0]], float [[MUL_D0]], i1 false)
; GCN-NEXT: [[MUL_N1:%.*]] = extractelement <2 x float> [[TMP1]], i64 1
; GCN-NEXT: [[MUL_D1:%.*]] = extractelement <2 x float> [[TMP3]], i64 1
; GCN-NEXT: [[DS1:%.*]] = call { float, i1 } @llvm.amdgcn.div.scale.f32(float [[MUL_N1]], float [[MUL_D1]], i1 false)
; GCN-NEXT: [[DS2:%.*]] = call { float, i1 } @llvm.amdgcn.div.scale.f32(float [[MUL_N2]], float [[MUL_D2]], i1 false)
; GCN-NEXT: [[R0:%.*]] = extractvalue { float, i1 } [[DS0]], 0
; GCN-NEXT: [[R1:%.*]] = extractvalue { float, i1 } [[DS1]], 0
; GCN-NEXT: [[R2:%.*]] = extractvalue { float, i1 } [[DS2]], 0
; GCN-NEXT: [[SUM01:%.*]] = fadd float [[R0]], [[R1]]
; GCN-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[R2]]
; GCN-NEXT: store float [[SUM]], ptr addrspace(1) [[OUTPUT]], align 4
; GCN-NEXT: ret void
;
entry:
%n0 = load float, ptr addrspace(1) %num, align 4
%nptr1 = getelementptr float, ptr addrspace(1) %num, i64 1
%n1 = load float, ptr addrspace(1) %nptr1, align 4
%nptr2 = getelementptr float, ptr addrspace(1) %num, i64 2
%n2 = load float, ptr addrspace(1) %nptr2, align 4
%d0 = load float, ptr addrspace(1) %den, align 4
%dptr1 = getelementptr float, ptr addrspace(1) %den, i64 1
%d1 = load float, ptr addrspace(1) %dptr1, align 4
%dptr2 = getelementptr float, ptr addrspace(1) %den, i64 2
%d2 = load float, ptr addrspace(1) %dptr2, align 4
%mul_n0 = fmul float %n0, 2.0
%mul_n1 = fmul float %n1, 2.0
%mul_n2 = fmul float %n2, 2.0
%mul_d0 = fmul float %d0, 4.0
%mul_d1 = fmul float %d1, 4.0
%mul_d2 = fmul float %d2, 4.0
%ds0 = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %mul_n0, float %mul_d0, i1 false)
%ds1 = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %mul_n1, float %mul_d1, i1 false)
%ds2 = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %mul_n2, float %mul_d2, i1 false)
%r0 = extractvalue { float, i1 } %ds0, 0
%r1 = extractvalue { float, i1 } %ds1, 0
%r2 = extractvalue { float, i1 } %ds2, 0
%sum01 = fadd float %r0, %r1
%sum = fadd float %sum01, %r2
store float %sum, ptr addrspace(1) %output, align 4
ret void
}
define amdgpu_kernel void @kernel_fmed3(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @kernel_fmed3(
; GCN-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[APTR2:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 2
; GCN-NEXT: [[A2:%.*]] = load float, ptr addrspace(1) [[APTR2]], align 4
; GCN-NEXT: [[BPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 2
; GCN-NEXT: [[B2:%.*]] = load float, ptr addrspace(1) [[BPTR2]], align 4
; GCN-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[A]], align 4
; GCN-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[B]], align 4
; GCN-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP0]], [[TMP1]]
; GCN-NEXT: [[ADD2:%.*]] = fadd float [[A2]], [[B2]]
; GCN-NEXT: [[ADD0:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
; GCN-NEXT: [[MED0:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD0]], float [[ADD0]], float 1.000000e+00)
; GCN-NEXT: [[ADD1:%.*]] = extractelement <2 x float> [[TMP2]], i64 1
; GCN-NEXT: [[MED1:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD1]], float [[ADD1]], float 1.000000e+00)
; GCN-NEXT: [[MED2:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD2]], float [[ADD2]], float 1.000000e+00)
; GCN-NEXT: [[SUM01:%.*]] = fadd float [[MED0]], [[MED1]]
; GCN-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[MED2]]
; GCN-NEXT: store float [[SUM]], ptr addrspace(1) [[OUTPUT]], align 4
; GCN-NEXT: ret void
;
entry:
%a0 = load float, ptr addrspace(1) %a, align 4
%aptr1 = getelementptr float, ptr addrspace(1) %a, i64 1
%a1 = load float, ptr addrspace(1) %aptr1, align 4
%aptr2 = getelementptr float, ptr addrspace(1) %a, i64 2
%a2 = load float, ptr addrspace(1) %aptr2, align 4
%b0 = load float, ptr addrspace(1) %b, align 4
%bptr1 = getelementptr float, ptr addrspace(1) %b, i64 1
%b1 = load float, ptr addrspace(1) %bptr1, align 4
%bptr2 = getelementptr float, ptr addrspace(1) %b, i64 2
%b2 = load float, ptr addrspace(1) %bptr2, align 4
%add0 = fadd float %a0, %b0
%add1 = fadd float %a1, %b1
%add2 = fadd float %a2, %b2
%med0 = call float @llvm.amdgcn.fmed3.f32(float %add0, float %add0, float 1.0)
%med1 = call float @llvm.amdgcn.fmed3.f32(float %add1, float %add1, float 1.0)
%med2 = call float @llvm.amdgcn.fmed3.f32(float %add2, float %add2, float 1.0)
%sum01 = fadd float %med0, %med1
%sum = fadd float %sum01, %med2
store float %sum, ptr addrspace(1) %output, align 4
ret void
}
define amdgpu_kernel void @kernel_fmed3_1(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @kernel_fmed3_1(
; GCN-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[A0:%.*]] = load float, ptr addrspace(1) [[A]], align 4
; GCN-NEXT: [[APTR1:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 1
; GCN-NEXT: [[A1:%.*]] = load float, ptr addrspace(1) [[APTR1]], align 4
; GCN-NEXT: [[APTR2:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 2
; GCN-NEXT: [[A2:%.*]] = load float, ptr addrspace(1) [[APTR2]], align 4
; GCN-NEXT: [[APTR3:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 3
; GCN-NEXT: [[A3:%.*]] = load float, ptr addrspace(1) [[APTR3]], align 4
; GCN-NEXT: [[B0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
; GCN-NEXT: [[BPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 1
; GCN-NEXT: [[B1:%.*]] = load float, ptr addrspace(1) [[BPTR1]], align 4
; GCN-NEXT: [[BPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 2
; GCN-NEXT: [[B2:%.*]] = load float, ptr addrspace(1) [[BPTR2]], align 4
; GCN-NEXT: [[BPTR3:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 3
; GCN-NEXT: [[B3:%.*]] = load float, ptr addrspace(1) [[BPTR3]], align 4
; GCN-NEXT: [[ADD0:%.*]] = fadd float 5.000000e+00, [[B0]]
; GCN-NEXT: [[ADD1:%.*]] = fadd float 5.000000e+00, [[B1]]
; GCN-NEXT: [[ADD2:%.*]] = fadd float 5.000000e+00, [[B2]]
; GCN-NEXT: [[ADD3:%.*]] = fadd float 5.000000e+00, [[B3]]
; GCN-NEXT: [[SUB0:%.*]] = fadd float 1.000000e+00, [[B0]]
; GCN-NEXT: [[SUB1:%.*]] = fadd float 1.000000e+00, [[B1]]
; GCN-NEXT: [[SUB2:%.*]] = fadd float 1.000000e+00, [[B2]]
; GCN-NEXT: [[SUB3:%.*]] = fadd float 1.000000e+00, [[B3]]
; GCN-NEXT: [[MED0:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD0]], float [[SUB0]], float 1.000000e+00)
; GCN-NEXT: [[MED1:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD1]], float [[SUB1]], float 1.000000e+00)
; GCN-NEXT: [[MED2:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD2]], float [[SUB2]], float 1.000000e+00)
; GCN-NEXT: [[MED3:%.*]] = call float @llvm.amdgcn.fmed3.f32(float [[ADD3]], float [[SUB3]], float 1.000000e+00)
; GCN-NEXT: [[SUM01:%.*]] = fadd float [[MED0]], [[MED1]]
; GCN-NEXT: [[SUM02:%.*]] = fadd float [[MED2]], [[MED3]]
; GCN-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM02]]
; GCN-NEXT: store float [[SUM]], ptr addrspace(1) [[OUTPUT]], align 4
; GCN-NEXT: ret void
;
entry:
%a0 = load float, ptr addrspace(1) %a, align 4
%aptr1 = getelementptr float, ptr addrspace(1) %a, i64 1
%a1 = load float, ptr addrspace(1) %aptr1, align 4
%aptr2 = getelementptr float, ptr addrspace(1) %a, i64 2
%a2 = load float, ptr addrspace(1) %aptr2, align 4
%aptr3 = getelementptr float, ptr addrspace(1) %a, i64 3
%a3 = load float, ptr addrspace(1) %aptr3, align 4
%b0 = load float, ptr addrspace(1) %b, align 4
%bptr1 = getelementptr float, ptr addrspace(1) %b, i64 1
%b1 = load float, ptr addrspace(1) %bptr1, align 4
%bptr2 = getelementptr float, ptr addrspace(1) %b, i64 2
%b2 = load float, ptr addrspace(1) %bptr2, align 4
%bptr3 = getelementptr float, ptr addrspace(1) %b, i64 3
%b3 = load float, ptr addrspace(1) %bptr3, align 4
%add0 = fadd float 5.0, %b0
%add1 = fadd float 5.0, %b1
%add2 = fadd float 5.0, %b2
%add3 = fadd float 5.0, %b3
%sub0 = fadd float 1.0, %b0
%sub1 = fadd float 1.0, %b1
%sub2 = fadd float 1.0, %b2
%sub3 = fadd float 1.0, %b3
%med0 = call float @llvm.amdgcn.fmed3.f32(float %add0, float %sub0, float 1.0)
%med1 = call float @llvm.amdgcn.fmed3.f32(float %add1, float %sub1, float 1.0)
%med2 = call float @llvm.amdgcn.fmed3.f32(float %add2, float %sub2, float 1.0)
%med3 = call float @llvm.amdgcn.fmed3.f32(float %add3, float %sub3, float 1.0)
%sum01 = fadd float %med0, %med1
%sum02 = fadd float %med2, %med3
%sum = fadd float %sum01, %sum02
store float %sum, ptr addrspace(1) %output, align 4
ret void
}
define amdgpu_kernel void @test_single_exp_hreduction(
; GCN-LABEL: define amdgpu_kernel void @test_single_exp_hreduction(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[P0:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 0
; GCN-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[P0]], align 4
; GCN-NEXT: [[TMP1:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP0]])
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[TMP1]])
; GCN-NEXT: store float [[EXP0]], ptr addrspace(1) [[OUTPUT]], align 4
; GCN-NEXT: ret void
;
ptr addrspace(1) %input, ptr addrspace(1) %output) {
entry:
%p0 = getelementptr float, ptr addrspace(1) %input, i64 0
%p1 = getelementptr float, ptr addrspace(1) %input, i64 1
%p2 = getelementptr float, ptr addrspace(1) %input, i64 2
%p3 = getelementptr float, ptr addrspace(1) %input, i64 3
%a0 = load float, ptr addrspace(1) %p0, align 4
%a1 = load float, ptr addrspace(1) %p1, align 4
%a2 = load float, ptr addrspace(1) %p2, align 4
%a3 = load float, ptr addrspace(1) %p3, align 4
%add01 = fadd fast float %a0, %a1
%add23 = fadd fast float %a2, %a3
%sum = fadd fast float %add01, %add23
%exp0 = tail call float @llvm.amdgcn.exp2.f32(float %sum)
store float %exp0, ptr addrspace(1) %output, align 4
ret void
}
define amdgpu_kernel void @test_hreduction_into_exp(
; GCN-LABEL: define amdgpu_kernel void @test_hreduction_into_exp(
; GCN-SAME: ptr addrspace(1) [[INPUT:%.*]], ptr addrspace(1) [[OUTPUT:%.*]], <16 x i32> [[A:%.*]], <16 x i32> [[B:%.*]], i32 [[SCALE_IDX:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[P0:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 0
; GCN-NEXT: [[P4:%.*]] = getelementptr float, ptr addrspace(1) [[INPUT]], i64 4
; GCN-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[P0]], align 4
; GCN-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[P4]], align 4
; GCN-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP0]])
; GCN-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP1]])
; GCN-NEXT: [[EXP0:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[TMP2]])
; GCN-NEXT: [[EXP1:%.*]] = tail call float @llvm.amdgcn.exp2.f32(float [[TMP3]])
; GCN-NEXT: [[VEC0:%.*]] = insertelement <2 x float> poison, float [[EXP0]], i64 0
; GCN-NEXT: [[VEC1:%.*]] = insertelement <2 x float> [[VEC0]], float [[EXP1]], i64 1
; GCN-NEXT: [[VEC_I32:%.*]] = bitcast <2 x float> [[VEC1]] to <2 x i32>
; GCN-NEXT: [[SCALE0:%.*]] = extractelement <2 x i32> [[VEC_I32]], i64 0
; GCN-NEXT: [[SCALE1:%.*]] = extractelement <2 x i32> [[VEC_I32]], i64 1
; GCN-NEXT: [[WMMA0:%.*]] = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> [[A]], i32 0, <16 x i32> [[B]], i16 0, <8 x float> zeroinitializer, i32 0, i32 0, i32 [[SCALE0]], i32 0, i32 0, i32 [[SCALE_IDX]], i1 false, i1 false)
; GCN-NEXT: [[WMMA1:%.*]] = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 0, <16 x i32> [[A]], i32 0, <16 x i32> [[B]], i16 0, <8 x float> [[WMMA0]], i32 0, i32 0, i32 [[SCALE1]], i32 0, i32 0, i32 [[SCALE_IDX]], i1 false, i1 false)
; GCN-NEXT: store <8 x float> [[WMMA1]], ptr addrspace(1) [[OUTPUT]], align 32
; GCN-NEXT: ret void
;
ptr addrspace(1) %input, ptr addrspace(1) %output,
<16 x i32> %A, <16 x i32> %B, i32 %scale_idx) {
entry:
%p0 = getelementptr float, ptr addrspace(1) %input, i64 0
%p1 = getelementptr float, ptr addrspace(1) %input, i64 1
%p2 = getelementptr float, ptr addrspace(1) %input, i64 2
%p3 = getelementptr float, ptr addrspace(1) %input, i64 3
%p4 = getelementptr float, ptr addrspace(1) %input, i64 4
%p5 = getelementptr float, ptr addrspace(1) %input, i64 5
%p6 = getelementptr float, ptr addrspace(1) %input, i64 6
%p7 = getelementptr float, ptr addrspace(1) %input, i64 7
%a0 = load float, ptr addrspace(1) %p0, align 4
%a1 = load float, ptr addrspace(1) %p1, align 4
%a2 = load float, ptr addrspace(1) %p2, align 4
%a3 = load float, ptr addrspace(1) %p3, align 4
%b0 = load float, ptr addrspace(1) %p4, align 4
%b1 = load float, ptr addrspace(1) %p5, align 4
%b2 = load float, ptr addrspace(1) %p6, align 4
%b3 = load float, ptr addrspace(1) %p7, align 4
%add_a01 = fadd fast float %a0, %a1
%add_a23 = fadd fast float %a2, %a3
%sum0 = fadd fast float %add_a01, %add_a23
%add_b01 = fadd fast float %b0, %b1
%add_b23 = fadd fast float %b2, %b3
%sum1 = fadd fast float %add_b01, %add_b23
%exp0 = tail call float @llvm.amdgcn.exp2.f32(float %sum0)
%exp1 = tail call float @llvm.amdgcn.exp2.f32(float %sum1)
%vec0 = insertelement <2 x float> poison, float %exp0, i64 0
%vec1 = insertelement <2 x float> %vec0, float %exp1, i64 1
%vec_i32 = bitcast <2 x float> %vec1 to <2 x i32>
%scale0 = extractelement <2 x i32> %vec_i32, i64 0
%scale1 = extractelement <2 x i32> %vec_i32, i64 1
%wmma0 = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(
i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> zeroinitializer,
i32 0, i32 0, i32 %scale0, i32 0, i32 0, i32 %scale_idx, i1 false, i1 false)
%wmma1 = tail call <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(
i32 0, <16 x i32> %A, i32 0, <16 x i32> %B, i16 0, <8 x float> %wmma0,
i32 0, i32 0, i32 %scale1, i32 0, i32 0, i32 %scale_idx, i1 false, i1 false)
store <8 x float> %wmma1, ptr addrspace(1) %output, align 32
ret void
}
define amdgpu_kernel void @kernel_alternate(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %output) {
; GCN-LABEL: define amdgpu_kernel void @kernel_alternate(
; GCN-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[OUTPUT:%.*]]) {
; GCN-NEXT: [[ENTRY:.*:]]
; GCN-NEXT: [[A0:%.*]] = load float, ptr addrspace(1) [[A]], align 4
; GCN-NEXT: [[APTR1:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 1
; GCN-NEXT: [[A1:%.*]] = load float, ptr addrspace(1) [[APTR1]], align 4
; GCN-NEXT: [[APTR2:%.*]] = getelementptr float, ptr addrspace(1) [[A]], i64 2
; GCN-NEXT: [[A2:%.*]] = load float, ptr addrspace(1) [[APTR2]], align 4
; GCN-NEXT: [[B0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
; GCN-NEXT: [[BPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 1
; GCN-NEXT: [[B1:%.*]] = load float, ptr addrspace(1) [[BPTR1]], align 4
; GCN-NEXT: [[BPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[B]], i64 2
; GCN-NEXT: [[B2:%.*]] = load float, ptr addrspace(1) [[BPTR2]], align 4
; GCN-NEXT: [[ADD0:%.*]] = fadd float [[A0]], [[B0]]
; GCN-NEXT: [[SUB0:%.*]] = fsub float [[A0]], [[B0]]
; GCN-NEXT: [[ADD1:%.*]] = fadd float [[A1]], [[B1]]
; GCN-NEXT: [[SUB1:%.*]] = fsub float [[A1]], [[B1]]
; GCN-NEXT: [[ADD2:%.*]] = fadd float [[A2]], [[B2]]
; GCN-NEXT: [[SUB2:%.*]] = fsub float [[A2]], [[B2]]
; GCN-NEXT: [[E0:%.*]] = call float @llvm.amdgcn.exp2.f32(float [[ADD0]])
; GCN-NEXT: [[E1:%.*]] = call float @llvm.amdgcn.exp2.f32(float [[SUB0]])
; GCN-NEXT: [[E2:%.*]] = call float @llvm.amdgcn.log.f32(float [[ADD1]])
; GCN-NEXT: [[E3:%.*]] = call float @llvm.amdgcn.log.f32(float [[SUB1]])
; GCN-NEXT: [[E4:%.*]] = call float @llvm.amdgcn.exp2.f32(float [[ADD2]])
; GCN-NEXT: [[E5:%.*]] = call float @llvm.amdgcn.exp2.f32(float [[SUB2]])
; GCN-NEXT: [[OPTR0:%.*]] = getelementptr float, ptr addrspace(1) [[OUTPUT]], i64 0
; GCN-NEXT: [[OPTR1:%.*]] = getelementptr float, ptr addrspace(1) [[OUTPUT]], i64 1
; GCN-NEXT: [[OPTR2:%.*]] = getelementptr float, ptr addrspace(1) [[OUTPUT]], i64 2
; GCN-NEXT: [[OPTR3:%.*]] = getelementptr float, ptr addrspace(1) [[OUTPUT]], i64 3
; GCN-NEXT: [[OPTR4:%.*]] = getelementptr float, ptr addrspace(1) [[OUTPUT]], i64 4
; GCN-NEXT: [[OPTR5:%.*]] = getelementptr float, ptr addrspace(1) [[OUTPUT]], i64 5
; GCN-NEXT: store float [[E0]], ptr addrspace(1) [[OPTR0]], align 4
; GCN-NEXT: store float [[E1]], ptr addrspace(1) [[OPTR1]], align 4
; GCN-NEXT: store float [[E2]], ptr addrspace(1) [[OPTR2]], align 4
; GCN-NEXT: store float [[E3]], ptr addrspace(1) [[OPTR3]], align 4
; GCN-NEXT: store float [[E4]], ptr addrspace(1) [[OPTR4]], align 4
; GCN-NEXT: store float [[E5]], ptr addrspace(1) [[OPTR5]], align 4
; GCN-NEXT: ret void
;
entry:
%a0 = load float, ptr addrspace(1) %a, align 4
%aptr1 = getelementptr float, ptr addrspace(1) %a, i64 1
%a1 = load float, ptr addrspace(1) %aptr1, align 4
%aptr2 = getelementptr float, ptr addrspace(1) %a, i64 2
%a2 = load float, ptr addrspace(1) %aptr2, align 4
%b0 = load float, ptr addrspace(1) %b, align 4
%bptr1 = getelementptr float, ptr addrspace(1) %b, i64 1
%b1 = load float, ptr addrspace(1) %bptr1, align 4
%bptr2 = getelementptr float, ptr addrspace(1) %b, i64 2
%b2 = load float, ptr addrspace(1) %bptr2, align 4
%add0 = fadd float %a0, %b0
%sub0 = fsub float %a0, %b0
%add1 = fadd float %a1, %b1
%sub1 = fsub float %a1, %b1
%add2 = fadd float %a2, %b2
%sub2 = fsub float %a2, %b2
%e0 = call float @llvm.amdgcn.exp2.f32(float %add0)
%e1 = call float @llvm.amdgcn.exp2.f32(float %sub0)
%e2 = call float @llvm.amdgcn.log.f32(float %add1)
%e3 = call float @llvm.amdgcn.log.f32(float %sub1)
%e4 = call float @llvm.amdgcn.exp2.f32(float %add2)
%e5 = call float @llvm.amdgcn.exp2.f32(float %sub2)
%optr0 = getelementptr float, ptr addrspace(1) %output, i64 0
%optr1 = getelementptr float, ptr addrspace(1) %output, i64 1
%optr2 = getelementptr float, ptr addrspace(1) %output, i64 2
%optr3 = getelementptr float, ptr addrspace(1) %output, i64 3
%optr4 = getelementptr float, ptr addrspace(1) %output, i64 4
%optr5 = getelementptr float, ptr addrspace(1) %output, i64 5
store float %e0, ptr addrspace(1) %optr0, align 4
store float %e1, ptr addrspace(1) %optr1, align 4
store float %e2, ptr addrspace(1) %optr2, align 4
store float %e3, ptr addrspace(1) %optr3, align 4
store float %e4, ptr addrspace(1) %optr4, align 4
store float %e5, ptr addrspace(1) %optr5, align 4
ret void
}
declare float @llvm.amdgcn.fmed3.f32(float, float, float)
declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1)
declare half @llvm.amdgcn.exp2.f16(half)
declare float @llvm.amdgcn.exp2.f32(float)
declare <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 immarg, <16 x i32>, i32 immarg, <16 x i32>, i16 immarg, <8 x float>, i32 immarg, i32 immarg, i32, i32 immarg, i32 immarg, i32, i1 immarg, i1 immarg)