test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu9.50-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX9
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX1030

; Eight consecutive i8 weights are dequantized against a broadcast zero
; point and multiplied into an ordered fadd chain. The scalar byte loads
; already coalesce into one 64 bit load, so the byte loads and their
; conversions stay scalar. The products stay scalar as well and fuse into
; the fadd chain. The gfx9 targets subtract the zero point from four lanes
; as a vector, gfx1030 keeps everything scalar.
define float @dequant8_ordered(ptr addrspace(1) %mat, ptr addrspace(1) %vec, half %zph, float %acc) {
; CHECK-LABEL: define float @dequant8_ordered(
; CHECK-SAME: ptr addrspace(1) [[MAT:%.*]], ptr addrspace(1) [[VEC:%.*]], half [[ZPH:%.*]], float [[ACC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[ZP:%.*]] = fpext half [[ZPH]] to float
; CHECK-NEXT:    br label %[[BODY:.*]]
; CHECK:       [[BODY]]:
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x half>, ptr addrspace(1) [[VEC]], align 16
; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr addrspace(1) [[MAT]], align 2
; CHECK-NEXT:    [[TMP2:%.*]] = fpext <8 x half> [[TMP0]] to <8 x float>
; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x float> poison, float [[ZP]], i64 0
; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> poison, <8 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP6:%.*]] = fsub contract <8 x float> [[TMP3]], [[TMP5]]
; CHECK-NEXT:    [[TMP7:%.*]] = fmul contract <8 x float> [[TMP6]], [[TMP2]]
; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP7]], i64 0
; CHECK-NEXT:    [[A0:%.*]] = fadd contract float [[ACC]], [[TMP8]]
; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[TMP7]], i64 1
; CHECK-NEXT:    [[A1:%.*]] = fadd contract float [[A0]], [[TMP9]]
; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x float> [[TMP7]], i64 2
; CHECK-NEXT:    [[A2:%.*]] = fadd contract float [[A1]], [[TMP10]]
; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x float> [[TMP7]], i64 3
; CHECK-NEXT:    [[A3:%.*]] = fadd contract float [[A2]], [[TMP11]]
; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x float> [[TMP7]], i64 4
; CHECK-NEXT:    [[A4:%.*]] = fadd contract float [[A3]], [[TMP12]]
; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x float> [[TMP7]], i64 5
; CHECK-NEXT:    [[A5:%.*]] = fadd contract float [[A4]], [[TMP13]]
; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x float> [[TMP7]], i64 6
; CHECK-NEXT:    [[A6:%.*]] = fadd contract float [[A5]], [[TMP14]]
; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x float> [[TMP7]], i64 7
; CHECK-NEXT:    [[A7:%.*]] = fadd contract float [[A6]], [[TMP15]]
; CHECK-NEXT:    ret float [[A7]]
;
entry:
  %zp = fpext half %zph to float
  br label %body

body:
  %v0 = load half, ptr addrspace(1) %vec, align 16
  %vp1 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 2
  %v1 = load half, ptr addrspace(1) %vp1, align 2
  %vp2 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 4
  %v2 = load half, ptr addrspace(1) %vp2, align 4
  %vp3 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 6
  %v3 = load half, ptr addrspace(1) %vp3, align 2
  %vp4 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 8
  %v4 = load half, ptr addrspace(1) %vp4, align 8
  %vp5 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 10
  %v5 = load half, ptr addrspace(1) %vp5, align 2
  %vp6 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 12
  %v6 = load half, ptr addrspace(1) %vp6, align 4
  %vp7 = getelementptr inbounds i8, ptr addrspace(1) %vec, i64 14
  %v7 = load half, ptr addrspace(1) %vp7, align 2
  %q0 = load i8, ptr addrspace(1) %mat, align 2
  %mp1 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 1
  %q1 = load i8, ptr addrspace(1) %mp1, align 1
  %mp2 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 2
  %q2 = load i8, ptr addrspace(1) %mp2, align 2
  %mp3 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 3
  %q3 = load i8, ptr addrspace(1) %mp3, align 1
  %mp4 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 4
  %q4 = load i8, ptr addrspace(1) %mp4, align 2
  %mp5 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 5
  %q5 = load i8, ptr addrspace(1) %mp5, align 1
  %mp6 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 6
  %q6 = load i8, ptr addrspace(1) %mp6, align 2
  %mp7 = getelementptr inbounds i8, ptr addrspace(1) %mat, i64 7
  %q7 = load i8, ptr addrspace(1) %mp7, align 1
  %x0 = fpext half %v0 to float
  %f0 = sitofp i8 %q0 to float
  %d0 = fsub contract float %f0, %zp
  %m0 = fmul contract float %d0, %x0
  %a0 = fadd contract float %acc, %m0
  %x1 = fpext half %v1 to float
  %f1 = sitofp i8 %q1 to float
  %d1 = fsub contract float %f1, %zp
  %m1 = fmul contract float %d1, %x1
  %a1 = fadd contract float %a0, %m1
  %x2 = fpext half %v2 to float
  %f2 = sitofp i8 %q2 to float
  %d2 = fsub contract float %f2, %zp
  %m2 = fmul contract float %d2, %x2
  %a2 = fadd contract float %a1, %m2
  %x3 = fpext half %v3 to float
  %f3 = sitofp i8 %q3 to float
  %d3 = fsub contract float %f3, %zp
  %m3 = fmul contract float %d3, %x3
  %a3 = fadd contract float %a2, %m3
  %x4 = fpext half %v4 to float
  %f4 = sitofp i8 %q4 to float
  %d4 = fsub contract float %f4, %zp
  %m4 = fmul contract float %d4, %x4
  %a4 = fadd contract float %a3, %m4
  %x5 = fpext half %v5 to float
  %f5 = sitofp i8 %q5 to float
  %d5 = fsub contract float %f5, %zp
  %m5 = fmul contract float %d5, %x5
  %a5 = fadd contract float %a4, %m5
  %x6 = fpext half %v6 to float
  %f6 = sitofp i8 %q6 to float
  %d6 = fsub contract float %f6, %zp
  %m6 = fmul contract float %d6, %x6
  %a6 = fadd contract float %a5, %m6
  %x7 = fpext half %v7 to float
  %f7 = sitofp i8 %q7 to float
  %d7 = fsub contract float %f7, %zp
  %m7 = fmul contract float %d7, %x7
  %a7 = fadd contract float %a6, %m7
  ret float %a7
}

; A store chain over four converted bytes. The byte loads coalesce, so the
; loads and the conversions stay scalar. gfx90a stores the converted values
; with one vector store, gfx942 and gfx950 with two, and gfx1030 keeps the
; scalar stores.
define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX9-LABEL: define void @store4_sitofp(
; GFX9-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX9-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 4
; GFX9-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
; GFX9-NEXT:    store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
; GFX9-NEXT:    ret void
;
; GFX1030-LABEL: define void @store4_sitofp(
; GFX1030-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX1030-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1
; GFX1030-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2
; GFX1030-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3
; GFX1030-NEXT:    [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 4
; GFX1030-NEXT:    [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1
; GFX1030-NEXT:    [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 2
; GFX1030-NEXT:    [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1
; GFX1030-NEXT:    [[F0:%.*]] = sitofp i8 [[Q0]] to float
; GFX1030-NEXT:    [[F1:%.*]] = sitofp i8 [[Q1]] to float
; GFX1030-NEXT:    [[F2:%.*]] = sitofp i8 [[Q2]] to float
; GFX1030-NEXT:    [[F3:%.*]] = sitofp i8 [[Q3]] to float
; GFX1030-NEXT:    [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; GFX1030-NEXT:    [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
; GFX1030-NEXT:    [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
; GFX1030-NEXT:    store float [[F0]], ptr addrspace(1) [[OUT]], align 16
; GFX1030-NEXT:    store float [[F1]], ptr addrspace(1) [[O1]], align 4
; GFX1030-NEXT:    store float [[F2]], ptr addrspace(1) [[O2]], align 8
; GFX1030-NEXT:    store float [[F3]], ptr addrspace(1) [[O3]], align 4
; GFX1030-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
  %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
  %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
  %q0 = load i8, ptr addrspace(1) %in, align 4
  %q1 = load i8, ptr addrspace(1) %p1, align 1
  %q2 = load i8, ptr addrspace(1) %p2, align 2
  %q3 = load i8, ptr addrspace(1) %p3, align 1
  %f0 = sitofp i8 %q0 to float
  %f1 = sitofp i8 %q1 to float
  %f2 = sitofp i8 %q2 to float
  %f3 = sitofp i8 %q3 to float
  %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
  %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
  store float %f0, ptr addrspace(1) %out, align 16
  store float %f1, ptr addrspace(1) %o1, align 4
  store float %f2, ptr addrspace(1) %o2, align 8
  store float %f3, ptr addrspace(1) %o3, align 4
  ret void
}

; The same store chain with every byte load aligned to one byte. Global loads
; coalesce at any alignment, so the loads and the conversions stay scalar.
define void @store4_sitofp_global_a1(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX9-LABEL: define void @store4_sitofp_global_a1(
; GFX9-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX9-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 1
; GFX9-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
; GFX9-NEXT:    store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
; GFX9-NEXT:    ret void
;
; GFX1030-LABEL: define void @store4_sitofp_global_a1(
; GFX1030-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX1030-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 1
; GFX1030-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 2
; GFX1030-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 3
; GFX1030-NEXT:    [[Q0:%.*]] = load i8, ptr addrspace(1) [[IN]], align 1
; GFX1030-NEXT:    [[Q1:%.*]] = load i8, ptr addrspace(1) [[P1]], align 1
; GFX1030-NEXT:    [[Q2:%.*]] = load i8, ptr addrspace(1) [[P2]], align 1
; GFX1030-NEXT:    [[Q3:%.*]] = load i8, ptr addrspace(1) [[P3]], align 1
; GFX1030-NEXT:    [[F0:%.*]] = sitofp i8 [[Q0]] to float
; GFX1030-NEXT:    [[F1:%.*]] = sitofp i8 [[Q1]] to float
; GFX1030-NEXT:    [[F2:%.*]] = sitofp i8 [[Q2]] to float
; GFX1030-NEXT:    [[F3:%.*]] = sitofp i8 [[Q3]] to float
; GFX1030-NEXT:    [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; GFX1030-NEXT:    [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
; GFX1030-NEXT:    [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
; GFX1030-NEXT:    store float [[F0]], ptr addrspace(1) [[OUT]], align 16
; GFX1030-NEXT:    store float [[F1]], ptr addrspace(1) [[O1]], align 4
; GFX1030-NEXT:    store float [[F2]], ptr addrspace(1) [[O2]], align 8
; GFX1030-NEXT:    store float [[F3]], ptr addrspace(1) [[O3]], align 4
; GFX1030-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
  %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
  %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
  %q0 = load i8, ptr addrspace(1) %in, align 1
  %q1 = load i8, ptr addrspace(1) %p1, align 1
  %q2 = load i8, ptr addrspace(1) %p2, align 1
  %q3 = load i8, ptr addrspace(1) %p3, align 1
  %f0 = sitofp i8 %q0 to float
  %f1 = sitofp i8 %q1 to float
  %f2 = sitofp i8 %q2 to float
  %f3 = sitofp i8 %q3 to float
  %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
  %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
  store float %f0, ptr addrspace(1) %out, align 16
  store float %f1, ptr addrspace(1) %o1, align 4
  store float %f2, ptr addrspace(1) %o2, align 8
  store float %f3, ptr addrspace(1) %o3, align 4
  ret void
}

; The same store chain over LDS with the first byte load aligned to four. The
; byte loads coalesce into one dword load, so the loads and the conversions
; stay scalar.
define void @store4_sitofp_lds_a4(ptr addrspace(3) %in, ptr addrspace(1) %out) {
; GFX9-LABEL: define void @store4_sitofp_lds_a4(
; GFX9-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX9-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(3) [[IN]], align 4
; GFX9-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
; GFX9-NEXT:    store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
; GFX9-NEXT:    ret void
;
; GFX1030-LABEL: define void @store4_sitofp_lds_a4(
; GFX1030-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX1030-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 1
; GFX1030-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 2
; GFX1030-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 3
; GFX1030-NEXT:    [[Q0:%.*]] = load i8, ptr addrspace(3) [[IN]], align 4
; GFX1030-NEXT:    [[Q1:%.*]] = load i8, ptr addrspace(3) [[P1]], align 1
; GFX1030-NEXT:    [[Q2:%.*]] = load i8, ptr addrspace(3) [[P2]], align 2
; GFX1030-NEXT:    [[Q3:%.*]] = load i8, ptr addrspace(3) [[P3]], align 1
; GFX1030-NEXT:    [[F0:%.*]] = sitofp i8 [[Q0]] to float
; GFX1030-NEXT:    [[F1:%.*]] = sitofp i8 [[Q1]] to float
; GFX1030-NEXT:    [[F2:%.*]] = sitofp i8 [[Q2]] to float
; GFX1030-NEXT:    [[F3:%.*]] = sitofp i8 [[Q3]] to float
; GFX1030-NEXT:    [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; GFX1030-NEXT:    [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
; GFX1030-NEXT:    [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
; GFX1030-NEXT:    store float [[F0]], ptr addrspace(1) [[OUT]], align 16
; GFX1030-NEXT:    store float [[F1]], ptr addrspace(1) [[O1]], align 4
; GFX1030-NEXT:    store float [[F2]], ptr addrspace(1) [[O2]], align 8
; GFX1030-NEXT:    store float [[F3]], ptr addrspace(1) [[O3]], align 4
; GFX1030-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 1
  %p2 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 2
  %p3 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 3
  %q0 = load i8, ptr addrspace(3) %in, align 4
  %q1 = load i8, ptr addrspace(3) %p1, align 1
  %q2 = load i8, ptr addrspace(3) %p2, align 2
  %q3 = load i8, ptr addrspace(3) %p3, align 1
  %f0 = sitofp i8 %q0 to float
  %f1 = sitofp i8 %q1 to float
  %f2 = sitofp i8 %q2 to float
  %f3 = sitofp i8 %q3 to float
  %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
  %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
  store float %f0, ptr addrspace(1) %out, align 16
  store float %f1, ptr addrspace(1) %o1, align 4
  store float %f2, ptr addrspace(1) %o2, align 8
  store float %f3, ptr addrspace(1) %o3, align 4
  ret void
}

; The same store chain over LDS with every byte load aligned to one byte. The
; backend does not merge these byte loads, so the vector load keeps its saving.
; The gfx9 targets load, convert and store as vectors, gfx1030 keeps the scalar
; form.
define void @store4_sitofp_lds_a1(ptr addrspace(3) %in, ptr addrspace(1) %out) {
; GFX9-LABEL: define void @store4_sitofp_lds_a1(
; GFX9-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX9-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(3) [[IN]], align 1
; GFX9-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
; GFX9-NEXT:    store <4 x float> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
; GFX9-NEXT:    ret void
;
; GFX1030-LABEL: define void @store4_sitofp_lds_a1(
; GFX1030-SAME: ptr addrspace(3) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; GFX1030-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 1
; GFX1030-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 2
; GFX1030-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[IN]], i32 3
; GFX1030-NEXT:    [[Q0:%.*]] = load i8, ptr addrspace(3) [[IN]], align 1
; GFX1030-NEXT:    [[Q1:%.*]] = load i8, ptr addrspace(3) [[P1]], align 1
; GFX1030-NEXT:    [[Q2:%.*]] = load i8, ptr addrspace(3) [[P2]], align 1
; GFX1030-NEXT:    [[Q3:%.*]] = load i8, ptr addrspace(3) [[P3]], align 1
; GFX1030-NEXT:    [[F0:%.*]] = sitofp i8 [[Q0]] to float
; GFX1030-NEXT:    [[F1:%.*]] = sitofp i8 [[Q1]] to float
; GFX1030-NEXT:    [[F2:%.*]] = sitofp i8 [[Q2]] to float
; GFX1030-NEXT:    [[F3:%.*]] = sitofp i8 [[Q3]] to float
; GFX1030-NEXT:    [[O1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; GFX1030-NEXT:    [[O2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 2
; GFX1030-NEXT:    [[O3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 3
; GFX1030-NEXT:    store float [[F0]], ptr addrspace(1) [[OUT]], align 16
; GFX1030-NEXT:    store float [[F1]], ptr addrspace(1) [[O1]], align 4
; GFX1030-NEXT:    store float [[F2]], ptr addrspace(1) [[O2]], align 8
; GFX1030-NEXT:    store float [[F3]], ptr addrspace(1) [[O3]], align 4
; GFX1030-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 1
  %p2 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 2
  %p3 = getelementptr inbounds i8, ptr addrspace(3) %in, i32 3
  %q0 = load i8, ptr addrspace(3) %in, align 1
  %q1 = load i8, ptr addrspace(3) %p1, align 1
  %q2 = load i8, ptr addrspace(3) %p2, align 1
  %q3 = load i8, ptr addrspace(3) %p3, align 1
  %f0 = sitofp i8 %q0 to float
  %f1 = sitofp i8 %q1 to float
  %f2 = sitofp i8 %q2 to float
  %f3 = sitofp i8 %q3 to float
  %o1 = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  %o2 = getelementptr inbounds float, ptr addrspace(1) %out, i64 2
  %o3 = getelementptr inbounds float, ptr addrspace(1) %out, i64 3
  store float %f0, ptr addrspace(1) %out, align 16
  store float %f1, ptr addrspace(1) %o1, align 4
  store float %f2, ptr addrspace(1) %o2, align 8
  store float %f3, ptr addrspace(1) %o3, align 4
  ret void
}

; Four zero extended bytes are inserted into a vector that is stored, inside a
; loop of 16 iterations. The byte loads coalesce, and the load saving that never
; materializes is cancelled in every iteration, so the loop body stays scalar.
define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
; CHECK-LABEL: define void @zext4_bv_loop(
; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT:    [[OFF:%.*]] = shl i64 [[I]], 2
; CHECK-NEXT:    [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[OFF]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(1) [[P0]], align 4
; CHECK-NEXT:    [[V3:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
; CHECK-NEXT:    [[PO:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[OUT]], i64 [[I]]
; CHECK-NEXT:    store <4 x i32> [[V3]], ptr addrspace(1) [[PO]], align 16
; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[I_NEXT]], 16
; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %loop
loop:
  %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
  %off = shl i64 %i, 2
  %p0 = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %off
  %p1 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 1
  %p2 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 2
  %p3 = getelementptr inbounds i8, ptr addrspace(1) %p0, i64 3
  %l0 = load i8, ptr addrspace(1) %p0, align 4
  %l1 = load i8, ptr addrspace(1) %p1, align 1
  %l2 = load i8, ptr addrspace(1) %p2, align 2
  %l3 = load i8, ptr addrspace(1) %p3, align 1
  %x0 = zext i8 %l0 to i32
  %x1 = zext i8 %l1 to i32
  %x2 = zext i8 %l2 to i32
  %x3 = zext i8 %l3 to i32
  %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
  %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
  %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
  %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
  %po = getelementptr inbounds <4 x i32>, ptr addrspace(1) %out, i64 %i
  store <4 x i32> %v3, ptr addrspace(1) %po, align 16
  %i.next = add nuw nsw i64 %i, 1
  %c = icmp ult i64 %i.next, 16
  br i1 %c, label %loop, label %exit
exit:
  ret void
}

; Eight bytes are converted to half and stored. Two half lanes fit one
; register, so the byte loads keep their saving and the loads, the conversions
; and the stores stay vectorized.
define void @store8_sitofp_half(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; CHECK-LABEL: define void @store8_sitofp_half(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i64 4
; CHECK-NEXT:    [[O4:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[OUT]], i64 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN]], align 8
; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x half>
; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i8>, ptr addrspace(1) [[P4]], align 4
; CHECK-NEXT:    [[TMP4:%.*]] = sitofp <4 x i8> [[TMP3]] to <4 x half>
; CHECK-NEXT:    store <4 x half> [[TMP2]], ptr addrspace(1) [[OUT]], align 16
; CHECK-NEXT:    store <4 x half> [[TMP4]], ptr addrspace(1) [[O4]], align 8
; CHECK-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 1
  %p2 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 2
  %p3 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 3
  %p4 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 4
  %p5 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 5
  %p6 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 6
  %p7 = getelementptr inbounds i8, ptr addrspace(1) %in, i64 7
  %q0 = load i8, ptr addrspace(1) %in, align 8
  %q1 = load i8, ptr addrspace(1) %p1, align 1
  %q2 = load i8, ptr addrspace(1) %p2, align 2
  %q3 = load i8, ptr addrspace(1) %p3, align 1
  %q4 = load i8, ptr addrspace(1) %p4, align 4
  %q5 = load i8, ptr addrspace(1) %p5, align 1
  %q6 = load i8, ptr addrspace(1) %p6, align 2
  %q7 = load i8, ptr addrspace(1) %p7, align 1
  %f0 = sitofp i8 %q0 to half
  %f1 = sitofp i8 %q1 to half
  %f2 = sitofp i8 %q2 to half
  %f3 = sitofp i8 %q3 to half
  %f4 = sitofp i8 %q4 to half
  %f5 = sitofp i8 %q5 to half
  %f6 = sitofp i8 %q6 to half
  %f7 = sitofp i8 %q7 to half
  %o1 = getelementptr inbounds half, ptr addrspace(1) %out, i64 1
  %o2 = getelementptr inbounds half, ptr addrspace(1) %out, i64 2
  %o3 = getelementptr inbounds half, ptr addrspace(1) %out, i64 3
  %o4 = getelementptr inbounds half, ptr addrspace(1) %out, i64 4
  %o5 = getelementptr inbounds half, ptr addrspace(1) %out, i64 5
  %o6 = getelementptr inbounds half, ptr addrspace(1) %out, i64 6
  %o7 = getelementptr inbounds half, ptr addrspace(1) %out, i64 7
  store half %f0, ptr addrspace(1) %out, align 16
  store half %f1, ptr addrspace(1) %o1, align 2
  store half %f2, ptr addrspace(1) %o2, align 4
  store half %f3, ptr addrspace(1) %o3, align 2
  store half %f4, ptr addrspace(1) %o4, align 8
  store half %f5, ptr addrspace(1) %o5, align 2
  store half %f6, ptr addrspace(1) %o6, align 4
  store half %f7, ptr addrspace(1) %o7, align 2
  ret void
}