llvm/test/Transforms/SandboxVectorizer/load_store_vec.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection(enable-diff-types)<load-store-vec>" %s -S | FileCheck %s

define void @load_store_vec_basic(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_basic(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT:    store <4 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META0]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ptr2 = getelementptr i8, ptr %ptr, i32 2
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i8, ptr %ptr1
  %ld2 = load i16, ptr %ptr2
  store i8 %ld0, ptr %ptr0
  store i8 %ld1, ptr %ptr1
  store i16 %ld2, ptr %ptr2
  ret void
}

define void @load_store_vec_non_pow2(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_non_pow2(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <3 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT:    store <3 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META1]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i16, ptr %ptr1
  store i8 %ld0, ptr %ptr0
  store i16 %ld1, ptr %ptr1
  ret void
}

define void @load_store_vec_vectorize_vectors(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_vectorize_vectors(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <4 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META2:![0-9]+]]
; CHECK-NEXT:    store <4 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META2]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 2
  %ld0 = load <2 x i8>, ptr %ptr0
  %ld1 = load <2 x i8>, ptr %ptr1
  store <2 x i8> %ld0, ptr %ptr0
  store <2 x i8> %ld1, ptr %ptr1
  ret void
}

define void @load_store_vec_vectorize_vectors_diff_types(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_vectorize_vectors_diff_types(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <8 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META3:![0-9]+]]
; CHECK-NEXT:    store <8 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META3]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 2
  %ptr2 = getelementptr i8, ptr %ptr, i32 4
  %ld0 = load i16, ptr %ptr0
  %ld1 = load <2 x i8>, ptr %ptr1
  %ld2 = load <2 x i16>, ptr %ptr2
  store i16 %ld0, ptr %ptr0
  store <2 x i8> %ld1, ptr %ptr1
  store <2 x i16> %ld2, ptr %ptr2
  ret void
}

; Don't vectorize if there is a gap.
define void @load_store_vec_gap(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_gap(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 2
; CHECK-NEXT:    [[LD0:%.*]] = load i8, ptr [[PTR0]], align 1
; CHECK-NEXT:    [[LD1:%.*]] = load i16, ptr [[PTR1]], align 2
; CHECK-NEXT:    store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META4:![0-9]+]]
; CHECK-NEXT:    store i16 [[LD1]], ptr [[PTR1]], align 2, !sandboxvec [[META4]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 2
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i16, ptr %ptr1
  store i8 %ld0, ptr %ptr0
  store i16 %ld1, ptr %ptr1
  ret void
}

define void @load_store_vec_loads_not_consecutive(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_loads_not_consecutive(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 2
; CHECK-NEXT:    [[LD0:%.*]] = load i8, ptr [[PTR1]], align 1
; CHECK-NEXT:    [[LD1:%.*]] = load i16, ptr [[PTR0]], align 2
; CHECK-NEXT:    store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META5:![0-9]+]]
; CHECK-NEXT:    store i16 [[LD1]], ptr [[PTR1]], align 2, !sandboxvec [[META5]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 2
  %ld0 = load i8, ptr %ptr1
  %ld1 = load i16, ptr %ptr0
  store i8 %ld0, ptr %ptr0
  store i16 %ld1, ptr %ptr1
  ret void
}

; Vectorize same types, even if bottom-up-vec could do so too.
define void @load_store_vec_same_types(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_same_types(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META6:![0-9]+]]
; CHECK-NEXT:    store <2 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META6]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i8, ptr %ptr1
  store i8 %ld0, ptr %ptr0
  store i8 %ld1, ptr %ptr1
  ret void
}

; Don't vectorize mixed integers/floats.
define void @load_store_vec_mixed_int_float(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_mixed_int_float(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i32, ptr [[PTR]], i32 1
; CHECK-NEXT:    [[LD0:%.*]] = load i32, ptr [[PTR0]], align 4
; CHECK-NEXT:    [[LD1:%.*]] = load float, ptr [[PTR1]], align 4
; CHECK-NEXT:    store i32 [[LD0]], ptr [[PTR0]], align 4, !sandboxvec [[META7:![0-9]+]]
; CHECK-NEXT:    store float [[LD1]], ptr [[PTR1]], align 4, !sandboxvec [[META7]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i32, ptr %ptr, i32 0
  %ptr1 = getelementptr i32, ptr %ptr, i32 1
  %ld0 = load i32, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  store i32 %ld0, ptr %ptr0
  store float %ld1, ptr %ptr1
  ret void
}

define void @load_store_vec_mixed_int_float_vectors(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_mixed_int_float_vectors(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i32, ptr [[PTR]], i32 1
; CHECK-NEXT:    [[LD0:%.*]] = load i32, ptr [[PTR0]], align 4
; CHECK-NEXT:    [[LD1:%.*]] = load <2 x float>, ptr [[PTR1]], align 8
; CHECK-NEXT:    store i32 [[LD0]], ptr [[PTR0]], align 4, !sandboxvec [[META8:![0-9]+]]
; CHECK-NEXT:    store <2 x float> [[LD1]], ptr [[PTR1]], align 8, !sandboxvec [[META8]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i32, ptr %ptr, i32 0
  %ptr1 = getelementptr i32, ptr %ptr, i32 1
  %ld0 = load i32, ptr %ptr0
  %ld1 = load <2 x float>, ptr %ptr1
  store i32 %ld0, ptr %ptr0
  store <2 x float> %ld1, ptr %ptr1
  ret void
}

; Don't cross BBs (for now).
define ptr @load_store_vec_dont_cross_bbs(ptr %ptr) {
; CHECK-LABEL: define ptr @load_store_vec_dont_cross_bbs(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 1
; CHECK-NEXT:    [[LD0:%.*]] = load i8, ptr [[PTR0]], align 1
; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR1]], align 1
; CHECK-NEXT:    br label %[[BB:.*]]
; CHECK:       [[BB]]:
; CHECK-NEXT:    store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META9:![0-9]+]]
; CHECK-NEXT:    store i8 [[LD1]], ptr [[PTR1]], align 1, !sandboxvec [[META9]]
; CHECK-NEXT:    ret ptr [[PTR1]]
;
entry:
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i8, ptr %ptr1
  br label %bb

bb:
  store i8 %ld0, ptr %ptr0
  store i8 %ld1, ptr %ptr1
  ret ptr %ptr1
}

; Check that all dead GEPs are removed.
define void @load_store_vec_cleanup_geps(ptr %ptrA, ptr %ptrB) {
; CHECK-LABEL: define void @load_store_vec_cleanup_geps(
; CHECK-SAME: ptr [[PTRA:%.*]], ptr [[PTRB:%.*]]) {
; CHECK-NEXT:    [[PTRA0:%.*]] = getelementptr i8, ptr [[PTRA]], i32 0
; CHECK-NEXT:    [[PTRB0:%.*]] = getelementptr i8, ptr [[PTRB]], i32 0
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTRA0]], align 1, !sandboxvec [[META10:![0-9]+]]
; CHECK-NEXT:    store <2 x i8> [[VECIINITL]], ptr [[PTRB0]], align 1, !sandboxvec [[META10]]
; CHECK-NEXT:    ret void
;
  %ptrA0 = getelementptr i8, ptr %ptrA, i32 0
  %ptrA1 = getelementptr i8, ptr %ptrA, i32 1
  %ptrB0 = getelementptr i8, ptr %ptrB, i32 0
  %ptrB1 = getelementptr i8, ptr %ptrB, i32 1
  %ld0 = load i8, ptr %ptrA0
  %ld1 = load i8, ptr %ptrA1
  store i8 %ld0, ptr %ptrB0
  store i8 %ld1, ptr %ptrB1
  ret void
}

; Check that we don't try to erase GEPs with other users.
define ptr @load_store_vec_cleanup_gep_with_external_use(ptr %ptr) {
; CHECK-LABEL: define ptr @load_store_vec_cleanup_gep_with_external_use(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 1
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTR0]], align 1, !sandboxvec [[META11:![0-9]+]]
; CHECK-NEXT:    store <2 x i8> [[VECIINITL]], ptr [[PTR0]], align 1, !sandboxvec [[META11]]
; CHECK-NEXT:    ret ptr [[PTR1]]
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i8, ptr %ptr1
  store i8 %ld0, ptr %ptr0
  store i8 %ld1, ptr %ptr1
  ret ptr %ptr1
}

; Check that we schedule both loads and stores
define void @load_store_vec_schedule_stores_and_loads(ptr noalias %ptrA, ptr noalias %ptrB) {
; CHECK-LABEL: define void @load_store_vec_schedule_stores_and_loads(
; CHECK-SAME: ptr noalias [[PTRA:%.*]], ptr noalias [[PTRB:%.*]]) {
; CHECK-NEXT:    [[PTRA0:%.*]] = getelementptr i8, ptr [[PTRA]], i64 0
; CHECK-NEXT:    [[PTRB0:%.*]] = getelementptr i8, ptr [[PTRB]], i64 0
; CHECK-NEXT:    [[PTRB1:%.*]] = getelementptr i8, ptr [[PTRB]], i64 1
; CHECK-NEXT:    [[OTHER:%.*]] = load i8, ptr [[PTRB1]], align 1
; CHECK-NEXT:    [[VECIINITL:%.*]] = load <2 x i8>, ptr [[PTRA0]], align 1, !sandboxvec [[META12:![0-9]+]]
; CHECK-NEXT:    store i8 0, ptr [[PTRA0]], align 1
; CHECK-NEXT:    store <2 x i8> [[VECIINITL]], ptr [[PTRB0]], align 1, !sandboxvec [[META12]]
; CHECK-NEXT:    ret void
;
  %ptrA0 = getelementptr i8, ptr %ptrA, i64 0
  %ptrA1 = getelementptr i8, ptr %ptrA, i64 1
  %ptrB0 = getelementptr i8, ptr %ptrB, i64 0
  %ptrB1 = getelementptr i8, ptr %ptrB, i64 1

  %ld0 = load i8, ptr %ptrA0
  store i8 %ld0, ptr %ptrB0

  store i8 0, ptr %ptrA0
  %other = load i8, ptr %ptrB1

  %ld1 = load i8, ptr %ptrA1
  store i8 %ld1, ptr %ptrB1
  ret void
}

define void @load_store_vec_external_uses(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_external_uses(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 1
; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr i8, ptr [[PTR]], i32 2
; CHECK-NEXT:    [[LD0:%.*]] = load i8, ptr [[PTR0]], align 1
; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR1]], align 1
; CHECK-NEXT:    [[LD2:%.*]] = load i16, ptr [[PTR2]], align 2
; CHECK-NEXT:    store i8 [[LD0]], ptr [[PTR0]], align 1, !sandboxvec [[META13:![0-9]+]]
; CHECK-NEXT:    store i8 [[LD1]], ptr [[PTR1]], align 1, !sandboxvec [[META13]]
; CHECK-NEXT:    store i16 [[LD2]], ptr [[PTR2]], align 2, !sandboxvec [[META14:![0-9]+]]
; CHECK-NEXT:    [[EXT_USER:%.*]] = zext i8 [[LD1]] to i132
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ptr2 = getelementptr i8, ptr %ptr, i32 2
  %ld0 = load i8, ptr %ptr0
  %ld1 = load i8, ptr %ptr1
  %ld2 = load i16, ptr %ptr2
  store i8 %ld0, ptr %ptr0
  store i8 %ld1, ptr %ptr1
  store i16 %ld2, ptr %ptr2

  %ext_user = zext i8 %ld1 to i132
  ret void
}

; Store-constant pattern.
define void @load_store_vec_constants(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    store <3 x i8> <i8 42, i8 43, i8 44>, ptr [[PTR0]], align 1, !sandboxvec [[META15:![0-9]+]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ptr2 = getelementptr i8, ptr %ptr, i32 3
  store i8 42, ptr %ptr0
  store i16 43, ptr %ptr1
  store i8 44, ptr %ptr2
  ret void
}

; Same but with ConstantDataSequential.
define void @load_store_vec_constants_CDS(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants_CDS(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
; CHECK-NEXT:    store <4 x i8> <i8 0, i8 1, i8 2, i8 3>, ptr [[PTR0]], align 1, !sandboxvec [[META16:![0-9]+]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i8, ptr %ptr, i32 0
  %ptr1 = getelementptr i8, ptr %ptr, i32 1
  %ptr2 = getelementptr i8, ptr %ptr, i32 3
  store i8 0, ptr %ptr0
  store <2 x i8> <i8 1, i8 2>, ptr %ptr1
  store i8 3, ptr %ptr2
  ret void
}

; Same but with floats
define void @load_store_vec_constants_CDS_float(ptr %ptr) {
; CHECK-LABEL: define void @load_store_vec_constants_CDS_float(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    store <8 x float> <float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, ptr [[PTR0]], align 1, !sandboxvec [[META17:![0-9]+]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 3
  store float 1.0, ptr %ptr0
  store <2 x float> zeroinitializer, ptr %ptr1
  store <5 x float> zeroinitializer, ptr %ptr2
  ret void
}

;.
; CHECK: [[META0]] = distinct !{!"sandboxregion"}
; CHECK: [[META1]] = distinct !{!"sandboxregion"}
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
; CHECK: [[META4]] = distinct !{!"sandboxregion"}
; CHECK: [[META5]] = distinct !{!"sandboxregion"}
; CHECK: [[META6]] = distinct !{!"sandboxregion"}
; CHECK: [[META7]] = distinct !{!"sandboxregion"}
; CHECK: [[META8]] = distinct !{!"sandboxregion"}
; CHECK: [[META9]] = distinct !{!"sandboxregion"}
; CHECK: [[META10]] = distinct !{!"sandboxregion"}
; CHECK: [[META11]] = distinct !{!"sandboxregion"}
; CHECK: [[META12]] = distinct !{!"sandboxregion"}
; CHECK: [[META13]] = distinct !{!"sandboxregion"}
; CHECK: [[META14]] = distinct !{!"sandboxregion"}
; CHECK: [[META15]] = distinct !{!"sandboxregion"}
; CHECK: [[META16]] = distinct !{!"sandboxregion"}
; CHECK: [[META17]] = distinct !{!"sandboxregion"}
;.