test/Transforms/SandboxVectorizer/topdown_vec.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-collect-seeds=loads -sbvec-passes="seed-collection<tr-save,bottom-up-vec(top-down),tr-accept>" %s -S | FileCheck %s

define void @load_fadd_store(ptr %ptr, ptr %ptr2) {
; CHECK-LABEL: define void @load_fadd_store(
; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META0:![0-9]+]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META0]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1

  %fadd0 = fadd float %ld0, %ld0
  %fadd1 = fadd float %ld1, %ld1

  ret void
}

define void @load_chain_store(ptr %ptr, ptr %ptr2) {
; CHECK-LABEL: define void @load_chain_store(
; CHECK-SAME: ptr [[PTR:%.*]], ptr [[PTR2:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META1:![0-9]+]]
; CHECK-NEXT:    [[VEC:%.*]] = fmul <2 x float> [[VECL]], splat (float 3.000000e+00), !sandboxvec [[META1]]
; CHECK-NEXT:    [[VEC1:%.*]] = fadd <2 x float> [[VEC]], splat (float 2.000000e+00), !sandboxvec [[META1]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1

  %fmul0 = fmul float %ld0, 3.0
  %fmul1 = fmul float %ld1, 3.0

  %fadd0 = fadd float %fmul0, 2.0
  %fadd1 = fadd float %fmul1, 2.0

  ret void
}

define float @load_fadd_external_use(ptr %ptr) {
; CHECK-LABEL: define float @load_fadd_external_use(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META2:![0-9]+]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[VECL]], !sandboxvec [[META2]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VEC]], i32 0, !sandboxvec [[META2]]
; CHECK-NEXT:    ret float [[UNPACK]]
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1

  %fadd0 = fadd float %ld0, %ld0
  %fadd1 = fadd float %ld1, %ld1

  ret float %fadd0
}

define float @single_user_no_duplicate(ptr %ptr) {
; CHECK-LABEL: define float @single_user_no_duplicate(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META3:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META3]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META3]]
; CHECK-NEXT:    [[FADD:%.*]] = fadd float [[UNPACK]], [[UNPACK1]]
; CHECK-NEXT:    ret float [[FADD]]
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1

  %fadd = fadd float %ld0, %ld1
  ret float %fadd
}

define void @users_in_different_blocks(ptr %ptr, i1 %c) {
; CHECK-LABEL: define void @users_in_different_blocks(
; CHECK-SAME: ptr [[PTR:%.*]], i1 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META4:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META4]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META4]]
; CHECK-NEXT:    br i1 [[C]], label %[[BB0:.*]], label %[[BB1:.*]]
; CHECK:       [[BB0]]:
; CHECK-NEXT:    [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    ret void
; CHECK:       [[BB1]]:
; CHECK-NEXT:    [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT:    ret void
;
entry:
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  br i1 %c, label %if.then, label %if.else

if.then:
  %fadd0 = fadd float %ld0, %ld0
  ret void

if.else:
  %fadd1 = fadd float %ld1, %ld1
  ret void
}

define void @user_opcode_mismatch(ptr %ptr) {
; CHECK-LABEL: define void @user_opcode_mismatch(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META5:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META5]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META5]]
; CHECK-NEXT:    [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0 = fadd float %ld0, %ld0
  %fmul1 = fmul float %ld1, %ld1
  ret void
}

define void @user_type_mismatch(ptr %ptr) {
; CHECK-LABEL: define void @user_type_mismatch(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META6:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META6]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META6]]
; CHECK-NEXT:    [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    [[EXT1:%.*]] = fpext float [[UNPACK1]] to double
; CHECK-NEXT:    [[FADD1:%.*]] = fadd double [[EXT1]], [[EXT1]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0 = fadd float %ld0, %ld0
  %ext1 = fpext float %ld1 to double
  %fadd1 = fadd double %ext1, %ext1
  ret void
}

define void @user_operand_index_mismatch(ptr %ptr, float %x) {
; CHECK-LABEL: define void @user_operand_index_mismatch(
; CHECK-SAME: ptr [[PTR:%.*]], float [[X:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META7:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META7]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META7]]
; CHECK-NEXT:    [[FSUB0:%.*]] = fsub float [[UNPACK]], [[X]]
; CHECK-NEXT:    [[FSUB1:%.*]] = fsub float [[X]], [[UNPACK1]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fsub0 = fsub float %ld0, %x
  %fsub1 = fsub float %x, %ld1
  ret void
}

define void @user_operand_index_mismatch_3wide(ptr %ptr, float %x) {
; CHECK-LABEL: define void @user_operand_index_mismatch_3wide(
; CHECK-SAME: ptr [[PTR:%.*]], float [[X:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <3 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META8:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META8]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META8]]
; CHECK-NEXT:    [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META8]]
; CHECK-NEXT:    [[FSUB0:%.*]] = fsub float [[UNPACK]], [[X]]
; CHECK-NEXT:    [[FSUB1:%.*]] = fsub float [[X]], [[UNPACK1]]
; CHECK-NEXT:    [[FSUB2:%.*]] = fsub float [[UNPACK2]], 0.000000e+00
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 2

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4
  %ld2 = load float, ptr %ptr2, align 4

  %fsub0 = fsub float %ld0, %x
  %fsub1 = fsub float %x, %ld1
  %fsub2 = fsub float %ld2, 0.0
  ret void
}

define void @user_already_vectorized(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META9:![0-9]+]]
; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META9]]
; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META9]]
; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META9]]
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META9]]
; CHECK-NEXT:    [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META9]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META9]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0 = fadd float %ld0, %a
  %fadd1 = fadd float %ld1, %a
  %fadd0b = fadd float %ld0, %b
  %fadd1b = fadd float %ld1, %b
  ret void
}

define void @multiple_users_of_loads(ptr %ptr, float %a, float %b, float %c) {
; CHECK-LABEL: define void @multiple_users_of_loads(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PACK5:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META10:![0-9]+]]
; CHECK-NEXT:    [[PACK6:%.*]] = insertelement <2 x float> [[PACK5]], float [[A]], i32 1, !sandboxvec [[META10]]
; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META10]]
; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[B]], i32 1, !sandboxvec [[META10]]
; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x float> poison, float [[C]], i32 0, !sandboxvec [[META10]]
; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[C]], i32 1, !sandboxvec [[META10]]
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META10]]
; CHECK-NEXT:    [[VEC7:%.*]] = fadd <2 x float> [[VECL]], [[PACK6]], !sandboxvec [[META10]]
; CHECK-NEXT:    [[VEC4:%.*]] = fadd <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META10]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META10]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0a = fadd float %ld0, %a
  %fadd0b = fadd float %ld0, %b
  %fadd0c = fadd float %ld0, %c

  %fadd1a = fadd float %ld1, %a
  %fadd1b = fadd float %ld1, %b
  %fadd1c = fadd float %ld1, %c
  ret void
}

define void @multiple_users_of_loads_diff_opcode(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @multiple_users_of_loads_diff_opcode(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META11:![0-9]+]]
; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <2 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META11]]
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META11]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META11]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META11]]
; CHECK-NEXT:    [[FADD0A:%.*]] = fadd float [[UNPACK]], [[A]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK2]], !sandboxvec [[META11]]
; CHECK-NEXT:    [[FSUB1A:%.*]] = fsub float [[UNPACK1]], [[A]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0a = fadd float %ld0, %a
  %fadd0b = fadd float %ld0, %b

  %fsub1a = fsub float %ld1, %a
  %fadd1b = fadd float %ld1, %b
  ret void
}

define void @users_in_arbitrary_order(ptr %ptr, float %a, float %b, float %c) {
; CHECK-LABEL: define void @users_in_arbitrary_order(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PACK8:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META12:![0-9]+]]
; CHECK-NEXT:    [[PACK9:%.*]] = insertelement <2 x float> [[PACK8]], float [[A]], i32 1, !sandboxvec [[META12]]
; CHECK-NEXT:    [[PACK5:%.*]] = insertelement <2 x float> poison, float [[B]], i32 0, !sandboxvec [[META12]]
; CHECK-NEXT:    [[PACK6:%.*]] = insertelement <2 x float> [[PACK5]], float [[B]], i32 1, !sandboxvec [[META12]]
; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <2 x float> poison, float [[A]], i32 0, !sandboxvec [[META12]]
; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <2 x float> [[PACK2]], float [[A]], i32 1, !sandboxvec [[META12]]
; CHECK-NEXT:    [[PACK:%.*]] = insertelement <2 x float> poison, float [[C]], i32 0, !sandboxvec [[META12]]
; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <2 x float> [[PACK]], float [[C]], i32 1, !sandboxvec [[META12]]
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META12]]
; CHECK-NEXT:    [[VEC10:%.*]] = fadd <2 x float> [[VECL]], [[PACK9]], !sandboxvec [[META12]]
; CHECK-NEXT:    [[VEC7:%.*]] = fadd <2 x float> [[VECL]], [[PACK6]], !sandboxvec [[META12]]
; CHECK-NEXT:    [[VEC4:%.*]] = fsub <2 x float> [[VECL]], [[PACK3]], !sandboxvec [[META12]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <2 x float> [[VECL]], [[PACK1]], !sandboxvec [[META12]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0a = fadd float %ld0, %a
  %fadd1a = fadd float %ld1, %a
  %fadd0b = fadd float %ld0, %b
  %fadd1b = fadd float %ld1, %b
  %fsub0a = fsub float %ld0, %a
  %fadd0c = fadd float %ld0, %c
  %fadd1c = fadd float %ld1, %c
  %fsub1a = fsub float %ld1, %a

  ret void
}

define void @user_diff_fast_math_flags(ptr %ptr) {
; CHECK-LABEL: define void @user_diff_fast_math_flags(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META13:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META13]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META13]]
; CHECK-NEXT:    [[FADD0:%.*]] = fadd fast float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0 = fadd fast float %ld0, %ld0
  %fadd1 = fadd float %ld1, %ld1
  ret void
}

define void @user_diff_wrap_flags(ptr %ptr) {
; CHECK-LABEL: define void @user_diff_wrap_flags(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 4, !sandboxvec [[META14:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x i32> [[VECL]], i32 0, !sandboxvec [[META14]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECL]], i32 1, !sandboxvec [[META14]]
; CHECK-NEXT:    [[ADD0:%.*]] = add nsw i32 [[UNPACK]], 1
; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[UNPACK1]], 1
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr i32, ptr %ptr, i32 0
  %ptr1 = getelementptr i32, ptr %ptr, i32 1

  %ld0 = load i32, ptr %ptr0, align 4
  %ld1 = load i32, ptr %ptr1, align 4

  %add0 = add nsw i32 %ld0, 1
  %add1 = add i32 %ld1, 1
  ret void
}

define void @user_duplicate_operand_other(ptr %ptr, float %other) {
; CHECK-LABEL: define void @user_duplicate_operand_other(
; CHECK-SAME: ptr [[PTR:%.*]], float [[OTHER:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META15:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <2 x float> [[VECL]], i32 0, !sandboxvec [[META15]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <2 x float> [[VECL]], i32 1, !sandboxvec [[META15]]
; CHECK-NEXT:    [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    [[FADD1:%.*]] = fadd float [[UNPACK1]], [[OTHER]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4

  %fadd0 = fadd float %ld0, %ld0
  %fadd1 = fadd float %ld1, %other
  ret void
}

define void @user_stores_not_consecutive(ptr %ptr) {
; CHECK-LABEL: define void @user_stores_not_consecutive(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <2 x float>, ptr [[PTR0]], align 4, !sandboxvec [[META16:![0-9]+]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1

  %ld0 = load float, ptr %ptr0, align 4
  %ld1 = load float, ptr %ptr1, align 4
  ret void
}

; FIXME: Following test needs diamond reuse multi-input support in topdown
; vectorizer.
;define void @user_diamond_reuse_multi_input(ptr %ptr, ptr %ptr2) {
;  %ptr0 = getelementptr float, ptr %ptr, i32 0
;  %ptr1 = getelementptr float, ptr %ptr, i32 1
;  %ptr2_0 = getelementptr float, ptr %ptr2, i32 0
;  %ptr2_2 = getelementptr float, ptr %ptr2, i32 2
;
;  %ld0 = load float, ptr %ptr0, align 4
;  %ld1 = load float, ptr %ptr1, align 4
;
;  %sub0 = fsub float %ld0, 0.000000e+00
;  %sub1 = fsub float %ld1, 1.000000e+00
;
;  %add0 = fadd float %ld0, 0.000000e+00
;  %add1 = fadd float %ld1, 0.000000e+00
;
;  %diam0 = fsub float %sub0, %add0
;  %diam1 = fsub float %sub1, %add1
;
;  store float %diam0, ptr %ptr2_0, align 4
;  store float %diam1, ptr %ptr2_2, align 4
;  ret void
;}

define void @load_fadd_store_3wide(ptr %ptr) {
; CHECK-LABEL: define void @load_fadd_store_3wide(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META17:![0-9]+]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[VECL]], !sandboxvec [[META17]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 2
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  %ld2 = load float, ptr %ptr2

  %fadd0 = fadd float %ld0, %ld0
  %fadd1 = fadd float %ld1, %ld1
  %fadd2 = fadd float %ld2, %ld2
  ret void
}

define void @user_already_vectorized_3wide(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @user_already_vectorized_3wide(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT:    [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <3 x float> poison, float [[A]], i32 0, !sandboxvec [[META18:![0-9]+]]
; CHECK-NEXT:    [[PACK4:%.*]] = insertelement <3 x float> [[PACK3]], float [[A]], i32 1, !sandboxvec [[META18]]
; CHECK-NEXT:    [[PACK5:%.*]] = insertelement <3 x float> [[PACK4]], float [[A]], i32 2, !sandboxvec [[META18]]
; CHECK-NEXT:    [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META18]]
; CHECK-NEXT:    [[PACK1:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META18]]
; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <3 x float> [[PACK1]], float [[B]], i32 2, !sandboxvec [[META18]]
; CHECK-NEXT:    [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META18]]
; CHECK-NEXT:    [[VEC6:%.*]] = fadd <3 x float> [[VECL]], [[PACK5]], !sandboxvec [[META18]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK2]], !sandboxvec [[META18]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 2
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  %ld2 = load float, ptr %ptr2

  %fadd0a = fadd float %ld0, %a
  %fadd0b = fadd float %ld0, %b
  %fadd1a = fadd float %ld1, %a
  %fadd1b = fadd float %ld1, %b
  %fadd2a = fadd float %ld2, %a
  %fadd2b = fadd float %ld2, %b
  ret void
}

define void @user_opcode_mismatch_3wide(ptr %ptr) {
; CHECK-LABEL: define void @user_opcode_mismatch_3wide(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META19:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META19]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META19]]
; CHECK-NEXT:    [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META19]]
; CHECK-NEXT:    [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    [[FADD1:%.*]] = fadd float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT:    [[FMUL2:%.*]] = fmul float [[UNPACK2]], [[UNPACK2]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 2
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  %ld2 = load float, ptr %ptr2

  %fadd0 = fadd float %ld0, %ld0
  %fadd1 = fadd float %ld1, %ld1
  %fmul2 = fmul float %ld2, %ld2
  ret void
}

define void @claimed_rollback_partial_bundle(ptr %ptr, float %a, float %b) {
; CHECK-LABEL: define void @claimed_rollback_partial_bundle(
; CHECK-SAME: ptr [[PTR:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT:    [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[PACK:%.*]] = insertelement <3 x float> poison, float [[B]], i32 0, !sandboxvec [[META20:![0-9]+]]
; CHECK-NEXT:    [[PACK2:%.*]] = insertelement <3 x float> [[PACK]], float [[B]], i32 1, !sandboxvec [[META20]]
; CHECK-NEXT:    [[PACK3:%.*]] = insertelement <3 x float> [[PACK2]], float [[B]], i32 2, !sandboxvec [[META20]]
; CHECK-NEXT:    [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META20]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META20]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META20]]
; CHECK-NEXT:    [[FADD0A:%.*]] = fadd float [[UNPACK]], [[A]]
; CHECK-NEXT:    [[VEC:%.*]] = fadd <3 x float> [[VECL]], [[PACK3]], !sandboxvec [[META20]]
; CHECK-NEXT:    [[FADD1A:%.*]] = fadd float [[UNPACK1]], [[A]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 2
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  %ld2 = load float, ptr %ptr2

  %fadd0a = fadd float %ld0, %a
  %fadd0b = fadd float %ld0, %b
  %fadd1a = fadd float %ld1, %a
  %fadd1b = fadd float %ld1, %b
  %fadd2b = fadd float %ld2, %b
  ret void
}

define void @user_middle_lane_mismatch_3wide(ptr %ptr) {
; CHECK-LABEL: define void @user_middle_lane_mismatch_3wide(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[G0:%.*]] = getelementptr float, ptr [[PTR]], i32 0
; CHECK-NEXT:    [[VECL:%.*]] = load <3 x float>, ptr [[G0]], align 4, !sandboxvec [[META21:![0-9]+]]
; CHECK-NEXT:    [[UNPACK:%.*]] = extractelement <3 x float> [[VECL]], i32 0, !sandboxvec [[META21]]
; CHECK-NEXT:    [[UNPACK1:%.*]] = extractelement <3 x float> [[VECL]], i32 1, !sandboxvec [[META21]]
; CHECK-NEXT:    [[UNPACK2:%.*]] = extractelement <3 x float> [[VECL]], i32 2, !sandboxvec [[META21]]
; CHECK-NEXT:    [[FADD0:%.*]] = fadd float [[UNPACK]], [[UNPACK]]
; CHECK-NEXT:    [[FMUL1:%.*]] = fmul float [[UNPACK1]], [[UNPACK1]]
; CHECK-NEXT:    [[FADD2:%.*]] = fadd float [[UNPACK2]], [[UNPACK2]]
; CHECK-NEXT:    ret void
;
  %ptr0 = getelementptr float, ptr %ptr, i32 0
  %ptr1 = getelementptr float, ptr %ptr, i32 1
  %ptr2 = getelementptr float, ptr %ptr, i32 2
  %ld0 = load float, ptr %ptr0
  %ld1 = load float, ptr %ptr1
  %ld2 = load float, ptr %ptr2

  %fadd0 = fadd float %ld0, %ld0
  %fmul1 = fmul float %ld1, %ld1
  %fadd2 = fadd float %ld2, %ld2
  ret void
}

;.
; CHECK: [[META0]] = distinct !{!"sandboxregion"}
; CHECK: [[META1]] = distinct !{!"sandboxregion"}
; CHECK: [[META2]] = distinct !{!"sandboxregion"}
; CHECK: [[META3]] = distinct !{!"sandboxregion"}
; CHECK: [[META4]] = distinct !{!"sandboxregion"}
; CHECK: [[META5]] = distinct !{!"sandboxregion"}
; CHECK: [[META6]] = distinct !{!"sandboxregion"}
; CHECK: [[META7]] = distinct !{!"sandboxregion"}
; CHECK: [[META8]] = distinct !{!"sandboxregion"}
; CHECK: [[META9]] = distinct !{!"sandboxregion"}
; CHECK: [[META10]] = distinct !{!"sandboxregion"}
; CHECK: [[META11]] = distinct !{!"sandboxregion"}
; CHECK: [[META12]] = distinct !{!"sandboxregion"}
; CHECK: [[META13]] = distinct !{!"sandboxregion"}
; CHECK: [[META14]] = distinct !{!"sandboxregion"}
; CHECK: [[META15]] = distinct !{!"sandboxregion"}
; CHECK: [[META16]] = distinct !{!"sandboxregion"}
; CHECK: [[META17]] = distinct !{!"sandboxregion"}
; CHECK: [[META18]] = distinct !{!"sandboxregion"}
; CHECK: [[META19]] = distinct !{!"sandboxregion"}
; CHECK: [[META20]] = distinct !{!"sandboxregion"}
; CHECK: [[META21]] = distinct !{!"sandboxregion"}
;.