llvm/test/Transforms/LoopVectorize/AArch64/diff-checks-interleave-groups.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "middle.block" --version 6
; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -memory-check-merge-threshold=0 -S %s | FileCheck %s

target triple = "arm64-apple-macosx"

; With -memory-check-merge-threshold=0, the %gep.*.n accesses keep the members
; of interleave groups in separate runtime check groups.

; The wide load of %gep.a.1 and %gep.a.0 is placed before the store to %gep.c.
define i32 @load_group_reordered_with_store(ptr %a, ptr %c, i64 %n) {
; CHECK-LABEL: define i32 @load_group_reordered_with_store(
; CHECK-SAME: ptr [[A:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[C7:%.*]] = ptrtoaddr ptr [[C]] to i64
; CHECK-NEXT:    [[A6:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK:       [[VECTOR_MEMCHECK]]:
; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[A6]], [[C7]]
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[N]], 3
; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[C7]], [[A6]]
; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], [[TMP2]]
; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], 1
; CHECK-NEXT:    [[DIFF_CHECK8:%.*]] = icmp ult i64 [[TMP5]], 31
; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK8]]
; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[C7]], -4
; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], [[A6]]
; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
; CHECK-NEXT:    [[DIFF_CHECK9:%.*]] = icmp ult i64 [[TMP8]], 31
; CHECK-NEXT:    [[CONFLICT_RDX10:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK9]]
; CHECK-NEXT:    br i1 [[CONFLICT_RDX10]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP9]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP11:%.*]] = add nuw nsw i64 [[TMP10]], 1
; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 -4
; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP13]], align 4
; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT:    [[STRIDED_VEC11:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEXT:    [[TMP14:%.*]] = add nuw nsw i64 [[INDEX]], [[N]]
; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP14]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP15]], align 8
; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[TMP16]], align 8
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP17]], [[MIDDLE_BLOCK:label %.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %off.0 = shl nuw nsw i64 %iv, 1
  %off.1 = add nuw nsw i64 %off.0, 1
  %gep.a.1 = getelementptr inbounds i32, ptr %a, i64 %off.1
  %l.1 = load i32, ptr %gep.a.1, align 4
  %iv.n = add nuw nsw i64 %iv, %n
  %gep.a.n = getelementptr inbounds i64, ptr %a, i64 %iv.n
  %l.n = load i64, ptr %gep.a.n, align 8
  %gep.c = getelementptr inbounds i64, ptr %c, i64 %iv
  store i64 %l.n, ptr %gep.c, align 8
  %gep.a.0 = getelementptr inbounds i32, ptr %a, i64 %off.0
  %l.0 = load i32, ptr %gep.a.0, align 4
  %add = add i32 %l.0, %l.1
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret i32 %add
}

; The wide store of %gep.c.1 and %gep.c.0 is placed after the load from %gep.a.
define void @store_group_reordered_with_load(ptr %a, ptr %c, i64 %n) {
; CHECK-LABEL: define void @store_group_reordered_with_load(
; CHECK-SAME: ptr [[A:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[A5:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-NEXT:    [[C4:%.*]] = ptrtoaddr ptr [[C]] to i64
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK:       [[VECTOR_MEMCHECK]]:
; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[C4]], [[A5]]
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], 1
; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[A5]], -4
; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], [[C4]]
; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
; CHECK-NEXT:    [[DIFF_CHECK6:%.*]] = icmp ult i64 [[TMP4]], 31
; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK6]]
; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 0
; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i64 4, i64 [[TMP5]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP7]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[INDEX]], [[N]]
; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[TMP8]]
; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
; CHECK-NEXT:    [[TMP12:%.*]] = trunc <4 x i64> [[WIDE_LOAD]] to <4 x i32>
; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP10]]
; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> [[STRIDED_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP14]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
; CHECK-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP15]], [[MIDDLE_BLOCK:label %.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.n = add nuw nsw i64 %iv, %n
  %gep.c.n = getelementptr inbounds i64, ptr %c, i64 %iv.n
  %l.n = load i32, ptr %gep.c.n, align 4
  %off.0 = shl nuw nsw i64 %iv, 1
  %off.1 = add nuw nsw i64 %off.0, 1
  %gep.c.1 = getelementptr inbounds i32, ptr %c, i64 %off.1
  store i32 %l.n, ptr %gep.c.1, align 4
  %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
  %l = load i64, ptr %gep.a, align 8
  %trunc = trunc i64 %l to i32
  %gep.c.0 = getelementptr inbounds i32, ptr %c, i64 %off.0
  store i32 %trunc, ptr %gep.c.0, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}