test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^vector.ph" --version 6
; RUN: opt -passes=loop-vectorize -hoist-runtime-checks=true -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=HOIST
; RUN: opt -passes=loop-vectorize -hoist-runtime-checks=false -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=NO-HOIST

; Only the destination advances: the distance varies in the parent loop.
define void @rmw_variant_distance(ptr %dst, ptr %src, i64 %rows) {
; HOIST-LABEL: define void @rmw_variant_distance(
; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
; HOIST-NEXT:  [[ENTRY:.*]]:
; HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[ROWS]], 6
; HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64
; HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
; HOIST:       [[OUTER_HEADER]]:
; HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
; HOIST-NEXT:    [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16
; HOIST-NEXT:    [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[OFFSET]]
; HOIST-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
; HOIST:       [[VECTOR_MEMCHECK]]:
; HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
; HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; HOIST-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; HOIST-NEXT:    br i1 [[FOUND_CONFLICT]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; HOIST:       [[VECTOR_PH]]:
;
; NO-HOIST-LABEL: define void @rmw_variant_distance(
; NO-HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
; NO-HOIST-NEXT:  [[ENTRY:.*]]:
; NO-HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64
; NO-HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
; NO-HOIST:       [[OUTER_HEADER]]:
; NO-HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
; NO-HOIST-NEXT:    [[TMP1:%.*]] = shl i64 [[OUTER_IV]], 6
; NO-HOIST-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 64
; NO-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
; NO-HOIST-NEXT:    [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16
; NO-HOIST-NEXT:    [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[OFFSET]]
; NO-HOIST-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
; NO-HOIST:       [[VECTOR_MEMCHECK]]:
; NO-HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST_BASE]], [[SCEVGEP1]]
; NO-HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; NO-HOIST-NEXT:    [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-HOIST-NEXT:    br i1 [[DIFF_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; NO-HOIST:       [[VECTOR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %offset = mul i64 %outer.iv, 16
  %dst.base = getelementptr inbounds i32, ptr %dst, i64 %offset
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %loop.header ]
  %src.gep = getelementptr inbounds i32, ptr %src, i64 %iv
  %src.load = load i32, ptr %src.gep, align 4
  %dst.gep = getelementptr inbounds i32, ptr %dst.base, i64 %iv
  %dst.load = load i32, ptr %dst.gep, align 4
  %sum = add i32 %src.load, %dst.load
  store i32 %sum, ptr %dst.gep, align 4
  %iv.next = add nuw i64 %iv, 1
  %inner.done = icmp eq i64 %iv.next, 16
  br i1 %inner.done, label %outer.latch, label %loop.header

outer.latch:
  %outer.iv.next = add nuw i64 %outer.iv, 1
  %outer.done = icmp eq i64 %outer.iv.next, %rows
  br i1 %outer.done, label %exit, label %outer.header

exit:
  ret void
}

; Both pointers advance equally: their distance is invariant in the parent loop.
define void @rmw_invariant_distance(ptr %dst, ptr %src, i64 %rows) {
; HOIST-LABEL: define void @rmw_invariant_distance(
; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
; HOIST-NEXT:  [[ENTRY:.*]]:
; HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[ROWS]], 6
; HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
; HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
; HOIST:       [[OUTER_HEADER]]:
; HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
; HOIST-NEXT:    [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16
; HOIST-NEXT:    [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[OFFSET]]
; HOIST-NEXT:    [[SRC_BASE:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[OFFSET]]
; HOIST-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
; HOIST:       [[VECTOR_MEMCHECK]]:
; HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
; HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; HOIST-NEXT:    [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; HOIST-NEXT:    br i1 [[DIFF_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; HOIST:       [[VECTOR_PH]]:
;
; NO-HOIST-LABEL: define void @rmw_invariant_distance(
; NO-HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
; NO-HOIST-NEXT:  [[ENTRY:.*]]:
; NO-HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
; NO-HOIST:       [[OUTER_HEADER]]:
; NO-HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
; NO-HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[OUTER_IV]], 6
; NO-HOIST-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 64
; NO-HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; NO-HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
; NO-HOIST-NEXT:    [[OFFSET:%.*]] = mul i64 [[OUTER_IV]], 16
; NO-HOIST-NEXT:    [[DST_BASE:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[OFFSET]]
; NO-HOIST-NEXT:    [[SRC_BASE:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[OFFSET]]
; NO-HOIST-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
; NO-HOIST:       [[VECTOR_MEMCHECK]]:
; NO-HOIST-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST_BASE]], [[SCEVGEP1]]
; NO-HOIST-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC_BASE]], [[SCEVGEP]]
; NO-HOIST-NEXT:    [[DIFF_CHECK:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-HOIST-NEXT:    br i1 [[DIFF_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; NO-HOIST:       [[VECTOR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %offset = mul i64 %outer.iv, 16
  %dst.base = getelementptr inbounds i32, ptr %dst, i64 %offset
  %src.base = getelementptr inbounds i32, ptr %src, i64 %offset
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %loop.header ]
  %src.gep = getelementptr inbounds i32, ptr %src.base, i64 %iv
  %src.load = load i32, ptr %src.gep, align 4
  %dst.gep = getelementptr inbounds i32, ptr %dst.base, i64 %iv
  %dst.load = load i32, ptr %dst.gep, align 4
  %sum = add i32 %src.load, %dst.load
  store i32 %sum, ptr %dst.gep, align 4
  %iv.next = add nuw i64 %iv, 1
  %inner.done = icmp eq i64 %iv.next, 16
  br i1 %inner.done, label %outer.latch, label %loop.header

outer.latch:
  %outer.iv.next = add nuw i64 %outer.iv, 1
  %outer.done = icmp eq i64 %outer.iv.next, %rows
  br i1 %outer.done, label %exit, label %outer.header

exit:
  ret void
}