llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s

target triple = "arm64-apple-macosx"

define i64 @find_first_mismatch(ptr %A, ptr %B, i64 %n) {
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
  %ld.A = load i8, ptr %gep.A, align 1
  %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
  %ld.B = load i8, ptr %gep.B, align 1
  %cmp = icmp ne i8 %ld.A, %ld.B
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp ne i64 %iv.next, %n
  br i1 %ec, label %loop.header, label %exit

early.exit:
  ret i64 %iv

exit:
  ret i64 -1
}

; Same as @find_first_mismatch, but the induction is not live out.
define i64 @find_first_mismatch_no_live_out(ptr %A, ptr %B, i64 %n) {
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
  %ld.A = load i8, ptr %gep.A, align 1
  %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
  %ld.B = load i8, ptr %gep.B, align 1
  %cmp = icmp ne i8 %ld.A, %ld.B
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp ne i64 %iv.next, %n
  br i1 %ec, label %loop.header, label %exit

early.exit:
  ret i64 1

exit:
  ret i64 0
}

define i64 @early_and_latch_exit_to_same_block(ptr %A, i64 %n) {
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
  %ld.A = load i8, ptr %gep.A, align 1
  %cmp = icmp eq i8 %ld.A, 42
  br i1 %cmp, label %exit, label %loop.latch

loop.latch:
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp ne i64 %iv.next, %n
  br i1 %ec, label %loop.header, label %exit

exit:
  %res = phi i64 [ %iv, %loop.header ], [ -1, %loop.latch ]
  ret i64 %res
}

define ptr @address_live_out(ptr %p, i64 %n) {
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %gep = getelementptr i8, ptr %p, i64 %iv
  %l = load i8, ptr %gep, align 1
  %c = icmp eq i8 %l, 0
  br i1 %c, label %early.exit, label %loop.latch

loop.latch:
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret ptr %gep

exit:
  ret ptr null
}

define i64 @index_cast_live_out(ptr %p, i32 %n) {
entry:
  br label %loop.header

loop.header:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %idx = zext i32 %iv to i64
  %gep = getelementptr i8, ptr %p, i64 %idx
  %l = load i8, ptr %gep, align 1
  %c = icmp eq i8 %l, 0
  br i1 %c, label %early.exit, label %loop.latch

loop.latch:
  %iv.next = add i32 %iv, 1
  %ec = icmp eq i32 %iv.next, %n
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret i64 %idx

exit:
  ret i64 -1
}

define i32 @masked_index_broadcast(ptr %a, i64 %n) {
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %add = add i64 %iv, %n
  %cmp = icmp ugt i64 %add, 0
  br i1 %cmp, label %loop.latch, label %body

body:
  %idx = and i64 %add, 4294967295
  %gep = getelementptr [8 x i8], ptr %a, i64 %idx
  %l = load i64, ptr %gep, align 8
  %c = icmp eq i64 %l, 0
  br i1 %c, label %loop.latch, label %early.exit

loop.latch:
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv, %n
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret i32 1

exit:
  ret i32 0
}

; CHECK-LABEL: define i64 @find_first_mismatch(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT:    [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT:    [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
; CHECK-NEXT:    ret i64 [[IV_LCSSA]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret i64 -1
;
;
; CHECK-LABEL: define i64 @find_first_mismatch_no_live_out(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT:    [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT:    [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret i64 1
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret i64 0
;
;
; CHECK-LABEL: define i64 @early_and_latch_exit_to_same_block(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT:    [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[LD_A]], 42
; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ -1, %[[LOOP_LATCH]] ]
; CHECK-NEXT:    ret i64 [[RES]]
;
;
; CHECK-LABEL: define ptr @address_live_out(
; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[GEP]], align 1
; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
; CHECK-NEXT:    br i1 [[C]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[GEP_LCSSA:%.*]] = phi ptr [ [[GEP]], %[[LOOP_HEADER]] ]
; CHECK-NEXT:    ret ptr [[GEP_LCSSA]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret ptr null
;
;
; CHECK-LABEL: define i64 @index_cast_live_out(
; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[IDX:%.*]] = zext i32 [[IV]] to i64
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[GEP]], align 1
; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
; CHECK-NEXT:    br i1 [[C]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[IDX_LCSSA:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ]
; CHECK-NEXT:    ret i64 [[IDX_LCSSA]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret i64 -1
;
;
; CHECK-LABEL: define i32 @masked_index_broadcast(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[IV]], [[N]]
; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[ADD]], 0
; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[BODY:.*]]
; CHECK:       [[BODY]]:
; CHECK-NEXT:    [[IDX:%.*]] = and i64 [[ADD]], 4294967295
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[GEP]], align 8
; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[L]], 0
; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[EARLY_EXIT:.*]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret i32 1
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret i32 0
;