llvm/test/Transforms/Inline/inline_store_to_load.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -passes="cgscc(inline)" < %s | FileCheck %s

; Test that the CGSCC inliner performs store-to-load forwarding for call
; arguments after inlining a function in the same caller. The first call
; (@init or @init_memset) is inlined, producing stores. Forwarding then
; resolves the subsequent load to a constant, enabling inlining of the
; second callee.

target datalayout = "p:64:64"

; Two paths: mode==0 is trivial (ret), otherwise too expensive to inline.
define internal i32 @callee(i32 %mode, ptr %p) {
entry:
  %cmp = icmp eq i32 %mode, 0
  br i1 %cmp, label %fast, label %slow
fast:
  %v = load i32, ptr %p
  ret i32 %v
slow:
  %a1 = load volatile i32, ptr %p
  %a2 = load volatile i32, ptr %p
  %x1 = add i32 %a1, %a2
  %a3 = load volatile i32, ptr %p
  %x2 = add i32 %x1, %a3
  %a4 = load volatile i32, ptr %p
  %x3 = add i32 %x2, %a4
  %a5 = load volatile i32, ptr %p
  %x4 = add i32 %x3, %a5
  ret i32 %x4
}

; Trivial when called with null, otherwise too expensive to inline.
define internal void @recursive_callee(ptr %x) {
; CHECK-LABEL: define internal void @recursive_callee(
; CHECK-SAME: ptr [[X:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[X]], null
; CHECK-NEXT:    br i1 [[CMP]], label %[[DONE:.*]], label %[[RECURSE:.*]]
; CHECK:       [[RECURSE]]:
; CHECK-NEXT:    [[NEXT:%.*]] = load ptr, ptr [[X]], align 8
; CHECK-NEXT:    call void @recursive_callee(ptr [[NEXT]])
; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[X]], align 4
; CHECK-NEXT:    br label %[[DONE]]
; CHECK:       [[DONE]]:
; CHECK-NEXT:    ret void
;
entry:
  %cmp = icmp eq ptr %x, null
  br i1 %cmp, label %done, label %recurse
recurse:
  %next = load ptr, ptr %x
  call void @recursive_callee(ptr %next)
  %v = load volatile i32, ptr %x
  br label %done
done:
  ret void
}

; Trivially cheap — inlined first, producing a store.
define internal void @init_i32(ptr %p) {
  store i32 0, ptr %p
  ret void
}

; Trivially cheap — inlined first, producing a memset.
define internal void @init_memset(ptr %p) {
  call void @llvm.memset.p0.i64(ptr %p, i8 0, i64 8, i1 false)
  ret void
}

; Trivially cheap — stores a non-constant argument value.
define internal void @init_arg(ptr %p, i32 %v) {
  store i32 %v, ptr %p
  ret void
}

; Trivially cheap — stores a non-zero i64 at a slot later loaded as ptr.
define internal void @init_i64_nonzero(ptr %p) {
  store i64 42, ptr %p
  ret void
}

declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)

; After inlining @init_i32: store 0 → %p.
; Forwarding resolves %mode to 0, making only the fast path reachable.
define i32 @caller_store_forward() {
; CHECK-LABEL: define i32 @caller_store_forward() {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P:%.*]] = alloca i32, align 4
; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
; CHECK-NEXT:    [[V_I:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT:    ret i32 [[V_I]]
;
entry:
  %p = alloca i32
  call void @init_i32(ptr %p)
  %mode = load i32, ptr %p
  %r = call i32 @callee(i32 %mode, ptr %p)
  ret i32 %r
}

; Memset-to-load forwarding converts the zero-filled integer to a null
; pointer, making the null check take the early exit.
define void @caller_memset_forward() {
; CHECK-LABEL: define void @caller_memset_forward() {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P:%.*]] = alloca ptr, align 8
; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[P]], i8 0, i64 8, i1 false)
; CHECK-NEXT:    ret void
;
entry:
  %p = alloca ptr
  call void @init_memset(ptr %p)
  %x = load ptr, ptr %p
  call void @recursive_callee(ptr %x)
  ret void
}

; Negative: available value is non-constant (the function argument).
; No forwarding, keeps @callee as a call.
define i32 @caller_non_constant(i32 %v) {
; CHECK-LABEL: define i32 @caller_non_constant(
; CHECK-SAME: i32 [[V:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P:%.*]] = alloca i32, align 4
; CHECK-NEXT:    store i32 [[V]], ptr [[P]], align 4
; CHECK-NEXT:    [[MODE:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[MODE]], 0
; CHECK-NEXT:    br i1 [[CMP_I]], label %[[FAST_I:.*]], label %[[SLOW_I:.*]]
; CHECK:       [[FAST_I]]:
; CHECK-NEXT:    [[V_I:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT:    br label %[[CALLEE_EXIT:.*]]
; CHECK:       [[SLOW_I]]:
; CHECK-NEXT:    [[A1_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[A2_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X1_I:%.*]] = add i32 [[A1_I]], [[A2_I]]
; CHECK-NEXT:    [[A3_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X2_I:%.*]] = add i32 [[X1_I]], [[A3_I]]
; CHECK-NEXT:    [[A4_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X3_I:%.*]] = add i32 [[X2_I]], [[A4_I]]
; CHECK-NEXT:    [[A5_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X4_I:%.*]] = add i32 [[X3_I]], [[A5_I]]
; CHECK-NEXT:    br label %[[CALLEE_EXIT]]
; CHECK:       [[CALLEE_EXIT]]:
; CHECK-NEXT:    [[R1:%.*]] = phi i32 [ [[V_I]], %[[FAST_I]] ], [ [[X4_I]], %[[SLOW_I]] ]
; CHECK-NEXT:    ret i32 [[R1]]
;
entry:
  %p = alloca i32
  call void @init_arg(ptr %p, i32 %v)
  %mode = load i32, ptr %p
  %r = call i32 @callee(i32 %mode, ptr %p)
  ret i32 %r
}

; Negative: available value has a type different from the load's and is not
; a null value (store i64 42, load ptr).
; The type-mismatch blocks forwarding of a non-zero integer as a pointer.
define void @caller_type_mismatch() {
; CHECK-LABEL: define void @caller_type_mismatch() {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P:%.*]] = alloca ptr, align 8
; CHECK-NEXT:    store i64 42, ptr [[P]], align 4
; CHECK-NEXT:    [[X:%.*]] = load ptr, ptr [[P]], align 8
; CHECK-NEXT:    call void @recursive_callee(ptr [[X]])
; CHECK-NEXT:    ret void
;
entry:
  %p = alloca ptr
  call void @init_i64_nonzero(ptr %p)
  %x = load ptr, ptr %p
  call void @recursive_callee(ptr %x)
  ret void
}

; Negative: volatile load must not be forwarded. @callee stays uninlined since
; %mode cannot be resolved to a constant.
define i32 @caller_volatile_load() {
; CHECK-LABEL: define i32 @caller_volatile_load() {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P:%.*]] = alloca i32, align 4
; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
; CHECK-NEXT:    [[MODE:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[MODE]], 0
; CHECK-NEXT:    br i1 [[CMP_I]], label %[[FAST_I:.*]], label %[[SLOW_I:.*]]
; CHECK:       [[FAST_I]]:
; CHECK-NEXT:    [[V_I:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT:    br label %[[CALLEE_EXIT:.*]]
; CHECK:       [[SLOW_I]]:
; CHECK-NEXT:    [[A1_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[A2_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X1_I:%.*]] = add i32 [[A1_I]], [[A2_I]]
; CHECK-NEXT:    [[A3_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X2_I:%.*]] = add i32 [[X1_I]], [[A3_I]]
; CHECK-NEXT:    [[A4_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X3_I:%.*]] = add i32 [[X2_I]], [[A4_I]]
; CHECK-NEXT:    [[A5_I:%.*]] = load volatile i32, ptr [[P]], align 4
; CHECK-NEXT:    [[X4_I:%.*]] = add i32 [[X3_I]], [[A5_I]]
; CHECK-NEXT:    br label %[[CALLEE_EXIT]]
; CHECK:       [[CALLEE_EXIT]]:
; CHECK-NEXT:    [[R1:%.*]] = phi i32 [ [[V_I]], %[[FAST_I]] ], [ [[X4_I]], %[[SLOW_I]] ]
; CHECK-NEXT:    ret i32 [[R1]]
;
entry:
  %p = alloca i32
  call void @init_i32(ptr %p)
  %mode = load volatile i32, ptr %p
  %r = call i32 @callee(i32 %mode, ptr %p)
  ret i32 %r
}