; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=gvn -S < %s | FileCheck %s

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4:5"

define void @f0(i1 %alwaysFalse, i64 %val, ptr %loc) {
; CHECK-LABEL: @f0(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store i64 [[VAL:%.*]], ptr [[LOC:%.*]], align 8
; CHECK-NEXT:    br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
; CHECK:       neverTaken:
; CHECK-NEXT:    [[PTR:%.*]] = load ptr addrspace(4), ptr [[LOC]], align 8
; CHECK-NEXT:    store i8 5, ptr addrspace(4) [[PTR]], align 1
; CHECK-NEXT:    ret void
; CHECK:       alwaysTaken:
; CHECK-NEXT:    ret void
;
  entry:
  store i64 %val, ptr %loc
  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken

  neverTaken:
  %ptr = load ptr addrspace(4), ptr %loc
  store i8 5, ptr addrspace(4) %ptr
  ret void

  alwaysTaken:
  ret void
}

define i64 @f1(i1 %alwaysFalse, ptr addrspace(4) %val, ptr %loc) {
; CHECK-LABEL: @f1(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store ptr addrspace(4) [[VAL:%.*]], ptr [[LOC:%.*]], align 8
; CHECK-NEXT:    br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
; CHECK:       neverTaken:
; CHECK-NEXT:    [[INT:%.*]] = load i64, ptr [[LOC]], align 8
; CHECK-NEXT:    ret i64 [[INT]]
; CHECK:       alwaysTaken:
; CHECK-NEXT:    ret i64 42
;
  entry:
  store ptr addrspace(4) %val, ptr %loc
  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken

  neverTaken:
  %int = load i64, ptr %loc
  ret i64 %int

  alwaysTaken:
  ret i64 42
}

;; Note: For terseness, we stop using the %alwaysfalse trick for the
;; tests below and just exercise the bits of forwarding logic directly.

declare void @llvm.memset.p4.i64(ptr addrspace(4) nocapture, i8, i64, i1) nounwind

; Can't forward as the load might be dead.  (Pretend we wrote out the alwaysfalse idiom above.)
define ptr addrspace(4) @neg_forward_memset(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memset(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memset.p4.i64(ptr addrspace(4) align 4 [[LOC:%.*]], i8 7, i64 8, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) [[REF]]
;
  entry:
  call void @llvm.memset.p4.i64(ptr addrspace(4) align 4 %loc, i8 7, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

define <1 x ptr addrspace(4)> @neg_forward_memset_vload(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memset_vload(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memset.p4.i64(ptr addrspace(4) align 4 [[LOC:%.*]], i8 7, i64 8, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load <1 x ptr addrspace(4)>, ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret <1 x ptr addrspace(4)> [[REF]]
;
  entry:
  call void @llvm.memset.p4.i64(ptr addrspace(4) align 4 %loc, i8 7, i64 8, i1 false)
  %ref = load <1 x ptr addrspace(4)>, ptr addrspace(4) %loc
  ret <1 x ptr addrspace(4)> %ref
}


; Can forward since we can do so w/o breaking types
define ptr addrspace(4) @forward_memset_zero(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_memset_zero(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memset.p4.i64(ptr addrspace(4) align 4 [[LOC:%.*]], i8 0, i64 8, i1 false)
; CHECK-NEXT:    ret ptr addrspace(4) null
;
  entry:
  call void @llvm.memset.p4.i64(ptr addrspace(4) align 4 %loc, i8 0, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

; Can't forward as the load might be dead.  (Pretend we wrote out the alwaysfalse idiom above.)
define ptr addrspace(4) @neg_forward_store(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_store(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store i64 5, ptr addrspace(4) [[LOC:%.*]], align 8
; CHECK-NEXT:    [[REF:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) [[REF]]
;
  entry:
  store i64 5, ptr addrspace(4) %loc
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

define <1 x ptr addrspace(4)> @neg_forward_store_vload(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_store_vload(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store i64 5, ptr addrspace(4) [[LOC:%.*]], align 8
; CHECK-NEXT:    [[REF:%.*]] = load <1 x ptr addrspace(4)>, ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret <1 x ptr addrspace(4)> [[REF]]
;
  entry:
  store i64 5, ptr addrspace(4) %loc
  %ref = load <1 x ptr addrspace(4)>, ptr addrspace(4) %loc
  ret <1 x ptr addrspace(4)> %ref
}

; Nulls have known bit patterns, so we can forward
define ptr addrspace(4) @forward_store_zero(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_store_zero(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store i64 0, ptr addrspace(4) [[LOC:%.*]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) null
;
  entry:
  store i64 0, ptr addrspace(4) %loc
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

; Nulls have known bit patterns, so we can forward
define ptr addrspace(4) @forward_store_zero2(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_store_zero2(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr addrspace(4) [[LOC:%.*]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) null
;
  entry:
  store <2 x i32> zeroinitializer, ptr addrspace(4) %loc
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}


@NonZeroConstant = constant <4 x i64> <i64 3, i64 3, i64 3, i64 3>
@NonZeroConstant2 = constant <4 x ptr addrspace(4)> <
  ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3),
  ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3),
  ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3),
  ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3)>
@ZeroConstant = constant <4 x i64> zeroinitializer


; Can't forward as the load might be dead.  (Pretend we wrote out the alwaysfalse idiom above.)
define ptr addrspace(4) @neg_forward_memcopy(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memcopy(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant, i64 8, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) [[REF]]
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

define ptr addrspace(4) @neg_forward_memcopy2(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memcopy2(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant, i64 8, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) [[REF]]
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

define ptr addrspace(4) @forward_memcopy(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_memcopy(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant2, i64 8, i1 false)
; CHECK-NEXT:    ret ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3)
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant2, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

define ptr addrspace(4) @forward_memcopy2(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_memcopy2(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant2, i64 8, i1 false)
; CHECK-NEXT:    ret ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3)
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant2, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

define <1 x ptr addrspace(4)> @neg_forward_memcpy_vload(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memcpy_vload(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant, i64 8, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load <1 x ptr addrspace(4)>, ptr addrspace(4) [[LOC]], align 8
; CHECK-NEXT:    ret <1 x ptr addrspace(4)> [[REF]]
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant, i64 8, i1 false)
  %ref = load <1 x ptr addrspace(4)>, ptr addrspace(4) %loc
  ret <1 x ptr addrspace(4)> %ref
}

define <4 x ptr addrspace(4)> @neg_forward_memcpy_vload2(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memcpy_vload2(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant, i64 32, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load <4 x ptr addrspace(4)>, ptr addrspace(4) [[LOC]], align 32
; CHECK-NEXT:    ret <4 x ptr addrspace(4)> [[REF]]
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant, i64 32, i1 false)
  %ref = load <4 x ptr addrspace(4)>, ptr addrspace(4) %loc
  ret <4 x ptr addrspace(4)> %ref
}

define <4 x i64> @neg_forward_memcpy_vload3(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_forward_memcpy_vload3(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant2, i64 32, i1 false)
; CHECK-NEXT:    [[REF:%.*]] = load <4 x i64>, ptr addrspace(4) [[LOC]], align 32
; CHECK-NEXT:    ret <4 x i64> [[REF]]
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant2, i64 32, i1 false)
  %ref = load <4 x i64>, ptr addrspace(4) %loc
  ret <4 x i64> %ref
}

define <1 x ptr addrspace(4)> @forward_memcpy_vload3(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_memcpy_vload3(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @NonZeroConstant2, i64 32, i1 false)
; CHECK-NEXT:    ret <1 x ptr addrspace(4)> <ptr addrspace(4) getelementptr (i64, ptr addrspace(4) null, i32 3)>
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @NonZeroConstant2, i64 32, i1 false)
  %ref = load <4 x ptr addrspace(4)>, ptr addrspace(4) %loc
  %val = extractelement <4 x ptr addrspace(4)> %ref, i32 0
  %ret = insertelement <1 x ptr addrspace(4)> undef, ptr addrspace(4) %val, i32 0
  ret <1 x ptr addrspace(4)> %ret
}

; Can forward since we can do so w/o breaking types
define ptr addrspace(4) @forward_memcpy_zero(ptr addrspace(4) %loc) {
; CHECK-LABEL: @forward_memcpy_zero(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 [[LOC:%.*]], ptr @ZeroConstant, i64 8, i1 false)
; CHECK-NEXT:    ret ptr addrspace(4) null
;
entry:
  call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 4 %loc, ptr @ZeroConstant, i64 8, i1 false)
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc
  ret ptr addrspace(4) %ref
}

declare void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) nocapture, ptr nocapture, i64, i1) nounwind


; Same as the neg_forward_store cases, but for non defs.
; (Pretend we wrote out the alwaysfalse idiom above.)
define ptr addrspace(4) @neg_store_clobber(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_store_clobber(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store <2 x i64> splat (i64 4), ptr addrspace(4) [[LOC:%.*]], align 16
; CHECK-NEXT:    [[LOC_OFF:%.*]] = getelementptr ptr addrspace(4), ptr addrspace(4) [[LOC]], i64 1
; CHECK-NEXT:    [[REF:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[LOC_OFF]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) [[REF]]
;
entry:
  store <2 x i64> <i64 4, i64 4>, ptr addrspace(4) %loc
  %loc.off = getelementptr ptr addrspace(4), ptr addrspace(4) %loc, i64 1
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc.off
  ret ptr addrspace(4) %ref
}

declare void @use(<2 x i64>) inaccessiblememonly

; Same as the neg_forward_store cases, but for non defs.
; (Pretend we wrote out the alwaysfalse idiom above.)
define ptr addrspace(4) @neg_load_clobber(ptr addrspace(4) %loc) {
; CHECK-LABEL: @neg_load_clobber(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[V:%.*]] = load <2 x i64>, ptr addrspace(4) [[LOC:%.*]], align 16
; CHECK-NEXT:    call void @use(<2 x i64> [[V]])
; CHECK-NEXT:    [[LOC_OFF:%.*]] = getelementptr ptr addrspace(4), ptr addrspace(4) [[LOC]], i64 1
; CHECK-NEXT:    [[REF:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[LOC_OFF]], align 8
; CHECK-NEXT:    ret ptr addrspace(4) [[REF]]
;
entry:
  %v = load <2 x i64>, ptr addrspace(4) %loc
  call void @use(<2 x i64> %v)
  %loc.off = getelementptr ptr addrspace(4), ptr addrspace(4) %loc, i64 1
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc.off
  ret ptr addrspace(4) %ref
}

define ptr addrspace(4) @store_clobber_zero(ptr addrspace(4) %loc) {
; CHECK-LABEL: @store_clobber_zero(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr addrspace(4) [[LOC:%.*]], align 16
; CHECK-NEXT:    [[LOC_OFF:%.*]] = getelementptr ptr addrspace(4), ptr addrspace(4) [[LOC]], i64 1
; CHECK-NEXT:    ret ptr addrspace(4) null
;
entry:
  store <2 x i64> zeroinitializer, ptr addrspace(4) %loc
  %loc.off = getelementptr ptr addrspace(4), ptr addrspace(4) %loc, i64 1
  %ref = load ptr addrspace(4), ptr addrspace(4) %loc.off
  ret ptr addrspace(4) %ref
}


define void @smaller_vector(ptr %p) {
; CHECK-LABEL: @smaller_vector(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[V4:%.*]] = load <4 x ptr addrspace(4)>, ptr [[P:%.*]], align 32
; CHECK-NEXT:    [[V2:%.*]] = load <2 x ptr addrspace(4)>, ptr [[P]], align 32
; CHECK-NEXT:    call void @use.v2(<2 x ptr addrspace(4)> [[V2]])
; CHECK-NEXT:    call void @use.v4(<4 x ptr addrspace(4)> [[V4]])
; CHECK-NEXT:    ret void
;
entry:
  %v4 = load <4 x ptr addrspace(4)>, ptr %p, align 32
  %v2 = load <2 x ptr addrspace(4)>, ptr %p, align 32
  call void @use.v2(<2 x ptr addrspace(4)> %v2)
  call void @use.v4(<4 x ptr addrspace(4)> %v4)
  ret void
}

define ptr addrspace(4) @vector_extract(ptr %p) {
; CHECK-LABEL: @vector_extract(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[V4:%.*]] = load <4 x ptr addrspace(4)>, ptr [[P:%.*]], align 32
; CHECK-NEXT:    [[RES:%.*]] = load ptr addrspace(4), ptr [[P]], align 32
; CHECK-NEXT:    call void @use.v4(<4 x ptr addrspace(4)> [[V4]])
; CHECK-NEXT:    ret ptr addrspace(4) [[RES]]
;
entry:
  %v4 = load <4 x ptr addrspace(4)>, ptr %p, align 32
  %res = load ptr addrspace(4), ptr %p, align 32
  call void @use.v4(<4 x ptr addrspace(4)> %v4)
  ret ptr addrspace(4) %res
}

declare void @use.v2(<2 x ptr addrspace(4)>)
declare void @use.v4(<4 x ptr addrspace(4)>)
define ptr addrspace(5) @multini(i1 %alwaysFalse, ptr addrspace(4) %val, ptr %loc) {
; CHECK-LABEL: @multini(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    store ptr addrspace(4) [[VAL:%.*]], ptr [[LOC:%.*]], align 8
; CHECK-NEXT:    br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
; CHECK:       neverTaken:
; CHECK-NEXT:    [[DIFFERENTAS:%.*]] = load ptr addrspace(5), ptr [[LOC]], align 8
; CHECK-NEXT:    ret ptr addrspace(5) [[DIFFERENTAS]]
; CHECK:       alwaysTaken:
; CHECK-NEXT:    ret ptr addrspace(5) null
;
entry:
  store ptr addrspace(4) %val, ptr %loc
  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken

neverTaken:
  %differentas = load ptr addrspace(5), ptr %loc
  ret ptr addrspace(5) %differentas

alwaysTaken:
  ret ptr addrspace(5) null
}