blob: e2d6c58a80cc302f20a6d58bf5b8e4a1140cacf6 [file]
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; sm_100 and ptx88 are required for the 256-bit masked stores below.
; RUN: opt < %s -passes=load-store-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --implicit-check-not=mem.cache_hint
define i32 @load_identical_hints(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @load_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8, !mem.cache_hint [[META0:![0-9]+]]
; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
%sum = add i32 %x, %y
ret i32 %sum
}
define i32 @load_missing_hint(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @load_missing_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
%y = load i32, ptr addrspace(1) %p1, align 4
%sum = add i32 %x, %y
ret i32 %sum
}
define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @load_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !1
%sum = add i32 %x, %y
ret i32 %sum
}
define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
; CHECK-LABEL: define void @store_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8, !mem.cache_hint [[META2:![0-9]+]]
; CHECK-NEXT: ret void
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !2
ret void
}
define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
; CHECK-LABEL: define void @store_missing_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
; CHECK-NEXT: ret void
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
store i32 %y, ptr addrspace(1) %p1, align 4
ret void
}
define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
; CHECK-LABEL: define void @store_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
; CHECK-NEXT: ret void
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !3
ret void
}
define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @load_missing_first_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%x = load i32, ptr addrspace(1) %p, align 8
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
%sum = add i32 %x, %y
ret i32 %sum
}
define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
; CHECK-LABEL: define void @store_missing_first_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
; CHECK-NEXT: ret void
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
store i32 %x, ptr addrspace(1) %p, align 8
store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !2
ret void
}
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @masked_load_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison), !mem.cache_hint [[META0]]
; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
%x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
%z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !0
%xy = add i32 %x, %y
%sum = add i32 %xy, %z
ret i32 %sum
}
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @masked_load_missing_first_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
%x = load i32, ptr addrspace(1) %p, align 16
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
%z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !0
%xy = add i32 %x, %y
%sum = add i32 %xy, %z
ret i32 %sum
}
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @masked_load_missing_last_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
%x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
%z = load i32, ptr addrspace(1) %p3, align 4
%xy = add i32 %x, %y
%sum = add i32 %xy, %z
ret i32 %sum
}
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
; CHECK-LABEL: define i32 @masked_load_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
; CHECK-NEXT: ret i32 [[SUM]]
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
%x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
%y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
%z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !1
%xy = add i32 %x, %y
%sum = add i32 %xy, %z
ret i32 %sum
}
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
; CHECK-LABEL: define void @masked_store_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>), !mem.cache_hint [[META2]]
; CHECK-NEXT: ret void
;
%p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !2
ret void
}
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
; CHECK-LABEL: define void @masked_store_missing_first_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
store i64 %x, ptr addrspace(1) %p, align 32
store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !2
ret void
}
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
; CHECK-LABEL: define void @masked_store_missing_last_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
store i64 %z, ptr addrspace(1) %p3, align 8
ret void
}
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
; CHECK-LABEL: define void @masked_store_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
%p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
%p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !3
ret void
}
!0 = !{i32 0, !4}
!1 = !{i32 0, !5}
!2 = !{i32 1, !4}
!3 = !{i32 1, !5}
!4 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
!5 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_cache_hint", i64 1364590687093260288}
;.
; CHECK: [[META0]] = !{i32 0, [[META1:![0-9]+]]}
; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
; CHECK: [[META2]] = !{i32 1, [[META1]]}
;.