; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-SDAG %s
; RUN: llc -global-isel -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-GISEL %s

define void @memset_pattern_i128_len0_dynvalue(ptr addrspace(1) align 16 %a, i128 %value) {
; GFX942-LABEL: memset_pattern_i128_len0_dynvalue:
; GFX942:       ; %bb.0:
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i128 %value, i64 0, i1 false)
  ret void
}

define void @memset_pattern_i128_len1_dynvalue(ptr addrspace(1) align 16 %a, i128 %value) {
; GFX942-LABEL: memset_pattern_i128_len1_dynvalue:
; GFX942:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-NEXT:    s_waitcnt vmcnt(0)
; GFX942-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i128 %value, i64 1, i1 false)
  ret void
}

define void @memset_pattern_i128_len1(ptr addrspace(1) align 16 %a) {
; GFX942-SDAG-LABEL: memset_pattern_i128_len1:
; GFX942-SDAG:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xdddddddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xcccccccc
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_len1:
; GFX942-GISEL:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b32 s0, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s1, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s2, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s3, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 1, i1 false)
  ret void
}

define void @memset_pattern_i128_constlen_mainloop_and_residual_taken(ptr addrspace(1) align 16 %a) {
; GFX942-SDAG-LABEL: memset_pattern_i128_constlen_mainloop_and_residual_taken:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    s_mov_b32 s7, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    s_mov_b32 s6, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    s_mov_b32 s5, 0xcccccccc
; GFX942-SDAG-NEXT:    s_mov_b32 s4, 0xdddddddd
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
; GFX942-SDAG-NEXT:    s_mov_b64 s[2:3], 0x1000
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-SDAG-NEXT:  .LBB3_1: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s0, s0, 16
; GFX942-SDAG-NEXT:    s_addc_u32 s1, s1, 0
; GFX942-SDAG-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:240
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:224
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:208
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:192
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:176
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:160
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:144
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:128
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:112
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:96
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:80
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:64
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:48
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:32
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off offset:16
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[2:3], v[4:7], off
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[2:3], v[2:3], 0, s[2:3]
; GFX942-SDAG-NEXT:    s_and_b64 vcc, exec, s[4:5]
; GFX942-SDAG-NEXT:    s_cbranch_vccnz .LBB3_1
; GFX942-SDAG-NEXT:  ; %bb.2: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0x100
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v0, 0xdddddddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v1, 0xcccccccc
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xaaaaaaaa
; GFX942-SDAG-NEXT:  .LBB3_3: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s0, s0, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s1, s1, 0
; GFX942-SDAG-NEXT:    v_cmp_lt_u64_e64 s[2:3], s[0:1], 3
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[4:5], v[4:5], 0, 16
; GFX942-SDAG-NEXT:    s_and_b64 vcc, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_cbranch_vccnz .LBB3_3
; GFX942-SDAG-NEXT:  ; %bb.4: ; %memset.pattern-post-expansion
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_constlen_mainloop_and_residual_taken:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b32 s4, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s5, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s6, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s7, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-GISEL-NEXT:  .LBB3_1: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s0, s0, 16
; GFX942-GISEL-NEXT:    s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT:    s_cselect_b32 s2, 1, 0
; GFX942-GISEL-NEXT:    s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:16
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:32
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:48
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:64
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:80
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:96
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:112
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:128
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:144
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:160
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:176
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:192
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:208
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:224
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[6:7], v[2:5], off offset:240
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v6, vcc, 0x1000, v6
; GFX942-GISEL-NEXT:    s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT:    s_nop 0
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
; GFX942-GISEL-NEXT:    s_cmp_lg_u32 s2, 0
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB3_1
; GFX942-GISEL-NEXT:  ; %bb.2: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v4, vcc, 0x100, v0
; GFX942-GISEL-NEXT:    s_mov_b32 s4, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s5, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s6, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s7, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-GISEL-NEXT:  .LBB3_3: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s0, s0, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v4, vcc, 16, v4
; GFX942-GISEL-NEXT:    v_cmp_lt_u64_e64 s[2:3], s[0:1], 3
; GFX942-GISEL-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB3_3
; GFX942-GISEL-NEXT:  ; %bb.4: ; %memset.pattern-post-expansion
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 19, i1 false)
  ret void
}

define void @memset_pattern_i128_len1_lds(ptr addrspace(3) align 16 %a) {
; GFX942-SDAG-LABEL: memset_pattern_i128_len1_lds:
; GFX942-SDAG:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xdddddddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xcccccccc
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    ds_write_b128 v0, v[2:5]
; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_len1_lds:
; GFX942-GISEL:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b32 s0, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s1, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s2, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s3, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
; GFX942-GISEL-NEXT:    ds_write_b128 v0, v[2:5]
; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(3) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 1, i1 false)
  ret void
}

define void @memset_pattern_i128_len1_no_align(ptr addrspace(1) %a) {
; GFX942-SDAG-LABEL: memset_pattern_i128_len1_no_align:
; GFX942-SDAG:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xdddddddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xcccccccc
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_len1_no_align:
; GFX942-GISEL:       ; %bb.0: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b32 s0, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s1, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s2, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s3, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[0:1]
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 1, i1 false)
  ret void
}

define void @memset_pattern_i128_len16(ptr addrspace(1) align 16 %a) {
; GFX942-SDAG-LABEL: memset_pattern_i128_len16:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    s_mov_b32 s7, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    s_mov_b32 s6, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    s_mov_b32 s5, 0xcccccccc
; GFX942-SDAG-NEXT:    s_mov_b32 s4, 0xdddddddd
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-SDAG-NEXT:    s_mov_b64 s[2:3], 0x1000
; GFX942-SDAG-NEXT:  .LBB6_1: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s0, s0, 16
; GFX942-SDAG-NEXT:    s_addc_u32 s1, s1, 0
; GFX942-SDAG-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:240
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:224
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:208
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:192
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:176
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:160
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:144
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:128
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:112
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:96
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:80
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:64
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:48
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:32
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]
; GFX942-SDAG-NEXT:    s_and_b64 vcc, exec, s[4:5]
; GFX942-SDAG-NEXT:    s_cbranch_vccnz .LBB6_1
; GFX942-SDAG-NEXT:  ; %bb.2: ; %memset.pattern-post-expansion
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_len16:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b32 s4, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s5, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s6, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s7, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT:  .LBB6_1: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s0, s0, 16
; GFX942-GISEL-NEXT:    s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT:    s_cselect_b32 s2, 1, 0
; GFX942-GISEL-NEXT:    s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:32
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:48
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:64
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:80
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:96
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:112
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:128
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:144
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:160
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:176
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:192
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:208
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:224
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:240
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 0x1000, v0
; GFX942-GISEL-NEXT:    s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT:    s_nop 0
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT:    s_cmp_lg_u32 s2, 0
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB6_1
; GFX942-GISEL-NEXT:  ; %bb.2: ; %memset.pattern-post-expansion
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 16, i1 false)
  ret void
}

define void @memset_pattern_i128_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
; GFX942-SDAG-LABEL: memset_pattern_i128_dynlen:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB7_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, 0xdddddddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, 0xcccccccc
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v6, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB7_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 16
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB7_2
; GFX942-SDAG-NEXT:  .LBB7_3: ; %Flow1
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB7_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    s_mov_b32 s8, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s9, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s10, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s11, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[8:9]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[10:11]
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:  .LBB7_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 16, v0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-GISEL-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB7_2
; GFX942-GISEL-NEXT:  .LBB7_3: ; %Flow1
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
  ret void
}

define void @memset_pattern_i128_dynlen_lds(ptr addrspace(3) align 16 %a, i64 %len) {
; GFX942-SDAG-LABEL: memset_pattern_i128_dynlen_lds:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v6, v1
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB8_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xdddddddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xcccccccc
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, 0xbbbbbbbb
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, 0xaaaaaaaa
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB8_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-SDAG-NEXT:    ds_write_b128 v0, v[2:5]
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_add_u32_e32 v0, 16, v0
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB8_2
; GFX942-SDAG-NEXT:  .LBB8_3: ; %Flow1
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen_lds:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v6, v1
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v7, v2
; GFX942-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB8_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    s_mov_b32 s4, 0xdddddddd
; GFX942-GISEL-NEXT:    s_mov_b32 s5, 0xcccccccc
; GFX942-GISEL-NEXT:    s_mov_b32 s6, 0xbbbbbbbb
; GFX942-GISEL-NEXT:    s_mov_b32 s7, 0xaaaaaaaa
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT:  .LBB8_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s4, s4, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-GISEL-NEXT:    ds_write_b128 v0, v[2:5]
; GFX942-GISEL-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-GISEL-NEXT:    v_add_u32_e32 v0, 16, v0
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB8_2
; GFX942-GISEL-NEXT:  .LBB8_3: ; %Flow1
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(3) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
  ret void
}

define void @memset_pattern_i32_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
; GFX942-SDAG-LABEL: memset_pattern_i32_dynlen:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_and_b32_e32 v10, -4, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v11, v3
; GFX942-SDAG-NEXT:    v_and_b32_e32 v8, 3, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB9_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, 0xaabbccdd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, v4
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, v4
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[12:13], v[0:1]
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB9_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 4
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[12:13], v[4:7], off
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[12:13], v[12:13], 0, 64
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB9_2
; GFX942-SDAG-NEXT:  .LBB9_3: ; %Flow6
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB9_6
; GFX942-SDAG-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
; GFX942-SDAG-NEXT:    v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xaabbccdd
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB9_5: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
; GFX942-SDAG-NEXT:    global_store_dword v[0:1], v2, off
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 4
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB9_5
; GFX942-SDAG-NEXT:  .LBB9_6: ; %Flow4
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i32_dynlen:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[8:9], 3
; GFX942-GISEL-NEXT:    v_and_b32_e32 v8, 3, v2
; GFX942-GISEL-NEXT:    v_sub_co_u32_e32 v10, vcc, v2, v8
; GFX942-GISEL-NEXT:    s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT:    s_nop 0
; GFX942-GISEL-NEXT:    v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB9_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    s_mov_b32 s8, 0xaabbccdd
; GFX942-GISEL-NEXT:    s_mov_b32 s9, s8
; GFX942-GISEL-NEXT:    s_mov_b32 s10, s8
; GFX942-GISEL-NEXT:    s_mov_b32 s11, s8
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[8:9]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[10:11]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[0:1]
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:  .LBB9_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 4
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[12:13], v[4:7], off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v12, vcc, 64, v12
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e64 s[0:1], s[6:7], v[10:11]
; GFX942-GISEL-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB9_2
; GFX942-GISEL-NEXT:  .LBB9_3: ; %Flow6
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB9_6
; GFX942-GISEL-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    v_lshrrev_b64 v[2:3], 2, v[2:3]
; GFX942-GISEL-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v2, 0xaabbccdd
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:  .LBB9_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    global_store_dword v[0:1], v2, off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 4, v0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e64 s[0:1], s[6:7], v[8:9]
; GFX942-GISEL-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB9_5
; GFX942-GISEL-NEXT:  .LBB9_6: ; %Flow4
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i32 u0xaabbccdd, i64 %len, i1 false)
  ret void
}

define void @memset_pattern_i32_dynval_dynlen(ptr addrspace(1) align 16 %a, i32 %val, i64 %len) {
; GFX942-SDAG-LABEL: memset_pattern_i32_dynval_dynlen:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v8, v3
; GFX942-SDAG-NEXT:    v_and_b32_e32 v10, -4, v8
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v11, v4
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v9, v4
; GFX942-SDAG-NEXT:    v_and_b32_e32 v6, 3, v8
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, 0
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB10_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, v2
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[12:13], v[0:1]
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB10_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 4
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT:    global_store_dwordx4 v[12:13], v[2:5], off
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[12:13], v[12:13], 0, 64
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB10_2
; GFX942-SDAG-NEXT:  .LBB10_3: ; %Flow6
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB10_6
; GFX942-SDAG-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    v_lshlrev_b64 v[4:5], 2, v[8:9]
; GFX942-SDAG-NEXT:    v_and_b32_e32 v4, -16, v4
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5]
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB10_5: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-SDAG-NEXT:    global_store_dword v[0:1], v2, off
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 4
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB10_5
; GFX942-SDAG-NEXT:  .LBB10_6: ; %Flow4
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i32_dynval_dynlen:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v8, v3
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], 3
; GFX942-GISEL-NEXT:    v_and_b32_e32 v6, 3, v8
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v9, v4
; GFX942-GISEL-NEXT:    v_sub_co_u32_e32 v10, vcc, v8, v6
; GFX942-GISEL-NEXT:    s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT:    s_nop 0
; GFX942-GISEL-NEXT:    v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB10_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v3, v2
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v4, v2
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v5, v2
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[12:13], v[0:1]
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:  .LBB10_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 4
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    global_store_dwordx4 v[12:13], v[2:5], off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v12, vcc, 64, v12
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e64 s[0:1], s[6:7], v[10:11]
; GFX942-GISEL-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB10_2
; GFX942-GISEL-NEXT:  .LBB10_3: ; %Flow6
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB10_6
; GFX942-GISEL-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    v_lshrrev_b64 v[4:5], 2, v[8:9]
; GFX942-GISEL-NEXT:    v_lshl_add_u64 v[0:1], v[4:5], 4, v[0:1]
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:  .LBB10_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    global_store_dword v[0:1], v2, off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 4, v0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e64 s[0:1], s[6:7], v[6:7]
; GFX942-GISEL-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB10_5
; GFX942-GISEL-NEXT:  .LBB10_6: ; %Flow4
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i32 %val, i64 %len, i1 false)
  ret void
}

; For i96, the store size and the alloc size differ on amdgpu, this case is not
; optimized.
define void @memset_pattern_i96_dynval_dynlen(ptr addrspace(1) align 16 %a, i96 %val, i64 %len) {
; GFX942-SDAG-LABEL: memset_pattern_i96_dynval_dynlen:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, v6
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v6, v5
; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB11_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:  .LBB11_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-SDAG-NEXT:    global_store_dwordx3 v[0:1], v[2:4], off
; GFX942-SDAG-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 16
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB11_2
; GFX942-SDAG-NEXT:  .LBB11_3: ; %Flow1
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i96_dynval_dynlen:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v8, v5
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v9, v6
; GFX942-GISEL-NEXT:    s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB11_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:  .LBB11_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    global_store_dwordx3 v[0:1], v[2:4], off
; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 16, v0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e64 s[0:1], s[6:7], v[8:9]
; GFX942-GISEL-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB11_2
; GFX942-GISEL-NEXT:  .LBB11_3: ; %Flow1
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  call void @llvm.experimental.memset.pattern(ptr addrspace(1) align 16 %a, i96 %val, i64 %len, i1 false)
  ret void
}

define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %len) {
; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v8, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v11, v2
; GFX942-SDAG-NEXT:    v_and_b32_e32 v10, -2, v8
; GFX942-SDAG-NEXT:    v_and_b32_e32 v6, 1, v8
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, 0
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB12_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xccccdddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xaaaabbbb
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, s16, v0
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-SDAG-NEXT:  .LBB12_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s8, s8, 2
; GFX942-SDAG-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 32, v1
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB12_2
; GFX942-SDAG-NEXT:  .LBB12_3: ; %Flow3
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB12_6
; GFX942-SDAG-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 3, v8
; GFX942-SDAG-NEXT:    v_and_b32_e32 v1, -16, v1
; GFX942-SDAG-NEXT:    v_add3_u32 v2, v0, v1, s16
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v0, 0xccccdddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v1, 0xaaaabbbb
; GFX942-SDAG-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-SDAG-NEXT:  .LBB12_5: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s8, s8, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
; GFX942-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-SDAG-NEXT:    v_add_u32_e32 v2, 8, v2
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB12_5
; GFX942-SDAG-NEXT:  .LBB12_6: ; %Flow1
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v8, v1
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], 1
; GFX942-GISEL-NEXT:    v_and_b32_e32 v6, 1, v8
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v9, v2
; GFX942-GISEL-NEXT:    v_sub_co_u32_e32 v10, vcc, v8, v6
; GFX942-GISEL-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT:    s_nop 0
; GFX942-GISEL-NEXT:    v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[6:7], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB12_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    s_mov_b32 s8, 0xccccdddd
; GFX942-GISEL-NEXT:    s_mov_b32 s9, 0xaaaabbbb
; GFX942-GISEL-NEXT:    s_mov_b64 s[10:11], s[8:9]
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, s16, v0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[8:9]
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[10:11]
; GFX942-GISEL-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT:  .LBB12_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s8, s8, 2
; GFX942-GISEL-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, 32, v1
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB12_2
; GFX942-GISEL-NEXT:  .LBB12_3: ; %Flow3
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[6:7]
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB12_6
; GFX942-GISEL-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    v_lshrrev_b64 v[2:3], 1, v[8:9]
; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 4, v2
; GFX942-GISEL-NEXT:    v_add3_u32 v2, v0, v1, s16
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v0, 0xccccdddd
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 0xaaaabbbb
; GFX942-GISEL-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT:  .LBB12_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s8, s8, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
; GFX942-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-GISEL-NEXT:    v_add_u32_e32 v2, 8, v2
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB12_5
; GFX942-GISEL-NEXT:  .LBB12_6: ; %Flow1
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  %p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
  call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
  ret void
}

define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %len, i64 %val) {
; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen_dynval:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v9, v4
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, v2
; GFX942-SDAG-NEXT:    v_and_b32_e32 v6, -2, v4
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v8, v3
; GFX942-SDAG-NEXT:    v_and_b32_e32 v2, 1, v4
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB13_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, s16, v0
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v10, v8
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v11, v9
; GFX942-SDAG-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-SDAG-NEXT:  .LBB13_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s8, s8, 2
; GFX942-SDAG-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 32, v1
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB13_2
; GFX942-SDAG-NEXT:  .LBB13_3: ; %Flow3
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT:    s_cbranch_execz .LBB13_6
; GFX942-SDAG-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 3, v4
; GFX942-SDAG-NEXT:    v_and_b32_e32 v1, -16, v1
; GFX942-SDAG-NEXT:    v_add3_u32 v0, v0, v1, s16
; GFX942-SDAG-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-SDAG-NEXT:  .LBB13_5: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s8, s8, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-SDAG-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[2:3]
; GFX942-SDAG-NEXT:    buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-SDAG-NEXT:    v_add_u32_e32 v0, 8, v0
; GFX942-SDAG-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX942-SDAG-NEXT:    s_cbranch_execnz .LBB13_5
; GFX942-SDAG-NEXT:  .LBB13_6: ; %Flow1
; GFX942-SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen_dynval:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v6, v1
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v7, v2
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v8, v3
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[2:3], 1
; GFX942-GISEL-NEXT:    v_and_b32_e32 v2, 1, v6
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v9, v4
; GFX942-GISEL-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
; GFX942-GISEL-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT:    s_nop 0
; GFX942-GISEL-NEXT:    v_subbrev_co_u32_e32 v5, vcc, 0, v7, vcc
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[6:7], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB13_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, s16, v0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[10:11], v[8:9]
; GFX942-GISEL-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT:  .LBB13_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s8, s8, 2
; GFX942-GISEL-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[4:5]
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, 32, v1
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB13_2
; GFX942-GISEL-NEXT:  .LBB13_3: ; %Flow3
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[6:7]
; GFX942-GISEL-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT:    s_cbranch_execz .LBB13_6
; GFX942-GISEL-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    v_lshrrev_b64 v[4:5], 1, v[6:7]
; GFX942-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 4, v4
; GFX942-GISEL-NEXT:    v_add3_u32 v0, v0, v1, s16
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:    s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT:  .LBB13_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s8, s8, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s9, s9, 0
; GFX942-GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, s[8:9], v[2:3]
; GFX942-GISEL-NEXT:    buffer_store_dwordx2 v[8:9], v0, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-GISEL-NEXT:    v_add_u32_e32 v0, 8, v0
; GFX942-GISEL-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GFX942-GISEL-NEXT:    s_cbranch_execnz .LBB13_5
; GFX942-GISEL-NEXT:  .LBB13_6: ; %Flow1
; GFX942-GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  %p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
  call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 %val, i64 %len, i1 false)
  ret void
}

define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16 %a, i32 %offset, i64 %val) {
; GFX942-SDAG-LABEL: memset_pattern_i64_as7_len33_dynval:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT:    v_add_u32_e32 v0, s16, v0
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v6, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v7, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v8, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v9, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v10, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v11, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v12, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v13, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v14, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v15, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v16, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v17, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v18, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v19, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v20, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v21, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v22, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v23, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v24, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v25, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v26, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v27, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v28, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v29, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v30, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v31, v3
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v32, v1
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v33, v3
; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v1, v0
; GFX942-SDAG-NEXT:  .LBB14_1: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s4, s4, 32
; GFX942-SDAG-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[6:9], v1, s[0:3], 0 offen offset:16
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[10:13], v1, s[0:3], 0 offen offset:32
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[14:17], v1, s[0:3], 0 offen offset:48
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[18:21], v1, s[0:3], 0 offen offset:64
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[22:25], v1, s[0:3], 0 offen offset:80
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[26:29], v1, s[0:3], 0 offen offset:96
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[30:33], v1, s[0:3], 0 offen offset:112
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:128
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[6:9], v1, s[0:3], 0 offen offset:144
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[10:13], v1, s[0:3], 0 offen offset:160
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[14:17], v1, s[0:3], 0 offen offset:176
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[18:21], v1, s[0:3], 0 offen offset:192
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[22:25], v1, s[0:3], 0 offen offset:208
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[26:29], v1, s[0:3], 0 offen offset:224
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[30:33], v1, s[0:3], 0 offen offset:240
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 0x2000, v1
; GFX942-SDAG-NEXT:    s_and_b64 vcc, exec, s[6:7]
; GFX942-SDAG-NEXT:    s_cbranch_vccnz .LBB14_1
; GFX942-SDAG-NEXT:  ; %bb.2: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    buffer_store_dwordx2 v[2:3], v0, s[0:3], 0 offen offset:256
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i64_as7_len33_dynval:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v4, v1
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v5, v2
; GFX942-GISEL-NEXT:    v_add_u32_e32 v0, s16, v0
; GFX942-GISEL-NEXT:    s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], v[4:5]
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, v0
; GFX942-GISEL-NEXT:  .LBB14_1: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s4, s4, 32
; GFX942-GISEL-NEXT:    s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT:    s_cselect_b32 s6, 1, 0
; GFX942-GISEL-NEXT:    s_xor_b32 s6, s6, 1
; GFX942-GISEL-NEXT:    s_xor_b32 s6, s6, 1
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:16
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:32
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:48
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:64
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:80
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:96
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:112
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:128
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:144
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:160
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:176
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:192
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:208
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:224
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:240
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, 0x2000, v1
; GFX942-GISEL-NEXT:    s_cmp_lg_u32 s6, 0
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB14_1
; GFX942-GISEL-NEXT:  ; %bb.2: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    buffer_store_dwordx2 v[4:5], v0, s[0:3], 0 offen offset:256
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  %p = getelementptr inbounds i8, ptr addrspace(7) %a, i32 %offset
  call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(16) %p, i64 %val, i64 33, i1 false)
  ret void
}

define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7) inreg align 4 %a, i32 %offset, i64 inreg %len) {
; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen_unaligned_uniform_len:
; GFX942-SDAG:       ; %bb.0:
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT:    s_mov_b32 s9, s18
; GFX942-SDAG-NEXT:    s_and_b32 s4, s17, 1
; GFX942-SDAG-NEXT:    s_and_b32 s8, s17, -2
; GFX942-SDAG-NEXT:    s_mov_b32 s5, 0
; GFX942-SDAG-NEXT:    s_cmp_eq_u64 s[8:9], 0
; GFX942-SDAG-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-SDAG-NEXT:    s_cbranch_scc1 .LBB15_3
; GFX942-SDAG-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v2, 0xccccdddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v3, 0xaaaabbbb
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, s16, v0
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT:  .LBB15_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s6, s6, 2
; GFX942-SDAG-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
; GFX942-SDAG-NEXT:    buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    v_add_u32_e32 v1, 32, v1
; GFX942-SDAG-NEXT:    s_cbranch_vccnz .LBB15_2
; GFX942-SDAG-NEXT:  .LBB15_3: ; %memset.pattern-expansion-residual-cond
; GFX942-SDAG-NEXT:    s_cmp_eq_u64 s[4:5], 0
; GFX942-SDAG-NEXT:    s_cbranch_scc1 .LBB15_6
; GFX942-SDAG-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-SDAG-NEXT:    s_lshl_b32 s6, s17, 3
; GFX942-SDAG-NEXT:    s_and_b32 s6, s6, -16
; GFX942-SDAG-NEXT:    s_add_i32 s16, s16, s6
; GFX942-SDAG-NEXT:    v_add_u32_e32 v4, s16, v0
; GFX942-SDAG-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v0, 0xccccdddd
; GFX942-SDAG-NEXT:    v_mov_b32_e32 v1, 0xaaaabbbb
; GFX942-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-SDAG-NEXT:  .LBB15_5: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT:    s_add_u32 s6, s6, 1
; GFX942-SDAG-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GFX942-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v4, s[0:3], 0 offen
; GFX942-SDAG-NEXT:    v_add_u32_e32 v4, 8, v4
; GFX942-SDAG-NEXT:    s_cbranch_vccnz .LBB15_5
; GFX942-SDAG-NEXT:  .LBB15_6: ; %memset.pattern-post-expansion
; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
;
; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen_unaligned_uniform_len:
; GFX942-GISEL:       ; %bb.0:
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT:    s_mov_b32 s6, s17
; GFX942-GISEL-NEXT:    s_mov_b32 s7, s18
; GFX942-GISEL-NEXT:    s_and_b64 s[4:5], s[6:7], 1
; GFX942-GISEL-NEXT:    s_sub_u32 s8, s17, s4
; GFX942-GISEL-NEXT:    s_subb_u32 s9, s18, s5
; GFX942-GISEL-NEXT:    s_cmp_eq_u64 s[8:9], 0
; GFX942-GISEL-NEXT:    s_mov_b64 s[10:11], 0
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB15_3
; GFX942-GISEL-NEXT:  ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v2, 0xccccdddd
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v3, 0xaaaabbbb
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, s16, v0
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v4, v2
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v5, v3
; GFX942-GISEL-NEXT:  .LBB15_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s10, s10, 2
; GFX942-GISEL-NEXT:    s_addc_u32 s11, s11, 0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[10:11]
; GFX942-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc, s[8:9], v[6:7]
; GFX942-GISEL-NEXT:    buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT:    v_add_u32_e32 v1, 32, v1
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB15_2
; GFX942-GISEL-NEXT:  .LBB15_3: ; %memset.pattern-expansion-residual-cond
; GFX942-GISEL-NEXT:    s_xor_b32 s8, s4, 1
; GFX942-GISEL-NEXT:    s_cmp_lg_u32 s8, 0
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB15_6
; GFX942-GISEL-NEXT:  ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
; GFX942-GISEL-NEXT:    s_lshr_b64 s[6:7], s[6:7], 1
; GFX942-GISEL-NEXT:    s_lshl_b32 s6, s6, 4
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, s16
; GFX942-GISEL-NEXT:    v_add3_u32 v2, v0, s6, v1
; GFX942-GISEL-NEXT:    s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v0, 0xccccdddd
; GFX942-GISEL-NEXT:    v_mov_b32_e32 v1, 0xaaaabbbb
; GFX942-GISEL-NEXT:  .LBB15_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT:    s_add_u32 s6, s6, 1
; GFX942-GISEL-NEXT:    s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
; GFX942-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
; GFX942-GISEL-NEXT:    s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT:    v_add_u32_e32 v2, 8, v2
; GFX942-GISEL-NEXT:    s_cbranch_scc1 .LBB15_5
; GFX942-GISEL-NEXT:  .LBB15_6: ; %memset.pattern-post-expansion
; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
  %p = getelementptr i8, ptr addrspace(7) %a, i32 %offset
  call void @llvm.experimental.memset.pattern(ptr addrspace(7) align(4) %p, i64 u0xaaaabbbbccccdddd, i64 %len, i1 false)
  ret void
}