Google Git
Sign in
llvm/llvm-project/34d5bcddcfd2f3fa3ce074e1be49fe4eae920f82/./llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
blob: bb122ed694ce240345c3aa5f94887be45fe24f75 [file]
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -mtriple=amdgpu11.00-amd-amdhsa -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
; RUN: opt -mtriple=amdgpu11.00-amd-amdhsa -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
; RUN: opt -mtriple=amdgpu9.00-amd-amdhsa -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX9
declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
declare i64 @llvm.amdgcn.wave.shuffle.i64(i64, i32)
declare i16 @llvm.amdgcn.wave.shuffle.i16(i16, i32)
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
declare <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16>, i32)
declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
declare ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5), i32)
declare ptr @llvm.amdgcn.wave.shuffle.p0(ptr, i32)
declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
; Quad perm: lane ^ 1 swaps adjacent pairs.
define i32 @test_quad_perm_xor1_w32(i32 %val) {
; GFX11-LABEL: @test_quad_perm_xor1_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Quad perm: lane ^ 3 reverses within each quad.
define i32 @test_quad_perm_xor3_w32(i32 %val) {
; GFX11-LABEL: @test_quad_perm_xor3_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 27, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor3_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor3_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 3
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Half-row mirror: reverse within every 8-lane group.
define i32 @test_half_row_mirror_w32(i32 %val) {
; GFX11-LABEL: @test_half_row_mirror_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 321, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_half_row_mirror_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_half_row_mirror_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low3 = and i32 %lane, 7
%rev = sub i32 7, %low3
%high = and i32 %lane, -8
%idx = or i32 %high, %rev
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Full-row mirror: reverse within every 16-lane row.
define i32 @test_full_row_mirror_w32(i32 %val) {
; GFX11-LABEL: @test_full_row_mirror_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 320, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_full_row_mirror_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_full_row_mirror_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
%rev = sub i32 15, %low4
%high = and i32 %lane, -16
%idx = or i32 %high, %rev
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Row rotate right by 3 within each 16-lane row.
define i32 @test_row_rotate_right3_w32(i32 %val) {
; GFX11-LABEL: @test_row_rotate_right3_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 301, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_rotate_right3_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_row_rotate_right3_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX9-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
%rot = add i32 %low4, 3
%wrapped = and i32 %rot, 15
%high = and i32 %lane, -16
%idx = or i32 %high, %wrapped
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Row share: broadcast lane 3 within each 16-lane row.
define i32 @test_row_share3_w32(i32 %val) {
; GFX11-LABEL: @test_row_share3_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_share3_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_row_share3_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%high = and i32 %lane, -16
%idx = or i32 %high, 3
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Row xmask: xor each lane with mask 5 within each 16-lane row.
define i32 @test_row_xmask5_w32(i32 %val) {
; GFX11-LABEL: @test_row_xmask5_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_xmask5_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_row_xmask5_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
%xored = xor i32 %low4, 5
%high = and i32 %lane, -16
%idx = or i32 %high, %xored
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Half-wave swap (wave64 only): lane ^ 32.
define i32 @test_half_wave_swap_w64(i32 %val) {
; GFX11-LABEL: @test_half_wave_swap_w64(
; GFX11-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LO]], 32
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_half_wave_swap_w64(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_half_wave_swap_w64(
; GFX9-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 32
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%idx = xor i32 %lane, 32
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; DPP8: rotate left by 1 within each 8-lane group.
define i32 @test_dpp8_rotate1_w32(i32 %val) {
; GFX11-LABEL: @test_dpp8_rotate1_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL:%.*]], i32 2054353)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_dpp8_rotate1_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_dpp8_rotate1_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX9-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low3 = and i32 %lane, 7
%rot = add i32 %low3, 1
%wrapped = and i32 %rot, 7
%high = and i32 %lane, -8
%idx = or i32 %high, %wrapped
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Permlane16: (lane * 3) & 15, a bijection on Z/16Z.
define i32 @test_permlane16_mul3_w32(i32 %val) {
; GFX11-LABEL: @test_permlane16_mul3_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane16.i32(i32 poison, i32 [[VAL:%.*]], i32 1392285232, i32 -629924168, i1 false, i1 false)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlane16_mul3_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
; GFX11-W64-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlane16_mul3_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
; GFX9-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
%mul3 = mul i32 %low4, 3
%perm = and i32 %mul3, 15
%high = and i32 %lane, -16
%idx = or i32 %high, %perm
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; PermlaneX16: lane ^ 16. Cross-row swap; identity selectors.
define i32 @test_permlanex16_xor16_w32(i32 %val) {
; GFX11-LABEL: @test_permlanex16_xor16_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlanex16_xor16_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 16
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlanex16_xor16_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 16
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 16
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; PermlaneX16: lane ^ 17. Cross-row + XOR-by-1 within row.
define i32 @test_permlanex16_xor17_w32(i32 %val) {
; GFX11-LABEL: @test_permlanex16_xor17_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1732584193, i32 -271733879, i1 false, i1 false)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlanex16_xor17_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 17
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlanex16_xor17_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 17
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 17
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; PermlaneX16: lane ^ 31. Cross-row + reverse within row.
define i32 @test_permlanex16_xor31_w32(i32 %val) {
; GFX11-LABEL: @test_permlanex16_xor31_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 -1985229329, i32 19088743, i1 false, i1 false)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlanex16_xor31_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 31
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlanex16_xor31_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 31
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 31
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Float type should work the same as i32.
define float @test_quad_perm_xor1_float_w32(float %val) {
; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float poison, float [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret float [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_float_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret float [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_float_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret float [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
ret float %result
}
; LDS (32-bit) pointer should optimize the same as i32: the gate must not
; reject 32-bit pointer types via Type::getPrimitiveSizeInBits().
define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
; GFX11-LABEL: @test_quad_perm_xor1_ptr3_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) poison, ptr addrspace(3) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret ptr addrspace(3) [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr3_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret ptr addrspace(3) [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_ptr3_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret ptr addrspace(3) [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) %val, i32 %idx)
ret ptr addrspace(3) %result
}
; Scratch (32-bit) pointer, should optimize like other 32-bit pointers.
define ptr addrspace(5) @test_quad_perm_xor1_ptr5_w32(ptr addrspace(5) %val) {
; GFX11-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) poison, ptr addrspace(5) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret ptr addrspace(5) [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret ptr addrspace(5) [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret ptr addrspace(5) [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) %val, i32 %idx)
ret ptr addrspace(5) %result
}
; <2 x i16> vector (32-bit), should optimize like i32.
define <2 x i16> @test_quad_perm_xor1_v2i16_w32(<2 x i16> %val) {
; GFX11-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.update.dpp.v2i16(<2 x i16> poison, <2 x i16> [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret <2 x i16> [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret <2 x i16> [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret <2 x i16> [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
ret <2 x i16> %result
}
; Half-row-share with <2 x i16>: falls back to ds_swizzle on GFX9, using the bitcast path.
define <2 x i16> @test_half_row_share3_v2i16(<2 x i16> %val) {
; GFX11-LABEL: @test_half_row_share3_v2i16(
; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
; GFX11-NEXT: ret <2 x i16> [[RESULT]]
;
; GFX11-W64-LABEL: @test_half_row_share3_v2i16(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
; GFX11-W64-NEXT: ret <2 x i16> [[RESULT]]
;
; GFX9-LABEL: @test_half_row_share3_v2i16(
; GFX9-NEXT: [[TMP1:%.*]] = bitcast <2 x i16> [[VAL:%.*]] to i32
; GFX9-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[TMP1]], i32 120)
; GFX9-NEXT: [[RESULT:%.*]] = bitcast i32 [[TMP2]] to <2 x i16>
; GFX9-NEXT: ret <2 x i16> [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%masked = and i32 %tid, -8
%idx = or i32 %masked, 3
%result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
ret <2 x i16> %result
}
; Negative: 64-bit type, should not optimize.
define i64 @test_quad_perm_xor1_i64_negative(i64 %val) {
; GFX11-LABEL: @test_quad_perm_xor1_i64_negative(
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT: ret i64 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_i64_negative(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i64 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_i64_negative(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i64 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 %val, i32 %idx)
ret i64 %result
}
; Negative: 64-bit pointer, should not optimize.
define ptr @test_quad_perm_xor1_p0_negative(ptr %val) {
; GFX11-LABEL: @test_quad_perm_xor1_p0_negative(
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT: ret ptr [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_p0_negative(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret ptr [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_p0_negative(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret ptr [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr %val, i32 %idx)
ret ptr %result
}
; Negative: 16-bit type, should not optimize.
define i16 @test_quad_perm_xor1_i16_negative(i16 %val) {
; GFX11-LABEL: @test_quad_perm_xor1_i16_negative(
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT: ret i16 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_i16_negative(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i16 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_i16_negative(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i16 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%result = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 %val, i32 %idx)
ret i16 %result
}
; Negative: poison index, should not optimize.
define i32 @test_poison_index_negative(i32 %val) {
; GFX11-LABEL: @test_poison_index_negative(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_poison_index_negative(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_poison_index_negative(
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
; GFX9-NEXT: ret i32 [[RESULT]]
;
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 poison)
ret i32 %result
}
; Negative: non-constant index, should not be optimized.
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_nonconstant_shuffle_w32(
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Identity shuffle: lane_id itself.
define i32 @test_identity_shuffle_w32(i32 %val) {
; GFX11-LABEL: @test_identity_shuffle_w32(
; GFX11-NEXT: ret i32 [[RESULT:%.*]]
;
; GFX11-W64-LABEL: @test_identity_shuffle_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_identity_shuffle_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
ret i32 %result
}
; Negative: index out of range (lane + 32 >= 32 for wave32).
define i32 @test_out_of_range_w32(i32 %val) {
; GFX11-LABEL: @test_out_of_range_w32(
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_out_of_range_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_out_of_range_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = add i32 %lane, 32
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; ds_bpermute with a constant shuffle pattern (quad perm via byte address).
define i32 @test_bpermute_quad_xor1_w32(i32 %val) {
; GFX11-LABEL: @test_bpermute_quad_xor1_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_bpermute_quad_xor1_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
; GFX11-W64-NEXT: [[ADDR:%.*]] = xor i32 [[IDX]], 4
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_bpermute_quad_xor1_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
; GFX9-NEXT: [[ADDR:%.*]] = xor i32 [[IDX]], 4
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
%addr = shl i32 %idx, 2
%result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
ret i32 %result
}
; XOR-by-5 within each 16-lane row. On GFX10+ this matches DPP row_xmask;
; on GFX9 row_xmask is unavailable, so the generic ds_swizzle bitmask path
; takes over: AND=0x1F, OR=0, XOR=5 -> 5151.
define i32 @test_xor5_bitmask(i32 %val) {
; GFX11-LABEL: @test_xor5_bitmask(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_xor5_bitmask(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_xor5_bitmask(
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 5151)
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%idx = xor i32 %lane, 5
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; XOR-by-16 (cross-row). On GFX11 this matches permlanex16; on GFX9 neither
; permlanex16 nor row_xmask exist, so the bitmask path catches it:
; AND=0x1F, OR=0, XOR=16 -> 16415.
define i32 @test_xor16_bitmask(i32 %val) {
; GFX11-LABEL: @test_xor16_bitmask(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_xor16_bitmask(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_xor16_bitmask(
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 16415)
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%idx = xor i32 %lane, 16
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Broadcast lane 5 within each 16-lane row (lanes 0..15 read lane 5, lanes
; 16..31 read lane 21, etc.). On GFX10+ row_share covers it; on GFX9 there
; is no row_share, so the bitmask path emits AND=0x10, OR=5, XOR=0 -> 176.
define i32 @test_broadcast_in_rows16_bitmask(i32 %val) {
; GFX11-LABEL: @test_broadcast_in_rows16_bitmask(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_broadcast_in_rows16_bitmask(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_broadcast_in_rows16_bitmask(
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 176)
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%hi = and i32 %lane, -16
%idx = or i32 %hi, 5
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Rotate-left-by-1 using mbcnt.lo only (wave32 ID form); folds to ds_swizzle
; rotate mode (imm=0xC020) on GFX11 wave32, but not on wave64 targets (GFX11-W64,
; GFX9) where mbcnt.lo alone is not the full lane ID.
define i32 @test_rotate_add1_w32(i32 %val) {
; GFX11-LABEL: @test_rotate_add1_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 49184)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_add1_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX11-W64-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_rotate_add1_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX9-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%add = add i32 %lane, 1
%idx = and i32 %add, 31
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Rotate-right-by-1 canonicalized to rotate-left-by-31; folds to ds_swizzle
; rotate mode (imm=0xC3E0) on GFX11 wave32, but not on wave64 targets (GFX11-W64,
; GFX9) where mbcnt.lo alone is not the full lane ID.
define i32 @test_rotate_sub1_w32(i32 %val) {
; GFX11-LABEL: @test_rotate_sub1_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 50144)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_sub1_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 31
; GFX11-W64-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_rotate_sub1_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 31
; GFX9-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%add = add i32 %lane, 31
%idx = and i32 %add, 31
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Same rotate-left-by-1 via ds_bpermute (byte-addressed); folds to ds_swizzle
; rotate mode (imm=0xC020) on GFX11 wave32, but not on wave64 targets (GFX11-W64,
; GFX9) where mbcnt.lo alone is not the full lane ID.
define i32 @test_bpermute_rotate_add1_w32(i32 %val) {
; GFX11-LABEL: @test_bpermute_rotate_add1_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 49184)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_bpermute_rotate_add1_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ADD:%.*]] = shl nuw nsw i32 [[LANE]], 2
; GFX11-W64-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[ADD]], 4
; GFX11-W64-NEXT: [[ADDR:%.*]] = and i32 [[IDX]], 124
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_bpermute_rotate_add1_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ADD:%.*]] = shl nuw nsw i32 [[LANE]], 2
; GFX9-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[ADD]], 4
; GFX9-NEXT: [[ADDR:%.*]] = and i32 [[IDX]], 124
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%add = add i32 %lane, 1
%idx = and i32 %add, 31
%addr = shl i32 %idx, 2
%result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
ret i32 %result
}
; Wave64 rotate-left-by-1 with both 32-lane groups rotating independently;
; folds to ds_swizzle rotate mode on all targets.
define i32 @test_rotate_add1_w64(i32 %val) {
; GFX11-LABEL: @test_rotate_add1_w64(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 49184)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_add1_w64(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 49184)
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_rotate_add1_w64(
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 49184)
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%low5 = and i32 %lane, 31
%add = add i32 %low5, 1
%rot = and i32 %add, 31
%hi = and i32 %lane, 32
%idx = or i32 %rot, %hi
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Negative: full-wave64 rotate crosses the 32-lane group boundary, so
; hasPeriodicLayout<32> rejects it and no fold occurs.
define i32 @test_rotate_add1_full_w64(i32 %val) {
; GFX11-LABEL: @test_rotate_add1_full_w64(
; GFX11-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LO]], 1
; GFX11-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 63
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_add1_full_w64(
; GFX11-W64-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; GFX11-W64-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX11-W64-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 63
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_rotate_add1_full_w64(
; GFX9-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; GFX9-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX9-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 63
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%add = add i32 %lane, 1
%idx = and i32 %add, 63
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Identity shuffle (add %lane,0 folds to %lane). On GFX11 wave32 this folds to
; the source value; on wave64 targets (GFX11-W64, GFX9) mbcnt.lo alone is not
; the full lane ID so no fold occurs.
define i32 @test_identity_w32(i32 %val) {
; GFX11-LABEL: @test_identity_w32(
; GFX11-NEXT: ret i32 [[RESULT:%.*]]
;
; GFX11-W64-LABEL: @test_identity_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[IDX:%.*]] = and i32 [[LANE]], 31
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_identity_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[IDX:%.*]] = and i32 [[LANE]], 31
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%add = add i32 %lane, 0
%idx = and i32 %add, 31
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Two 32-lane halves rotate by different amounts (low half by 1, high half by 2).
; On wave64 hasPeriodicLayout<32> rejects the non-uniform pattern so no fold
; occurs. On GFX11 wave32 only one 32-lane group exists so the high-half shift
; is never evaluated; the pattern collapses to a uniform rotate-by-1 and folds.
define i32 @test_rotate_asymmetric_w64(i32 %val) {
; GFX11-LABEL: @test_rotate_asymmetric_w64(
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 49184)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_asymmetric_w64(
; GFX11-W64-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; GFX11-W64-NEXT: [[HI:%.*]] = and i32 [[LANE]], 32
; GFX11-W64-NEXT: [[HIGH_EXTRA:%.*]] = lshr exact i32 [[HI]], 5
; GFX11-W64-NEXT: [[SHIFT:%.*]] = add nuw nsw i32 [[HIGH_EXTRA]], 1
; GFX11-W64-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], [[SHIFT]]
; GFX11-W64-NEXT: [[ROT:%.*]] = and i32 [[ADD]], 31
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[ROT]], [[HI]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_rotate_asymmetric_w64(
; GFX9-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; GFX9-NEXT: [[HI:%.*]] = and i32 [[LANE]], 32
; GFX9-NEXT: [[HIGH_EXTRA:%.*]] = lshr exact i32 [[HI]], 5
; GFX9-NEXT: [[SHIFT:%.*]] = add nuw nsw i32 [[HIGH_EXTRA]], 1
; GFX9-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], [[SHIFT]]
; GFX9-NEXT: [[ROT:%.*]] = and i32 [[ADD]], 31
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[ROT]], [[HI]]
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
%low5 = and i32 %lane, 31
%hi = and i32 %lane, 32
%high_extra = lshr i32 %hi, 5
%shift = add i32 %high_extra, 1
%add = add i32 %low5, %shift
%rot = and i32 %add, 31
%idx = or i32 %rot, %hi
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
; Rotate-left-by-3 using an f32 source; createDsSwizzle bitcasts to i32.
define float @test_rotate_add3_f32_w32(float %val) {
; GFX11-LABEL: @test_rotate_add3_f32_w32(
; GFX11-NEXT: [[TMP1:%.*]] = bitcast float [[VAL:%.*]] to i32
; GFX11-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[TMP1]], i32 49248)
; GFX11-NEXT: [[RESULT:%.*]] = bitcast i32 [[TMP2]] to float
; GFX11-NEXT: ret float [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_add3_f32_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX11-W64-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX11-W64-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret float [[RESULT]]
;
; GFX9-LABEL: @test_rotate_add3_f32_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX9-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX9-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret float [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%add = add i32 %lane, 3
%idx = and i32 %add, 31
%result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
ret float %result
}
; Rotate-left-by-3 using a local-memory pointer source; createDsSwizzle
; ptrtoint/inttoptr-converts around the i32 ds_swizzle.
define ptr addrspace(3) @test_rotate_add3_ptr_w32(ptr addrspace(3) %val) {
; GFX11-LABEL: @test_rotate_add3_ptr_w32(
; GFX11-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[VAL:%.*]] to i32
; GFX11-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[TMP1]], i32 49248)
; GFX11-NEXT: [[RESULT:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(3)
; GFX11-NEXT: ret ptr addrspace(3) [[RESULT]]
;
; GFX11-W64-LABEL: @test_rotate_add3_ptr_w32(
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX11-W64-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret ptr addrspace(3) [[RESULT]]
;
; GFX9-LABEL: @test_rotate_add3_ptr_w32(
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX9-NEXT: [[IDX:%.*]] = and i32 [[ADD]], 31
; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT: ret ptr addrspace(3) [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%add = add i32 %lane, 3
%idx = and i32 %add, 31
%result = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) %val, i32 %idx)
ret ptr addrspace(3) %result
}