[GIsel] Add combine (sub a, (mul x, C)) -> (add a, (mul x, -C)) (#194282)
Copy this canonicalization from InstCombine so it can run on
post-legalized expansions. This is especially useful if the sub is a
neg.
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 6370197..2091554 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -359,6 +359,17 @@
[{ return Helper.matchCombineSubToAdd(*${mi}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFnNoErase(*${mi}, ${matchinfo}); }])>;
+// Fold (sub a, (mul x, C)) -> (add a, (mul x, -C)), if mul has one use.
+def sub_of_mul_const : GICombineRule<
+ (defs root:$d),
+ (match (G_CONSTANT $c, $imm),
+ (G_MUL $mul, $x, $c),
+ (G_SUB $d, $a, $mul),
+ [{ return MRI.hasOneNonDBGUse(${mul}.getReg()); }]),
+ (apply (G_SUB $negc, 0, $c),
+ (G_MUL $newmul, $x, $negc),
+ (G_ADD $d, $a, $newmul))>;
+
// (sub -1, x) -> (xor x, -1)
def sub_minus_one : GICombineRule<
(defs root:$root),
@@ -2389,7 +2400,7 @@
sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
combine_use_vector_truncate, merge_combines, overflow_combines,
truncsat_combines, lshr_of_trunc_of_lshr, ctls_combines, add_shift, sub_one_from_sub,
- binop_with_neg, sub_minus_one]>;
+ binop_with_neg, sub_minus_one, sub_of_mul_const]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-sub-of-mul-const.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-sub-of-mul-const.ll
new file mode 100644
index 0000000..2ebe27f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-sub-of-mul-const.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+; (sub a, (mul x, K)) where K is a constant lowers to MADD with -K.
+
+; Basic positive K: madd with -7.
+define i32 @sub_of_mul_const_pos_k(i32 %x, i32 %a) {
+; CHECK-LABEL: sub_of_mul_const_pos_k:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub w8, w0, w0, lsl #3
+; CHECK-NEXT: add w0, w1, w8
+; CHECK-NEXT: ret
+ %mul = mul i32 %x, 7
+ %sub = sub i32 %a, %mul
+ ret i32 %sub
+}
+
+; Negative K: madd with +7.
+define i32 @sub_of_mul_const_neg_k(i32 %x, i32 %a) {
+; CHECK-LABEL: sub_of_mul_const_neg_k:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w8, w0, #3
+; CHECK-NEXT: sub w8, w8, w0
+; CHECK-NEXT: add w0, w1, w8
+; CHECK-NEXT: ret
+ %mul = mul i32 %x, -7
+ %sub = sub i32 %a, %mul
+ ret i32 %sub
+}
+
+; A = 0: collapses via combine_add_with_zero to mul x, -K.
+define i32 @sub_of_mul_const_zero_a(i32 %x) {
+; CHECK-LABEL: sub_of_mul_const_zero_a:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub w0, w0, w0, lsl #3
+; CHECK-NEXT: ret
+ %mul = mul i32 %x, 7
+ %sub = sub i32 0, %mul
+ ret i32 %sub
+}
+
+; i64 width.
+define i64 @sub_of_mul_const_i64(i64 %x, i64 %a) {
+; CHECK-LABEL: sub_of_mul_const_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, x0, lsl #3
+; CHECK-NEXT: add x0, x1, x8
+; CHECK-NEXT: ret
+ %mul = mul i64 %x, 7
+ %sub = sub i64 %a, %mul
+ ret i64 %sub
+}
+
+; K is not a constant: bail (msub with the original operand).
+define i32 @sub_of_mul_non_const(i32 %x, i32 %y, i32 %a) {
+; CHECK-LABEL: sub_of_mul_non_const:
+; CHECK: // %bb.0:
+; CHECK-NEXT: msub w0, w0, w1, w2
+; CHECK-NEXT: ret
+ %mul = mul i32 %x, %y
+ %sub = sub i32 %a, %mul
+ ret i32 %sub
+}
+
+; Mul has multiple users: bail (would either duplicate or regress).
+define i32 @sub_of_mul_const_multi_use(i32 %x, i32 %a, ptr %p) {
+; CHECK-LABEL: sub_of_mul_const_multi_use:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl w8, w0, #3
+; CHECK-NEXT: sub w8, w8, w0
+; CHECK-NEXT: sub w0, w1, w8
+; CHECK-NEXT: str w8, [x2]
+; CHECK-NEXT: ret
+ %mul = mul i32 %x, 7
+ %sub = sub i32 %a, %mul
+ store i32 %mul, ptr %p
+ ret i32 %sub
+}
+
+; Vector splat: not handled in this scalar-only first cut.
+define <4 x i32> @sub_of_mul_const_vector_splat(<4 x i32> %x, <4 x i32> %a) {
+; CHECK-LABEL: sub_of_mul_const_vector_splat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #7
+; CHECK-NEXT: mls v1.4s, v0.4s, v2.4s
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %mul = mul <4 x i32> %x, <i32 7, i32 7, i32 7, i32 7>
+ %sub = sub <4 x i32> %a, %mul
+ ret <4 x i32> %sub
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-mul.ll b/llvm/test/CodeGen/AArch64/arm64-mul.ll
index 5ae27226..20b1c9f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-mul.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-mul.ll
@@ -152,12 +152,19 @@
; Check the sext_inreg case.
define i64 @t11(i64 %a) nounwind {
-; CHECK-LABEL: t11:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w8, #29594 // =0x739a
-; CHECK-NEXT: movk w8, #65499, lsl #16
-; CHECK-NEXT: smnegl x0, w0, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: t11:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov w8, #29594 // =0x739a
+; CHECK-SD-NEXT: movk w8, #65499, lsl #16
+; CHECK-SD-NEXT: smnegl x0, w0, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: t11:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, #35942 // =0x8c66
+; CHECK-GI-NEXT: movk w8, #36, lsl #16
+; CHECK-GI-NEXT: smull x0, w0, w8
+; CHECK-GI-NEXT: ret
entry:
%tmp1 = trunc i64 %a to i32
%tmp2 = sext i32 %tmp1 to i64
@@ -182,12 +189,20 @@
}
define i64 @t13(i32 %a, i64 %b) nounwind {
-; CHECK-LABEL: t13:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w8, #24910 // =0x614e
-; CHECK-NEXT: movk w8, #188, lsl #16
-; CHECK-NEXT: umsubl x0, w0, w8, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: t13:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov w8, #24910 // =0x614e
+; CHECK-SD-NEXT: movk w8, #188, lsl #16
+; CHECK-SD-NEXT: umsubl x0, w0, w8, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: t13:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x8, #-24910 // =0xffffffffffff9eb2
+; CHECK-GI-NEXT: mov w9, w0
+; CHECK-GI-NEXT: movk x8, #65347, lsl #16
+; CHECK-GI-NEXT: madd x0, x9, x8, x1
+; CHECK-GI-NEXT: ret
entry:
%tmp1 = zext i32 %a to i64
%tmp3 = mul i64 %tmp1, 12345678
@@ -196,12 +211,19 @@
}
define i64 @t14(i32 %a, i64 %b) nounwind {
-; CHECK-LABEL: t14:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w8, #40626 // =0x9eb2
-; CHECK-NEXT: movk w8, #65347, lsl #16
-; CHECK-NEXT: smsubl x0, w0, w8, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: t14:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov w8, #40626 // =0x9eb2
+; CHECK-SD-NEXT: movk w8, #65347, lsl #16
+; CHECK-SD-NEXT: smsubl x0, w0, w8, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: t14:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, #24910 // =0x614e
+; CHECK-GI-NEXT: movk w8, #188, lsl #16
+; CHECK-GI-NEXT: smaddl x0, w0, w8, x1
+; CHECK-GI-NEXT: ret
entry:
%tmp1 = sext i32 %a to i64
%tmp3 = mul i64 %tmp1, -12345678
diff --git a/llvm/test/CodeGen/AArch64/mul_pow2.ll b/llvm/test/CodeGen/AArch64/mul_pow2.ll
index 07becb3..8e33141 100644
--- a/llvm/test/CodeGen/AArch64/mul_pow2.ll
+++ b/llvm/test/CodeGen/AArch64/mul_pow2.ll
@@ -109,11 +109,17 @@
}
define i32 @test6_msub(i32 %x, i32 %y) {
-; CHECK-LABEL: test6_msub:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #6 // =0x6
-; CHECK-NEXT: msub w0, w0, w8, w1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6_msub:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov w8, #6 // =0x6
+; CHECK-SD-NEXT: msub w0, w0, w8, w1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6_msub:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #-6 // =0xfffffffa
+; CHECK-GI-NEXT: madd w0, w0, w8, w1
+; CHECK-GI-NEXT: ret
%mul = mul nsw i32 %x, 6
%sub = sub i32 %y, %mul
@@ -147,11 +153,18 @@
}
define i64 @test6_umsubl(i32 %x, i64 %y) {
-; CHECK-LABEL: test6_umsubl:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #6 // =0x6
-; CHECK-NEXT: umsubl x0, w0, w8, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6_umsubl:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov w8, #6 // =0x6
+; CHECK-SD-NEXT: umsubl x0, w0, w8, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6_umsubl:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov x8, #-6 // =0xfffffffffffffffa
+; CHECK-GI-NEXT: mov w9, w0
+; CHECK-GI-NEXT: madd x0, x9, x8, x1
+; CHECK-GI-NEXT: ret
%ext = zext i32 %x to i64
%mul = mul nsw i64 %ext, 6
@@ -160,11 +173,17 @@
}
define i64 @test6_smsubl(i32 %x, i64 %y) {
-; CHECK-LABEL: test6_smsubl:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #6 // =0x6
-; CHECK-NEXT: smsubl x0, w0, w8, x1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6_smsubl:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov w8, #6 // =0x6
+; CHECK-SD-NEXT: smsubl x0, w0, w8, x1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6_smsubl:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #-6 // =0xfffffffa
+; CHECK-GI-NEXT: smaddl x0, w0, w8, x1
+; CHECK-GI-NEXT: ret
%ext = sext i32 %x to i64
%mul = mul nsw i64 %ext, 6
@@ -173,11 +192,18 @@
}
define i64 @test6_umnegl(i32 %x) {
-; CHECK-LABEL: test6_umnegl:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #6 // =0x6
-; CHECK-NEXT: umnegl x0, w0, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6_umnegl:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov w8, #6 // =0x6
+; CHECK-SD-NEXT: umnegl x0, w0, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6_umnegl:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov x8, #-6 // =0xfffffffffffffffa
+; CHECK-GI-NEXT: mov w9, w0
+; CHECK-GI-NEXT: mul x0, x9, x8
+; CHECK-GI-NEXT: ret
%ext = zext i32 %x to i64
%mul = mul nsw i64 %ext, 6
@@ -186,11 +212,17 @@
}
define i64 @test6_smnegl(i32 %x) {
-; CHECK-LABEL: test6_smnegl:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #6 // =0x6
-; CHECK-NEXT: smnegl x0, w0, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test6_smnegl:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov w8, #6 // =0x6
+; CHECK-SD-NEXT: smnegl x0, w0, w8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test6_smnegl:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #-6 // =0xfffffffa
+; CHECK-GI-NEXT: smull x0, w0, w8
+; CHECK-GI-NEXT: ret
%ext = sext i32 %x to i64
%mul = mul nsw i64 %ext, 6
diff --git a/llvm/test/CodeGen/AArch64/pr58431.ll b/llvm/test/CodeGen/AArch64/pr58431.ll
index a373004..ea04c9c 100644
--- a/llvm/test/CodeGen/AArch64/pr58431.ll
+++ b/llvm/test/CodeGen/AArch64/pr58431.ll
@@ -6,10 +6,10 @@
; CHECK: // %bb.0:
; CHECK-NEXT: mov x8, #-7378697629483820647 // =0x9999999999999999
; CHECK-NEXT: mov w9, w0
-; CHECK-NEXT: mov w10, #10 // =0xa
+; CHECK-NEXT: mov x10, #-10 // =0xfffffffffffffff6
; CHECK-NEXT: eor x8, x8, #0x8000000000000003
; CHECK-NEXT: umulh x8, x9, x8
-; CHECK-NEXT: umsubl x0, w8, w10, x9
+; CHECK-NEXT: smaddl x0, w8, w10, x9
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%2 = trunc i64 %0 to i32
diff --git a/llvm/test/CodeGen/AArch64/rem-by-const.ll b/llvm/test/CodeGen/AArch64/rem-by-const.ll
index 1c6b241..eb8b9c2 100644
--- a/llvm/test/CodeGen/AArch64/rem-by-const.ll
+++ b/llvm/test/CodeGen/AArch64/rem-by-const.ll
@@ -26,9 +26,8 @@
; CHECK-GI-NEXT: sbfx w8, w8, #2, #6
; CHECK-GI-NEXT: ubfx w9, w8, #7, #1
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: lsl w9, w8, #3
-; CHECK-GI-NEXT: sub w8, w9, w8
-; CHECK-GI-NEXT: sub w0, w0, w8
+; CHECK-GI-NEXT: sub w8, w8, w8, lsl #3
+; CHECK-GI-NEXT: add w0, w0, w8
; CHECK-GI-NEXT: ret
entry:
%s = srem i8 %a, 7
@@ -56,8 +55,8 @@
; CHECK-GI-NEXT: asr w8, w8, #4
; CHECK-GI-NEXT: ubfx w9, w8, #7, #1
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: mov w9, #100 // =0x64
-; CHECK-GI-NEXT: msub w0, w8, w9, w0
+; CHECK-GI-NEXT: mov w9, #-100 // =0xffffff9c
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
; CHECK-GI-NEXT: ret
entry:
%s = srem i8 %a, 100
@@ -89,9 +88,8 @@
; CHECK-GI-NEXT: ubfx w9, w9, #1, #7
; CHECK-GI-NEXT: add w8, w9, w8
; CHECK-GI-NEXT: lsr w8, w8, #2
-; CHECK-GI-NEXT: lsl w9, w8, #3
-; CHECK-GI-NEXT: sub w8, w9, w8
-; CHECK-GI-NEXT: sub w0, w0, w8
+; CHECK-GI-NEXT: sub w8, w8, w8, lsl #3
+; CHECK-GI-NEXT: add w0, w0, w8
; CHECK-GI-NEXT: ret
entry:
%s = urem i8 %a, 7
@@ -114,10 +112,10 @@
; CHECK-GI-NEXT: mov w8, #41 // =0x29
; CHECK-GI-NEXT: and w9, w0, #0xff
; CHECK-GI-NEXT: mul w8, w9, w8
-; CHECK-GI-NEXT: mov w9, #100 // =0x64
+; CHECK-GI-NEXT: mov w9, #-100 // =0xffffff9c
; CHECK-GI-NEXT: lsr w8, w8, #8
; CHECK-GI-NEXT: lsr w8, w8, #4
-; CHECK-GI-NEXT: msub w0, w8, w9, w0
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
; CHECK-GI-NEXT: ret
entry:
%s = urem i8 %a, 100
@@ -145,9 +143,8 @@
; CHECK-GI-NEXT: asr w8, w8, #1
; CHECK-GI-NEXT: ubfx w9, w8, #15, #1
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: lsl w9, w8, #3
-; CHECK-GI-NEXT: sub w8, w9, w8
-; CHECK-GI-NEXT: sub w0, w0, w8
+; CHECK-GI-NEXT: sub w8, w8, w8, lsl #3
+; CHECK-GI-NEXT: add w0, w0, w8
; CHECK-GI-NEXT: ret
entry:
%s = srem i16 %a, 7
@@ -175,8 +172,8 @@
; CHECK-GI-NEXT: asr w8, w8, #3
; CHECK-GI-NEXT: ubfx w9, w8, #15, #1
; CHECK-GI-NEXT: add w8, w8, w9
-; CHECK-GI-NEXT: mov w9, #100 // =0x64
-; CHECK-GI-NEXT: msub w0, w8, w9, w0
+; CHECK-GI-NEXT: mov w9, #-100 // =0xffffff9c
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
; CHECK-GI-NEXT: ret
entry:
%s = srem i16 %a, 100
@@ -208,9 +205,8 @@
; CHECK-GI-NEXT: ubfx w9, w9, #1, #15
; CHECK-GI-NEXT: add w8, w9, w8
; CHECK-GI-NEXT: lsr w8, w8, #2
-; CHECK-GI-NEXT: lsl w9, w8, #3
-; CHECK-GI-NEXT: sub w8, w9, w8
-; CHECK-GI-NEXT: sub w0, w0, w8
+; CHECK-GI-NEXT: sub w8, w8, w8, lsl #3
+; CHECK-GI-NEXT: add w0, w0, w8
; CHECK-GI-NEXT: ret
entry:
%s = urem i16 %a, 7
@@ -233,10 +229,10 @@
; CHECK-GI-NEXT: ubfx w8, w0, #2, #14
; CHECK-GI-NEXT: mov w9, #5243 // =0x147b
; CHECK-GI-NEXT: mul w8, w8, w9
-; CHECK-GI-NEXT: mov w9, #100 // =0x64
+; CHECK-GI-NEXT: mov w9, #-100 // =0xffffff9c
; CHECK-GI-NEXT: lsr w8, w8, #16
; CHECK-GI-NEXT: lsr w8, w8, #1
-; CHECK-GI-NEXT: msub w0, w8, w9, w0
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
; CHECK-GI-NEXT: ret
entry:
%s = urem i16 %a, 100
@@ -266,9 +262,8 @@
; CHECK-GI-NEXT: add x8, x0, x8, asr #32
; CHECK-GI-NEXT: asr w8, w8, #2
; CHECK-GI-NEXT: add w8, w8, w8, lsr #31
-; CHECK-GI-NEXT: lsl w9, w8, #3
-; CHECK-GI-NEXT: sub w8, w9, w8
-; CHECK-GI-NEXT: sub w0, w0, w8
+; CHECK-GI-NEXT: sub w8, w8, w8, lsl #3
+; CHECK-GI-NEXT: add w0, w0, w8
; CHECK-GI-NEXT: ret
entry:
%s = srem i32 %a, 7
@@ -290,13 +285,13 @@
; CHECK-GI-LABEL: si32_100:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov w8, #34079 // =0x851f
-; CHECK-GI-NEXT: mov w9, #100 // =0x64
+; CHECK-GI-NEXT: mov w9, #-100 // =0xffffff9c
; CHECK-GI-NEXT: movk w8, #20971, lsl #16
; CHECK-GI-NEXT: smull x8, w0, w8
; CHECK-GI-NEXT: asr x8, x8, #32
; CHECK-GI-NEXT: asr w8, w8, #5
; CHECK-GI-NEXT: add w8, w8, w8, lsr #31
-; CHECK-GI-NEXT: msub w0, w8, w9, w0
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
; CHECK-GI-NEXT: ret
entry:
%s = srem i32 %a, 100
@@ -324,9 +319,8 @@
; CHECK-GI-NEXT: sub w9, w0, w8
; CHECK-GI-NEXT: add w8, w8, w9, lsr #1
; CHECK-GI-NEXT: lsr w8, w8, #2
-; CHECK-GI-NEXT: lsl w9, w8, #3
-; CHECK-GI-NEXT: sub w8, w9, w8
-; CHECK-GI-NEXT: sub w0, w0, w8
+; CHECK-GI-NEXT: sub w8, w8, w8, lsl #3
+; CHECK-GI-NEXT: add w0, w0, w8
; CHECK-GI-NEXT: ret
entry:
%s = urem i32 %a, 7
@@ -334,15 +328,25 @@
}
define i32 @ui32_100(i32 %a, i32 %b) {
-; CHECK-LABEL: ui32_100:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w8, #34079 // =0x851f
-; CHECK-NEXT: mov w9, #100 // =0x64
-; CHECK-NEXT: movk w8, #20971, lsl #16
-; CHECK-NEXT: umull x8, w0, w8
-; CHECK-NEXT: lsr x8, x8, #37
-; CHECK-NEXT: msub w0, w8, w9, w0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: ui32_100:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT: mov w9, #100 // =0x64
+; CHECK-SD-NEXT: movk w8, #20971, lsl #16
+; CHECK-SD-NEXT: umull x8, w0, w8
+; CHECK-SD-NEXT: lsr x8, x8, #37
+; CHECK-SD-NEXT: msub w0, w8, w9, w0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: ui32_100:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, #34079 // =0x851f
+; CHECK-GI-NEXT: mov w9, #-100 // =0xffffff9c
+; CHECK-GI-NEXT: movk w8, #20971, lsl #16
+; CHECK-GI-NEXT: umull x8, w0, w8
+; CHECK-GI-NEXT: lsr x8, x8, #37
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
+; CHECK-GI-NEXT: ret
entry:
%s = urem i32 %a, 100
ret i32 %s
@@ -371,9 +375,8 @@
; CHECK-GI-NEXT: smulh x8, x0, x8
; CHECK-GI-NEXT: asr x8, x8, #1
; CHECK-GI-NEXT: add x8, x8, x8, lsr #63
-; CHECK-GI-NEXT: lsl x9, x8, #3
-; CHECK-GI-NEXT: sub x8, x9, x8
-; CHECK-GI-NEXT: sub x0, x0, x8
+; CHECK-GI-NEXT: sub x8, x8, x8, lsl #3
+; CHECK-GI-NEXT: add x0, x0, x8
; CHECK-GI-NEXT: ret
entry:
%s = srem i64 %a, 7
@@ -398,7 +401,7 @@
; CHECK-GI-LABEL: si64_100:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mov x8, #55051 // =0xd70b
-; CHECK-GI-NEXT: mov w9, #100 // =0x64
+; CHECK-GI-NEXT: mov x9, #-100 // =0xffffffffffffff9c
; CHECK-GI-NEXT: movk x8, #28835, lsl #16
; CHECK-GI-NEXT: movk x8, #2621, lsl #32
; CHECK-GI-NEXT: movk x8, #41943, lsl #48
@@ -406,7 +409,7 @@
; CHECK-GI-NEXT: add x8, x8, x0
; CHECK-GI-NEXT: asr x8, x8, #6
; CHECK-GI-NEXT: add x8, x8, x8, lsr #63
-; CHECK-GI-NEXT: msub x0, x8, x9, x0
+; CHECK-GI-NEXT: madd x0, x8, x9, x0
; CHECK-GI-NEXT: ret
entry:
%s = srem i64 %a, 100
@@ -414,52 +417,50 @@
}
define i64 @ui64_7(i64 %a, i64 %b) {
-; CHECK-SD-LABEL: ui64_7:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: mov x8, #9363 // =0x2493
-; CHECK-SD-NEXT: movk x8, #37449, lsl #16
-; CHECK-SD-NEXT: movk x8, #18724, lsl #32
-; CHECK-SD-NEXT: movk x8, #9362, lsl #48
-; CHECK-SD-NEXT: umulh x8, x0, x8
-; CHECK-SD-NEXT: sub x9, x0, x8
-; CHECK-SD-NEXT: add x8, x8, x9, lsr #1
-; CHECK-SD-NEXT: lsr x8, x8, #2
-; CHECK-SD-NEXT: sub x8, x8, x8, lsl #3
-; CHECK-SD-NEXT: add x0, x0, x8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: ui64_7:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov x8, #9363 // =0x2493
-; CHECK-GI-NEXT: movk x8, #37449, lsl #16
-; CHECK-GI-NEXT: movk x8, #18724, lsl #32
-; CHECK-GI-NEXT: movk x8, #9362, lsl #48
-; CHECK-GI-NEXT: umulh x8, x0, x8
-; CHECK-GI-NEXT: sub x9, x0, x8
-; CHECK-GI-NEXT: add x8, x8, x9, lsr #1
-; CHECK-GI-NEXT: lsr x8, x8, #2
-; CHECK-GI-NEXT: lsl x9, x8, #3
-; CHECK-GI-NEXT: sub x8, x9, x8
-; CHECK-GI-NEXT: sub x0, x0, x8
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: ui64_7:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, #9363 // =0x2493
+; CHECK-NEXT: movk x8, #37449, lsl #16
+; CHECK-NEXT: movk x8, #18724, lsl #32
+; CHECK-NEXT: movk x8, #9362, lsl #48
+; CHECK-NEXT: umulh x8, x0, x8
+; CHECK-NEXT: sub x9, x0, x8
+; CHECK-NEXT: add x8, x8, x9, lsr #1
+; CHECK-NEXT: lsr x8, x8, #2
+; CHECK-NEXT: sub x8, x8, x8, lsl #3
+; CHECK-NEXT: add x0, x0, x8
+; CHECK-NEXT: ret
entry:
%s = urem i64 %a, 7
ret i64 %s
}
define i64 @ui64_100(i64 %a, i64 %b) {
-; CHECK-LABEL: ui64_100:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov x9, #62915 // =0xf5c3
-; CHECK-NEXT: lsr x8, x0, #2
-; CHECK-NEXT: movk x9, #23592, lsl #16
-; CHECK-NEXT: movk x9, #49807, lsl #32
-; CHECK-NEXT: movk x9, #10485, lsl #48
-; CHECK-NEXT: umulh x8, x8, x9
-; CHECK-NEXT: mov w9, #100 // =0x64
-; CHECK-NEXT: lsr x8, x8, #2
-; CHECK-NEXT: msub x0, x8, x9, x0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: ui64_100:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov x9, #62915 // =0xf5c3
+; CHECK-SD-NEXT: lsr x8, x0, #2
+; CHECK-SD-NEXT: movk x9, #23592, lsl #16
+; CHECK-SD-NEXT: movk x9, #49807, lsl #32
+; CHECK-SD-NEXT: movk x9, #10485, lsl #48
+; CHECK-SD-NEXT: umulh x8, x8, x9
+; CHECK-SD-NEXT: mov w9, #100 // =0x64
+; CHECK-SD-NEXT: lsr x8, x8, #2
+; CHECK-SD-NEXT: msub x0, x8, x9, x0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: ui64_100:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov x9, #62915 // =0xf5c3
+; CHECK-GI-NEXT: lsr x8, x0, #2
+; CHECK-GI-NEXT: movk x9, #23592, lsl #16
+; CHECK-GI-NEXT: movk x9, #49807, lsl #32
+; CHECK-GI-NEXT: movk x9, #10485, lsl #48
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: mov x9, #-100 // =0xffffffffffffff9c
+; CHECK-GI-NEXT: lsr x8, x8, #2
+; CHECK-GI-NEXT: madd x0, x8, x9, x0
+; CHECK-GI-NEXT: ret
entry:
%s = urem i64 %a, 100
ret i64 %s
@@ -565,18 +566,18 @@
; CHECK-GI-NEXT: extr x10, x11, x10, #1
; CHECK-GI-NEXT: lsr x11, x11, #1
; CHECK-GI-NEXT: adds x9, x10, x9
-; CHECK-GI-NEXT: mov w10, #7 // =0x7
+; CHECK-GI-NEXT: mov x10, #-7 // =0xfffffffffffffff9
; CHECK-GI-NEXT: adc x8, x11, x8
; CHECK-GI-NEXT: extr x9, x8, x9, #2
; CHECK-GI-NEXT: lsr x8, x8, #2
; CHECK-GI-NEXT: umulh x10, x9, x10
-; CHECK-GI-NEXT: lsl x11, x9, #3
-; CHECK-GI-NEXT: lsl x12, x8, #3
-; CHECK-GI-NEXT: sub x9, x11, x9
-; CHECK-GI-NEXT: sub x8, x12, x8
-; CHECK-GI-NEXT: subs x0, x0, x9
+; CHECK-GI-NEXT: sub x8, x8, x8, lsl #3
+; CHECK-GI-NEXT: sub x11, x9, x9, lsl #1
+; CHECK-GI-NEXT: sub x9, x9, x9, lsl #3
+; CHECK-GI-NEXT: add x8, x8, x11
+; CHECK-GI-NEXT: adds x0, x0, x9
; CHECK-GI-NEXT: add x8, x8, x10
-; CHECK-GI-NEXT: sbc x1, x1, x8
+; CHECK-GI-NEXT: adc x1, x1, x8
; CHECK-GI-NEXT: ret
entry:
%s = urem i128 %a, 7
@@ -647,14 +648,16 @@
; CHECK-GI-NEXT: add x10, x11, x12
; CHECK-GI-NEXT: add x8, x8, x14
; CHECK-GI-NEXT: add x8, x8, x10
-; CHECK-GI-NEXT: mov w10, #100 // =0x64
+; CHECK-GI-NEXT: mov x10, #-100 // =0xffffffffffffff9c
; CHECK-GI-NEXT: extr x9, x8, x9, #4
; CHECK-GI-NEXT: lsr x8, x8, #4
-; CHECK-GI-NEXT: umulh x11, x9, x10
-; CHECK-GI-NEXT: mul x9, x9, x10
-; CHECK-GI-NEXT: madd x8, x8, x10, x11
-; CHECK-GI-NEXT: subs x0, x0, x9
-; CHECK-GI-NEXT: sbc x1, x1, x8
+; CHECK-GI-NEXT: mul x11, x9, x10
+; CHECK-GI-NEXT: umulh x12, x9, x10
+; CHECK-GI-NEXT: sub x9, x9, x9, lsl #1
+; CHECK-GI-NEXT: madd x8, x8, x10, x9
+; CHECK-GI-NEXT: adds x0, x0, x11
+; CHECK-GI-NEXT: add x8, x8, x12
+; CHECK-GI-NEXT: adc x1, x1, x8
; CHECK-GI-NEXT: ret
entry:
%s = urem i128 %a, 100
diff --git a/llvm/test/CodeGen/AArch64/urem-lkk.ll b/llvm/test/CodeGen/AArch64/urem-lkk.ll
index c09924b..df3b495 100644
--- a/llvm/test/CodeGen/AArch64/urem-lkk.ll
+++ b/llvm/test/CodeGen/AArch64/urem-lkk.ll
@@ -22,24 +22,34 @@
; CHECK-GI-NEXT: lsr x8, x8, #32
; CHECK-GI-NEXT: sub w9, w0, w8
; CHECK-GI-NEXT: add w8, w8, w9, lsr #1
-; CHECK-GI-NEXT: mov w9, #95 // =0x5f
+; CHECK-GI-NEXT: mov w9, #-95 // =0xffffffa1
; CHECK-GI-NEXT: lsr w8, w8, #6
-; CHECK-GI-NEXT: msub w0, w8, w9, w0
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
; CHECK-GI-NEXT: ret
%1 = urem i32 %x, 95
ret i32 %1
}
define i32 @fold_urem_positive_even(i32 %x) {
-; CHECK-LABEL: fold_urem_positive_even:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #16323 // =0x3fc3
-; CHECK-NEXT: mov w9, #1060 // =0x424
-; CHECK-NEXT: movk w8, #63310, lsl #16
-; CHECK-NEXT: umull x8, w0, w8
-; CHECK-NEXT: lsr x8, x8, #42
-; CHECK-NEXT: msub w0, w8, w9, w0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fold_urem_positive_even:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov w8, #16323 // =0x3fc3
+; CHECK-SD-NEXT: mov w9, #1060 // =0x424
+; CHECK-SD-NEXT: movk w8, #63310, lsl #16
+; CHECK-SD-NEXT: umull x8, w0, w8
+; CHECK-SD-NEXT: lsr x8, x8, #42
+; CHECK-SD-NEXT: msub w0, w8, w9, w0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fold_urem_positive_even:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov w8, #16323 // =0x3fc3
+; CHECK-GI-NEXT: mov w9, #-1060 // =0xfffffbdc
+; CHECK-GI-NEXT: movk w8, #63310, lsl #16
+; CHECK-GI-NEXT: umull x8, w0, w8
+; CHECK-GI-NEXT: lsr x8, x8, #42
+; CHECK-GI-NEXT: madd w0, w8, w9, w0
+; CHECK-GI-NEXT: ret
%1 = urem i32 %x, 1060
ret i32 %1
}
@@ -102,18 +112,31 @@
; Don't fold i64 urem
define i64 @dont_fold_urem_i64(i64 %x) {
-; CHECK-LABEL: dont_fold_urem_i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x9, #58849 // =0xe5e1
-; CHECK-NEXT: lsr x8, x0, #1
-; CHECK-NEXT: movk x9, #48148, lsl #16
-; CHECK-NEXT: movk x9, #33436, lsl #32
-; CHECK-NEXT: movk x9, #21399, lsl #48
-; CHECK-NEXT: umulh x8, x8, x9
-; CHECK-NEXT: mov w9, #98 // =0x62
-; CHECK-NEXT: lsr x8, x8, #4
-; CHECK-NEXT: msub x0, x8, x9, x0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: dont_fold_urem_i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x9, #58849 // =0xe5e1
+; CHECK-SD-NEXT: lsr x8, x0, #1
+; CHECK-SD-NEXT: movk x9, #48148, lsl #16
+; CHECK-SD-NEXT: movk x9, #33436, lsl #32
+; CHECK-SD-NEXT: movk x9, #21399, lsl #48
+; CHECK-SD-NEXT: umulh x8, x8, x9
+; CHECK-SD-NEXT: mov w9, #98 // =0x62
+; CHECK-SD-NEXT: lsr x8, x8, #4
+; CHECK-SD-NEXT: msub x0, x8, x9, x0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: dont_fold_urem_i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov x9, #58849 // =0xe5e1
+; CHECK-GI-NEXT: lsr x8, x0, #1
+; CHECK-GI-NEXT: movk x9, #48148, lsl #16
+; CHECK-GI-NEXT: movk x9, #33436, lsl #32
+; CHECK-GI-NEXT: movk x9, #21399, lsl #48
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: mov x9, #-98 // =0xffffffffffffff9e
+; CHECK-GI-NEXT: lsr x8, x8, #4
+; CHECK-GI-NEXT: madd x0, x8, x9, x0
+; CHECK-GI-NEXT: ret
%1 = urem i64 %x, 98
ret i64 %1
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 7737f3c..3cc7e57 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -18,8 +18,8 @@
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, 7
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, -7
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; GFX6-NEXT: v_add_i32_e32 v1, vcc, -7, v0
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 7, v0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -48,8 +48,8 @@
; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, 7
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, v0, -7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, -7, v0
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 7, v0
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -77,9 +77,9 @@
; GFX9-NEXT: v_mul_lo_u32 v1, v0, -7
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, 7
-; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT: v_mul_hi_u32 v1, s2, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v1, -7, v[0:1]
; GFX9-NEXT: v_add_u32_e32 v1, -7, v0
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -107,9 +107,9 @@
; GFX10-NEXT: v_mul_lo_u32 v1, v0, -7
; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX10-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, 7
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX10-NEXT: v_mul_hi_u32 v1, s2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], s2, v1, -7, v[0:1]
; GFX10-NEXT: v_add_nc_u32_e32 v1, -7, v0
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -141,28 +141,27 @@
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX11-NEXT: v_mul_lo_u32 v0, v0, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_hi_u32 v3, s2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v3, -7, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, -7, v1
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_nc_u32_e32 v1, -7, v0
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v1, -7, v0
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_sub_nc_u16 v1, 6, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0x7f, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v1, 0x7f, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b16 v0, v0, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b16 v1, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
@@ -183,8 +182,8 @@
; GFX6-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4
; GFX6-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX6-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_mul_lo_u32 v3, v3, -7
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GFX6-NEXT: v_add_i32_e32 v3, vcc, -7, v2
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -213,8 +212,8 @@
; GFX8-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v4
; GFX8-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX8-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT: v_mul_lo_u32 v3, v3, -7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_add_u32_e32 v3, vcc, -7, v2
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -243,8 +242,7 @@
; GFX9-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX9-NEXT: v_add_u32_e32 v3, v3, v4
; GFX9-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX9-NEXT: v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, -7, v[2:3]
; GFX9-NEXT: v_add_u32_e32 v3, -7, v2
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -273,8 +271,7 @@
; GFX10-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v4
; GFX10-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s4, v3, -7, v[2:3]
; GFX10-NEXT: v_add_nc_u32_e32 v3, -7, v2
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
@@ -307,25 +304,24 @@
; GFX11-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX11-NEXT: v_add_nc_u32_e32 v3, v3, v4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX11-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, v2, v3
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, v3
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v2, -7, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_nc_u32_e32 v3, -7, v2
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: v_add_nc_u32_e32 v3, -7, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_sub_nc_u16 v3, 6, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0x7f, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v3, 0x7f, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b16 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b16 v1, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i7 @llvm.fshl.i7(i7 %lhs, i7 %rhs, i7 %amt)
@@ -1449,8 +1445,8 @@
; GFX6-NEXT: v_mul_hi_u32 v2, v0, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, v0, v1
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -1479,8 +1475,8 @@
; GFX8-NEXT: v_mul_hi_u32 v2, v0, v2
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v1
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -1505,12 +1501,12 @@
; GFX9-NEXT: s_bfe_u32 s1, s1, 0x170001
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_lo_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
+; GFX9-NEXT: v_mul_lo_u32 v2, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
+; GFX9-NEXT: v_mul_hi_u32 v2, s2, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, v1, v[0:1]
; GFX9-NEXT: v_add_u32_e32 v1, 0xffffffe8, v0
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
@@ -1536,9 +1532,9 @@
; GFX10-NEXT: v_mul_lo_u32 v1, 0xffffffe8, v0
; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX10-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX10-NEXT: v_mul_hi_u32 v1, s2, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], s2, 0xffffffe8, v1, v[0:1]
; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xffffffe8, v0
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -1568,26 +1564,25 @@
; GFX11-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX11-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX11-NEXT: v_mul_hi_u32 v3, s2, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, s2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, 0xffffffe8, v3, v[0:1]
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0xffffffe8, v1
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xffffffe8, v0
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xffffffe8, v0
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_sub_nc_u32_e32 v1, 23, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e64 v1, v1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshl_or_b32 v0, s0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
@@ -1609,8 +1604,8 @@
; GFX6-NEXT: v_mul_hi_u32 v5, v3, v5
; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
; GFX6-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX6-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_mul_lo_u32 v3, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GFX6-NEXT: v_add_i32_e32 v3, vcc, v2, v4
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -1639,8 +1634,8 @@
; GFX8-NEXT: v_mul_hi_u32 v5, v3, v5
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v5
; GFX8-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX8-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT: v_mul_lo_u32 v3, v3, v4
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v2, v4
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -1665,12 +1660,11 @@
; GFX9-NEXT: v_bfe_u32 v1, v1, 1, 23
; GFX9-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX9-NEXT: v_mul_lo_u32 v4, v3, v4
-; GFX9-NEXT: v_mul_hi_u32 v4, v3, v4
-; GFX9-NEXT: v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT: v_mul_lo_u32 v5, v3, v4
+; GFX9-NEXT: v_mul_hi_u32 v5, v3, v5
+; GFX9-NEXT: v_add_u32_e32 v3, v3, v5
; GFX9-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX9-NEXT: v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v4, v[2:3]
; GFX9-NEXT: v_add_u32_e32 v3, 0xffffffe8, v2
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -1697,8 +1691,7 @@
; GFX10-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v4
; GFX10-NEXT: v_mul_hi_u32 v3, v2, v3
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s4, 0xffffffe8, v3, v[2:3]
; GFX10-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
@@ -1728,24 +1721,23 @@
; GFX11-NEXT: v_mul_hi_u32 v4, v3, v4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_nc_u32_e32 v3, v3, v4
-; GFX11-NEXT: v_mul_hi_u32 v3, v2, v3
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, v2, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshl_or_b32 v0, v0, v2, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i24 @llvm.fshl.i24(i24 %lhs, i24 %rhs, i24 %amt)
@@ -1810,19 +1802,19 @@
; GFX6-NEXT: v_mov_b32_e32 v5, s4
; GFX6-NEXT: s_and_b32 s2, s2, 0xff
; GFX6-NEXT: v_alignbit_b32 v5, s3, v5, 24
-; GFX6-NEXT: v_mul_lo_u32 v4, v4, 24
+; GFX6-NEXT: v_mul_lo_u32 v4, v4, v3
; GFX6-NEXT: s_lshl_b32 s2, s2, 16
; GFX6-NEXT: v_or_b32_e32 v5, s2, v5
; GFX6-NEXT: v_mul_hi_u32 v2, v5, v2
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s1, v4
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, s1, v4
; GFX6-NEXT: v_add_i32_e32 v6, vcc, v4, v3
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, 24
+; GFX6-NEXT: v_mul_lo_u32 v2, v2, v3
; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
; GFX6-NEXT: v_add_i32_e32 v6, vcc, v4, v3
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v5, v2
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v2
; GFX6-NEXT: v_sub_i32_e32 v6, vcc, 23, v4
; GFX6-NEXT: v_add_i32_e32 v5, vcc, v2, v3
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
@@ -1919,19 +1911,19 @@
; GFX8-NEXT: s_lshr_b32 s9, s5, 8
; GFX8-NEXT: s_and_b32 s5, s5, 0xff
; GFX8-NEXT: s_lshl_b32 s5, s5, 8
-; GFX8-NEXT: v_mul_lo_u32 v2, v2, 24
+; GFX8-NEXT: v_mul_lo_u32 v2, v2, v1
; GFX8-NEXT: s_and_b32 s6, s9, 0xff
; GFX8-NEXT: s_or_b32 s5, s8, s5
; GFX8-NEXT: s_lshl_b32 s6, s6, 16
; GFX8-NEXT: s_or_b32 s5, s5, s6
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s4, v2
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s4, v2
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v2, v1
; GFX8-NEXT: v_mul_hi_u32 v0, s5, v0
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v2, v1
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
-; GFX8-NEXT: v_mul_lo_u32 v0, v0, 24
+; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 23, v2
; GFX8-NEXT: v_and_b32_e32 v2, 0xffffff, v2
@@ -1939,7 +1931,7 @@
; GFX8-NEXT: s_lshr_b32 s0, s2, 1
; GFX8-NEXT: v_and_b32_e32 v3, 0xffffff, v3
; GFX8-NEXT: v_lshrrev_b32_e64 v3, v3, s0
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s5, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s5, v0
; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v0, v1
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
@@ -1981,73 +1973,72 @@
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: s_or_b32 s0, s0, s6
; GFX9-NEXT: s_and_b32 s6, s7, 0xff
+; GFX9-NEXT: s_lshl_b32 s6, s6, 16
; GFX9-NEXT: s_lshr_b32 s9, s1, 8
-; GFX9-NEXT: s_lshl_b32 s6, s6, 16
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_or_b32 s0, s0, s6
-; GFX9-NEXT: s_lshl_b32 s1, s1, 8
-; GFX9-NEXT: s_and_b32 s6, s9, 0xff
-; GFX9-NEXT: s_or_b32 s1, s8, s1
-; GFX9-NEXT: s_lshl_b32 s6, s6, 16
+; GFX9-NEXT: s_or_b32 s6, s0, s6
+; GFX9-NEXT: s_and_b32 s0, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_and_b32 s1, s9, 0xff
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX9-NEXT: s_or_b32 s1, s1, s6
-; GFX9-NEXT: s_lshr_b32 s6, s2, 8
+; GFX9-NEXT: s_or_b32 s0, s8, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshr_b32 s7, s2, 16
+; GFX9-NEXT: s_or_b32 s7, s0, s1
+; GFX9-NEXT: s_lshr_b32 s0, s2, 8
+; GFX9-NEXT: s_lshr_b32 s1, s2, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
; GFX9-NEXT: s_lshr_b32 s8, s2, 24
; GFX9-NEXT: s_and_b32 s2, s2, 0xff
-; GFX9-NEXT: s_lshl_b32 s6, s6, 8
-; GFX9-NEXT: s_or_b32 s2, s2, s6
-; GFX9-NEXT: s_and_b32 s6, s7, 0xff
-; GFX9-NEXT: v_not_b32_e32 v1, 23
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: v_not_b32_e32 v2, 23
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: v_mul_lo_u32 v1, v0, v2
; GFX9-NEXT: s_lshr_b32 s9, s3, 8
-; GFX9-NEXT: s_lshl_b32 s6, s6, 16
-; GFX9-NEXT: s_and_b32 s3, s3, 0xff
-; GFX9-NEXT: v_mul_lo_u32 v1, v0, v1
-; GFX9-NEXT: s_or_b32 s2, s2, s6
-; GFX9-NEXT: s_lshl_b32 s3, s3, 8
-; GFX9-NEXT: s_and_b32 s6, s9, 0xff
-; GFX9-NEXT: s_or_b32 s3, s8, s3
-; GFX9-NEXT: s_lshl_b32 s6, s6, 16
-; GFX9-NEXT: s_or_b32 s3, s3, s6
-; GFX9-NEXT: s_lshr_b32 s6, s4, 8
-; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_or_b32 s2, s0, s1
+; GFX9-NEXT: s_and_b32 s0, s3, 0xff
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_and_b32 s1, s9, 0xff
+; GFX9-NEXT: s_or_b32 s0, s8, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_or_b32 s3, s0, s1
+; GFX9-NEXT: s_lshr_b32 s0, s4, 8
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: s_lshr_b32 s7, s4, 16
+; GFX9-NEXT: s_lshr_b32 s1, s4, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
; GFX9-NEXT: s_lshr_b32 s8, s4, 24
; GFX9-NEXT: s_and_b32 s4, s4, 0xff
-; GFX9-NEXT: s_lshl_b32 s6, s6, 8
-; GFX9-NEXT: s_or_b32 s4, s4, s6
-; GFX9-NEXT: s_and_b32 s6, s7, 0xff
-; GFX9-NEXT: s_lshl_b32 s6, s6, 16
-; GFX9-NEXT: s_or_b32 s4, s4, s6
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
-; GFX9-NEXT: v_mul_hi_u32 v1, s4, v0
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_and_b32 s1, s1, 0xff
+; GFX9-NEXT: s_or_b32 s0, s4, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: v_add_u32_e32 v3, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v1, s0, v3
+; GFX9-NEXT: s_and_b32 s1, s5, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_or_b32 s4, s8, s1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v1, v2, v[0:1]
; GFX9-NEXT: s_lshr_b32 s9, s5, 8
-; GFX9-NEXT: s_and_b32 s5, s5, 0xff
-; GFX9-NEXT: s_lshl_b32 s5, s5, 8
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, 24
-; GFX9-NEXT: s_and_b32 s6, s9, 0xff
-; GFX9-NEXT: s_or_b32 s5, s8, s5
-; GFX9-NEXT: s_lshl_b32 s6, s6, 16
-; GFX9-NEXT: s_or_b32 s5, s5, s6
-; GFX9-NEXT: v_mul_hi_u32 v0, s5, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, s4, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffe8, v1
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffe8, v1
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v1
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_sub_u32_e32 v2, 23, v1
+; GFX9-NEXT: s_and_b32 s0, s9, 0xff
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: v_add_u32_e32 v1, 0xffffffe8, v0
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT: s_or_b32 s0, s4, s0
+; GFX9-NEXT: v_add_u32_e32 v1, 0xffffffe8, v0
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT: v_mul_hi_u32 v1, s0, v3
+; GFX9-NEXT: v_sub_u32_e32 v4, 23, v0
+; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v1, v2, v[0:1]
; GFX9-NEXT: s_lshr_b32 s2, s2, 1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff, v1
-; GFX9-NEXT: v_lshrrev_b32_e64 v2, v2, s2
-; GFX9-NEXT: v_sub_u32_e32 v0, s5, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, s0, v1, v2
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff, v4
+; GFX9-NEXT: v_lshrrev_b32_e64 v1, v1, s2
; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffe8, v0
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -2057,10 +2048,11 @@
; GFX9-NEXT: v_sub_u32_e32 v2, 23, v0
; GFX9-NEXT: s_lshr_b32 s0, s3, 1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX9-NEXT: v_lshl_or_b32 v1, s6, v3, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; GFX9-NEXT: v_lshrrev_b32_e64 v2, v2, s0
; GFX9-NEXT: v_mov_b32_e32 v3, 8
-; GFX9-NEXT: v_lshl_or_b32 v0, s1, v0, v2
+; GFX9-NEXT: v_lshl_or_b32 v0, s7, v0, v2
; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX9-NEXT: v_and_or_b32 v2, v1, v2, v3
@@ -2079,104 +2071,104 @@
; GFX10-LABEL: s_fshl_v2i24:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX10-NEXT: s_lshr_b32 s10, s4, 8
-; GFX10-NEXT: s_lshr_b32 s11, s4, 16
-; GFX10-NEXT: s_and_b32 s10, s10, 0xff
-; GFX10-NEXT: s_lshr_b32 s12, s4, 24
+; GFX10-NEXT: s_lshr_b32 s14, s4, 8
+; GFX10-NEXT: s_lshr_b32 s15, s4, 16
+; GFX10-NEXT: s_and_b32 s14, s14, 0xff
+; GFX10-NEXT: s_lshr_b32 s16, s4, 24
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: s_and_b32 s4, s4, 0xff
-; GFX10-NEXT: s_and_b32 s11, s11, 0xff
-; GFX10-NEXT: s_lshl_b32 s10, s10, 8
-; GFX10-NEXT: s_lshl_b32 s11, s11, 16
-; GFX10-NEXT: s_or_b32 s4, s4, s10
-; GFX10-NEXT: s_lshr_b32 s13, s5, 8
+; GFX10-NEXT: s_and_b32 s15, s15, 0xff
+; GFX10-NEXT: s_lshl_b32 s14, s14, 8
+; GFX10-NEXT: s_lshr_b32 s17, s5, 8
; GFX10-NEXT: s_and_b32 s5, s5, 0xff
-; GFX10-NEXT: s_or_b32 s4, s4, s11
+; GFX10-NEXT: s_lshl_b32 s15, s15, 16
+; GFX10-NEXT: s_or_b32 s4, s4, s14
; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_and_b32 s17, s17, 0xff
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX10-NEXT: s_and_b32 s13, s13, 0xff
-; GFX10-NEXT: s_or_b32 s5, s12, s5
-; GFX10-NEXT: s_lshl_b32 s10, s13, 16
-; GFX10-NEXT: s_lshr_b32 s9, s1, 8
+; GFX10-NEXT: s_or_b32 s4, s4, s15
+; GFX10-NEXT: s_or_b32 s5, s16, s5
+; GFX10-NEXT: s_lshl_b32 s14, s17, 16
+; GFX10-NEXT: s_lshr_b32 s6, s0, 8
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: s_or_b32 s5, s5, s10
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff
+; GFX10-NEXT: s_or_b32 s5, s5, s14
+; GFX10-NEXT: s_lshr_b32 s7, s0, 16
; GFX10-NEXT: s_lshr_b32 s10, s2, 8
-; GFX10-NEXT: s_lshr_b32 s8, s0, 24
-; GFX10-NEXT: v_mul_lo_u32 v2, 0xffffffe8, v0
+; GFX10-NEXT: s_lshr_b32 s9, s1, 8
+; GFX10-NEXT: v_mul_lo_u32 v1, 0xffffffe8, v0
+; GFX10-NEXT: s_and_b32 s1, s1, 0xff
; GFX10-NEXT: s_lshr_b32 s11, s2, 16
; GFX10-NEXT: s_lshr_b32 s13, s3, 8
; GFX10-NEXT: s_and_b32 s3, s3, 0xff
-; GFX10-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10-NEXT: s_and_b32 s10, s10, 0xff
+; GFX10-NEXT: s_lshr_b32 s8, s0, 24
; GFX10-NEXT: s_lshr_b32 s12, s2, 24
; GFX10-NEXT: s_and_b32 s2, s2, 0xff
-; GFX10-NEXT: v_mul_hi_u32 v2, v0, v2
+; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX10-NEXT: s_lshl_b32 s1, s1, 8
; GFX10-NEXT: s_lshl_b32 s3, s3, 8
; GFX10-NEXT: s_or_b32 s1, s8, s1
-; GFX10-NEXT: s_lshl_b32 s8, s10, 8
; GFX10-NEXT: s_or_b32 s3, s12, s3
-; GFX10-NEXT: s_or_b32 s2, s2, s8
-; GFX10-NEXT: s_lshr_b32 s6, s0, 8
-; GFX10-NEXT: s_lshr_b32 s7, s0, 16
-; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v2
-; GFX10-NEXT: s_and_b32 s6, s6, 0xff
; GFX10-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10-NEXT: s_and_b32 s7, s7, 0xff
-; GFX10-NEXT: s_and_b32 s9, s9, 0xff
+; GFX10-NEXT: v_mov_b32_e32 v4, 8
+; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: v_mul_hi_u32 v2, s4, v0
-; GFX10-NEXT: v_mul_hi_u32 v0, s5, v0
-; GFX10-NEXT: s_lshl_b32 s6, s6, 8
-; GFX10-NEXT: s_lshl_b32 s7, s7, 16
-; GFX10-NEXT: s_or_b32 s0, s0, s6
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: s_or_b32 s0, s0, s7
-; GFX10-NEXT: v_mul_lo_u32 v2, v2, 24
-; GFX10-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, s4, v2
-; GFX10-NEXT: v_sub_nc_u32_e32 v0, s5, v0
-; GFX10-NEXT: s_and_b32 s4, s11, 0xff
-; GFX10-NEXT: s_and_b32 s5, s13, 0xff
-; GFX10-NEXT: s_lshl_b32 s4, s4, 16
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v0
+; GFX10-NEXT: v_mul_hi_u32 v5, s5, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-NEXT: s_and_b32 s4, s6, 0xff
+; GFX10-NEXT: s_and_b32 s5, s7, 0xff
+; GFX10-NEXT: s_and_b32 s7, s10, 0xff
+; GFX10-NEXT: s_and_b32 s10, s13, 0xff
+; GFX10-NEXT: s_lshl_b32 s7, s7, 8
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s6, 0xffffffe8, v2, v[0:1]
+; GFX10-NEXT: s_or_b32 s2, s2, s7
+; GFX10-NEXT: s_lshl_b32 s4, s4, 8
; GFX10-NEXT: s_lshl_b32 s5, s5, 16
-; GFX10-NEXT: s_or_b32 s2, s2, s4
-; GFX10-NEXT: s_or_b32 s3, s3, s5
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: s_or_b32 s0, s0, s4
+; GFX10-NEXT: s_or_b32 s0, s0, s5
+; GFX10-NEXT: v_mad_u64_u32 v[0:1], s6, 0xffffffe8, v5, v[1:2]
+; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xffffffe8, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX10-NEXT: s_and_b32 s6, s9, 0xff
+; GFX10-NEXT: s_and_b32 s9, s11, 0xff
+; GFX10-NEXT: s_lshl_b32 s8, s9, 16
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v0
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX10-NEXT: s_lshl_b32 s9, s10, 16
+; GFX10-NEXT: s_or_b32 s2, s2, s8
+; GFX10-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v1
+; GFX10-NEXT: s_or_b32 s3, s3, s9
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v1
; GFX10-NEXT: s_lshr_b32 s3, s3, 1
; GFX10-NEXT: s_lshr_b32 s2, s2, 1
-; GFX10-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX10-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, 23, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT: v_sub_nc_u32_e32 v2, 23, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffffff, v1
; GFX10-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, 23, v0
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, 23, v0
; GFX10-NEXT: v_and_b32_e32 v0, 0xffffff, v0
-; GFX10-NEXT: v_lshrrev_b32_e64 v3, v3, s2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX10-NEXT: s_lshl_b32 s2, s9, 16
+; GFX10-NEXT: v_lshrrev_b32_e64 v2, v2, s2
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff, v3
+; GFX10-NEXT: s_lshl_b32 s2, s6, 16
; GFX10-NEXT: s_or_b32 s1, s1, s2
-; GFX10-NEXT: v_lshl_or_b32 v2, s0, v2, v3
-; GFX10-NEXT: v_lshrrev_b32_e64 v4, v4, s3
-; GFX10-NEXT: v_mov_b32_e32 v3, 16
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; GFX10-NEXT: v_lshl_or_b32 v0, s1, v0, v4
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_lshl_or_b32 v1, s0, v1, v2
+; GFX10-NEXT: v_lshrrev_b32_e64 v3, v3, s3
+; GFX10-NEXT: v_mov_b32_e32 v2, 16
+; GFX10-NEXT: v_lshl_or_b32 v0, s1, v0, v3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
; GFX10-NEXT: v_and_b32_e32 v4, 0xff, v0
; GFX10-NEXT: v_bfe_u32 v5, v0, 8, 8
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v1, v3
; GFX10-NEXT: v_bfe_u32 v0, v0, 16, 8
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 24, v4
; GFX10-NEXT: v_lshl_or_b32 v0, v0, 8, v5
-; GFX10-NEXT: v_or3_b32 v1, v1, v3, v2
+; GFX10-NEXT: v_or3_b32 v1, v1, v2, v3
; GFX10-NEXT: v_readfirstlane_b32 s1, v0
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: ; return to shader part epilog
@@ -2184,121 +2176,118 @@
; GFX11-LABEL: s_fshl_v2i24:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
-; GFX11-NEXT: s_lshr_b32 s10, s4, 8
-; GFX11-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-NEXT: s_and_b32 s10, s10, 0xff
-; GFX11-NEXT: s_lshr_b32 s12, s4, 24
-; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX11-NEXT: s_and_b32 s4, s4, 0xff
-; GFX11-NEXT: s_and_b32 s11, s11, 0xff
-; GFX11-NEXT: s_lshl_b32 s10, s10, 8
-; GFX11-NEXT: s_lshl_b32 s11, s11, 16
-; GFX11-NEXT: s_or_b32 s4, s4, s10
-; GFX11-NEXT: s_lshr_b32 s13, s5, 8
+; GFX11-NEXT: s_lshr_b32 s14, s4, 8
+; GFX11-NEXT: s_lshr_b32 s17, s5, 8
; GFX11-NEXT: s_and_b32 s5, s5, 0xff
-; GFX11-NEXT: s_or_b32 s4, s4, s11
+; GFX11-NEXT: s_and_b32 s14, s14, 0xff
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX11-NEXT: s_lshr_b32 s15, s4, 16
+; GFX11-NEXT: s_lshr_b32 s16, s4, 24
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
; GFX11-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-NEXT: s_and_b32 s17, s17, 0xff
+; GFX11-NEXT: s_lshl_b32 s14, s14, 8
+; GFX11-NEXT: s_or_b32 s5, s16, s5
+; GFX11-NEXT: s_or_b32 s4, s4, s14
+; GFX11-NEXT: s_lshl_b32 s14, s17, 16
+; GFX11-NEXT: s_and_b32 s15, s15, 0xff
+; GFX11-NEXT: s_or_b32 s5, s5, s14
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX11-NEXT: s_and_b32 s13, s13, 0xff
-; GFX11-NEXT: s_or_b32 s5, s12, s5
-; GFX11-NEXT: s_lshl_b32 s10, s13, 16
-; GFX11-NEXT: s_lshr_b32 s9, s1, 8
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_or_b32 s5, s5, s10
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshl_b32 s15, s15, 16
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_or_b32 s4, s4, s15
; GFX11-NEXT: s_lshr_b32 s10, s2, 8
-; GFX11-NEXT: s_lshr_b32 s8, s0, 24
-; GFX11-NEXT: v_mul_lo_u32 v1, 0xffffffe8, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
; GFX11-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_and_b32 s9, s9, 0xff
-; GFX11-NEXT: s_and_b32 s10, s10, 0xff
+; GFX11-NEXT: v_mul_lo_u32 v1, 0xffffffe8, v0
+; GFX11-NEXT: s_lshr_b32 s8, s0, 24
; GFX11-NEXT: s_lshr_b32 s12, s2, 24
; GFX11-NEXT: s_and_b32 s2, s2, 0xff
-; GFX11-NEXT: s_or_b32 s1, s8, s1
-; GFX11-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX11-NEXT: s_lshl_b32 s8, s9, 16
-; GFX11-NEXT: s_lshl_b32 s9, s10, 8
-; GFX11-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-NEXT: s_or_b32 s2, s2, s9
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-NEXT: s_lshr_b32 s13, s3, 8
+; GFX11-NEXT: s_or_b32 s1, s8, s1
; GFX11-NEXT: s_and_b32 s3, s3, 0xff
-; GFX11-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-NEXT: v_add_nc_u32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX11-NEXT: s_lshl_b32 s3, s3, 8
; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_and_b32 s7, s7, 0xff
-; GFX11-NEXT: v_mul_hi_u32 v1, s4, v0
-; GFX11-NEXT: v_mul_hi_u32 v0, s5, v0
-; GFX11-NEXT: s_lshl_b32 s6, s6, 8
; GFX11-NEXT: s_or_b32 s3, s12, s3
-; GFX11-NEXT: s_lshl_b32 s7, s7, 16
-; GFX11-NEXT: s_or_b32 s0, s0, s6
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: s_or_b32 s0, s0, s7
-; GFX11-NEXT: v_mul_lo_u32 v1, v1, 24
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_lo_u32 v0, v0, 24
-; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, s5, v0
-; GFX11-NEXT: s_and_b32 s4, s11, 0xff
-; GFX11-NEXT: s_and_b32 s5, s13, 0xff
-; GFX11-NEXT: s_lshl_b32 s4, s4, 16
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v1
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v0
-; GFX11-NEXT: s_or_b32 s2, s2, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, v0, v1
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: v_mul_hi_u32 v4, s4, v1
+; GFX11-NEXT: v_mul_hi_u32 v5, s5, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, s5
+; GFX11-NEXT: s_and_b32 s5, s7, 0xff
+; GFX11-NEXT: s_and_b32 s7, s10, 0xff
+; GFX11-NEXT: s_and_b32 s4, s6, 0xff
+; GFX11-NEXT: s_and_b32 s6, s9, 0xff
+; GFX11-NEXT: s_and_b32 s9, s11, 0xff
+; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, 0xffffffe8, v4, v[0:1]
+; GFX11-NEXT: s_lshl_b32 s7, s7, 8
+; GFX11-NEXT: s_lshl_b32 s8, s9, 16
+; GFX11-NEXT: s_or_b32 s2, s2, s7
+; GFX11-NEXT: s_and_b32 s10, s13, 0xff
+; GFX11-NEXT: s_or_b32 s2, s2, s8
+; GFX11-NEXT: s_lshl_b32 s4, s4, 8
; GFX11-NEXT: s_lshr_b32 s2, s2, 1
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[1:2]
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_lshl_b32 s9, s10, 16
+; GFX11-NEXT: s_lshl_b32 s5, s5, 16
+; GFX11-NEXT: s_or_b32 s0, s0, s4
+; GFX11-NEXT: s_or_b32 s3, s3, s9
+; GFX11-NEXT: s_or_b32 s0, s0, s5
+; GFX11-NEXT: s_lshl_b32 s6, s6, 16
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xffffffe8, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0xffffffe8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
-; GFX11-NEXT: s_or_b32 s3, s3, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v1
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v1
-; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, 23, v1
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v0
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v0
+; GFX11-NEXT: v_sub_nc_u32_e32 v2, 23, v0
+; GFX11-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v1
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshrrev_b32_e64 v2, v2, s2
-; GFX11-NEXT: s_lshr_b32 s2, s3, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffffff, v0
-; GFX11-NEXT: v_lshl_or_b32 v1, s0, v1, v2
-; GFX11-NEXT: s_or_b32 s0, s1, s8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_bfe_u32 v2, v1, 8, 8
+; GFX11-NEXT: s_lshr_b32 s2, s3, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshl_or_b32 v0, s0, v0, v2
; GFX11-NEXT: v_lshrrev_b32_e64 v3, v3, s2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshl_or_b32 v0, s0, v0, v3
-; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 8
-; GFX11-NEXT: v_and_or_b32 v1, 0xff, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX11-NEXT: v_bfe_u32 v5, v0, 8, 8
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_bfe_u32 v0, v0, 16, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 24, v4
-; GFX11-NEXT: v_lshl_or_b32 v0, v0, 8, v5
+; GFX11-NEXT: s_or_b32 s0, s1, s6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or3_b32 v1, v1, v3, v4
-; GFX11-NEXT: v_readfirstlane_b32 s1, v0
+; GFX11-NEXT: v_bfe_u32 v2, v0, 8, 8
+; GFX11-NEXT: v_lshl_or_b32 v1, s0, v1, v3
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-NEXT: v_and_b32_e32 v4, 0xff, v1
+; GFX11-NEXT: v_bfe_u32 v5, v1, 8, 8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_bfe_u32 v1, v1, 16, 8
+; GFX11-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; GFX11-NEXT: v_lshl_or_b32 v1, v1, 8, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or3_b32 v0, v0, v3, v4
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: ; return to shader part epilog
%lhs = bitcast i48 %lhs.arg to <2 x i24>
%rhs = bitcast i48 %rhs.arg to <2 x i24>
@@ -2325,9 +2314,9 @@
; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v8
; GFX6-NEXT: v_mul_hi_u32 v8, v4, v6
; GFX6-NEXT: v_mul_hi_u32 v6, v5, v6
-; GFX6-NEXT: v_mul_lo_u32 v8, v8, 24
-; GFX6-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT: v_mul_lo_u32 v8, v8, v7
+; GFX6-NEXT: v_mul_lo_u32 v6, v6, v7
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GFX6-NEXT: v_add_i32_e32 v8, vcc, v4, v7
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc
@@ -2340,7 +2329,7 @@
; GFX6-NEXT: v_and_b32_e32 v4, 0xffffff, v8
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v2
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v5, v6
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v6
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v2, v7
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
@@ -2372,9 +2361,9 @@
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v8
; GFX8-NEXT: v_mul_hi_u32 v8, v4, v6
; GFX8-NEXT: v_mul_hi_u32 v6, v5, v6
-; GFX8-NEXT: v_mul_lo_u32 v8, v8, 24
-; GFX8-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v4, v8
+; GFX8-NEXT: v_mul_lo_u32 v8, v8, v7
+; GFX8-NEXT: v_mul_lo_u32 v6, v6, v7
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8
; GFX8-NEXT: v_add_u32_e32 v8, vcc, v4, v7
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc
@@ -2387,7 +2376,7 @@
; GFX8-NEXT: v_and_b32_e32 v4, 0xffffff, v8
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v2
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v5, v6
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v6
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v2, v7
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
@@ -2408,39 +2397,37 @@
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v6
-; GFX9-NEXT: v_not_b32_e32 v7, 23
+; GFX9-NEXT: v_not_b32_e32 v8, 23
; GFX9-NEXT: v_and_b32_e32 v4, 0xffffff, v4
; GFX9-NEXT: v_and_b32_e32 v5, 0xffffff, v5
; GFX9-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v6
; GFX9-NEXT: v_bfe_u32 v2, v2, 1, 23
; GFX9-NEXT: v_bfe_u32 v3, v3, 1, 23
-; GFX9-NEXT: v_mul_lo_u32 v7, v6, v7
+; GFX9-NEXT: v_mul_lo_u32 v7, v6, v8
; GFX9-NEXT: v_mul_hi_u32 v7, v6, v7
; GFX9-NEXT: v_add_u32_e32 v6, v6, v7
; GFX9-NEXT: v_mul_hi_u32 v7, v4, v6
-; GFX9-NEXT: v_mul_hi_u32 v6, v5, v6
-; GFX9-NEXT: v_mul_lo_u32 v7, v7, 24
-; GFX9-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX9-NEXT: v_sub_u32_e32 v4, v4, v7
-; GFX9-NEXT: v_sub_u32_e32 v5, v5, v6
-; GFX9-NEXT: v_add_u32_e32 v6, 0xffffffe8, v4
+; GFX9-NEXT: v_mul_hi_u32 v9, v5, v6
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v7, v8, v[4:5]
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v9, v8, v[5:6]
+; GFX9-NEXT: v_add_u32_e32 v5, 0xffffffe8, v6
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX9-NEXT: v_add_u32_e32 v7, 0xffffffe8, v4
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT: v_add_u32_e32 v6, 0xffffffe8, v4
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT: v_sub_u32_e32 v6, 23, v4
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX9-NEXT: v_add_u32_e32 v7, 0xffffffe8, v5
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, v6, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc
+; GFX9-NEXT: v_add_u32_e32 v6, 0xffffffe8, v5
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v5
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, v4, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
-; GFX9-NEXT: v_add_u32_e32 v4, 0xffffffe8, v2
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc
+; GFX9-NEXT: v_sub_u32_e32 v6, 23, v5
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX9-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, v6, v2
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, v5, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffe8, v4
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX9-NEXT: v_sub_u32_e32 v4, 23, v2
; GFX9-NEXT: v_and_b32_e32 v4, 0xffffff, v4
; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v2
@@ -2463,33 +2450,31 @@
; GFX10-NEXT: v_mul_hi_u32 v7, v6, v7
; GFX10-NEXT: v_add_nc_u32_e32 v6, v6, v7
; GFX10-NEXT: v_mul_hi_u32 v7, v4, v6
-; GFX10-NEXT: v_mul_hi_u32 v6, v5, v6
-; GFX10-NEXT: v_mul_lo_u32 v7, v7, 24
-; GFX10-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, v4, v7
-; GFX10-NEXT: v_sub_nc_u32_e32 v5, v5, v6
-; GFX10-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
+; GFX10-NEXT: v_mul_hi_u32 v8, v5, v6
+; GFX10-NEXT: v_mad_u64_u32 v[6:7], s4, 0xffffffe8, v7, v[4:5]
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], s4, 0xffffffe8, v8, v[5:6]
+; GFX10-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v6
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v4
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX10-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc_lo
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX10-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX10-NEXT: v_sub_nc_u32_e32 v7, 23, v5
+; GFX10-NEXT: v_sub_nc_u32_e32 v6, 23, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX10-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX10-NEXT: v_sub_nc_u32_e32 v7, 23, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff, v4
; GFX10-NEXT: v_lshrrev_b32_e32 v2, v6, v2
; GFX10-NEXT: v_and_b32_e32 v7, 0xffffff, v7
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, v4, v2
+; GFX10-NEXT: v_lshl_or_b32 v0, v0, v5, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v7, v3
-; GFX10-NEXT: v_lshl_or_b32 v1, v1, v5, v3
+; GFX10-NEXT: v_lshl_or_b32 v1, v1, v4, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_v2i24:
@@ -2497,54 +2482,50 @@
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
; GFX11-NEXT: v_bfe_u32 v2, v2, 1, 23
; GFX11-NEXT: v_bfe_u32 v3, v3, 1, 23
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_u32_f32_e32 v6, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_lo_u32 v7, 0xffffffe8, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_hi_u32 v7, v6, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_nc_u32_e32 v6, v6, v7
-; GFX11-NEXT: v_mul_hi_u32 v7, v4, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v7, v7, 24
-; GFX11-NEXT: v_sub_nc_u32_e32 v4, v4, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: v_mul_hi_u32 v6, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX11-NEXT: v_sub_nc_u32_e32 v5, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_hi_u32 v8, v4, v6
+; GFX11-NEXT: v_mul_hi_u32 v9, v5, v6
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v5
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v7, 0xffffff, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshl_or_b32 v0, v0, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v7, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshl_or_b32 v1, v1, v5, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i24> @llvm.fshl.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index df74cd4..7623c8b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -21,8 +21,8 @@
; GFX6-NEXT: v_mov_b32_e32 v0, s3
; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mul_i32 s3, s3, 7
-; GFX6-NEXT: s_sub_i32 s2, s2, s3
+; GFX6-NEXT: s_mul_i32 s3, s3, -7
+; GFX6-NEXT: s_add_i32 s2, s2, s3
; GFX6-NEXT: s_cmp_ge_u32 s2, 7
; GFX6-NEXT: s_cselect_b32 s3, 1, 0
; GFX6-NEXT: s_add_i32 s4, s2, -7
@@ -56,8 +56,8 @@
; GFX8-NEXT: v_mov_b32_e32 v0, s3
; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX8-NEXT: v_readfirstlane_b32 s3, v0
-; GFX8-NEXT: s_mul_i32 s3, s3, 7
-; GFX8-NEXT: s_sub_i32 s2, s2, s3
+; GFX8-NEXT: s_mul_i32 s3, s3, -7
+; GFX8-NEXT: s_add_i32 s2, s2, s3
; GFX8-NEXT: s_cmp_ge_u32 s2, 7
; GFX8-NEXT: s_cselect_b32 s3, 1, 0
; GFX8-NEXT: s_add_i32 s4, s2, -7
@@ -91,8 +91,8 @@
; GFX9-NEXT: s_mul_hi_u32 s4, s3, s4
; GFX9-NEXT: s_add_i32 s3, s3, s4
; GFX9-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX9-NEXT: s_mul_i32 s3, s3, 7
-; GFX9-NEXT: s_sub_i32 s2, s2, s3
+; GFX9-NEXT: s_mul_i32 s3, s3, -7
+; GFX9-NEXT: s_add_i32 s2, s2, s3
; GFX9-NEXT: s_cmp_ge_u32 s2, 7
; GFX9-NEXT: s_cselect_b32 s3, 1, 0
; GFX9-NEXT: s_add_i32 s4, s2, -7
@@ -126,8 +126,8 @@
; GFX10-NEXT: s_mul_hi_u32 s4, s3, s4
; GFX10-NEXT: s_add_i32 s3, s3, s4
; GFX10-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX10-NEXT: s_mul_i32 s3, s3, 7
-; GFX10-NEXT: s_sub_i32 s2, s2, s3
+; GFX10-NEXT: s_mul_i32 s3, s3, -7
+; GFX10-NEXT: s_add_i32 s2, s2, s3
; GFX10-NEXT: s_cmp_ge_u32 s2, 7
; GFX10-NEXT: s_cselect_b32 s3, 1, 0
; GFX10-NEXT: s_add_i32 s4, s2, -7
@@ -166,8 +166,8 @@
; GFX11-NEXT: s_add_i32 s3, s3, s4
; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s3, s3, 7
-; GFX11-NEXT: s_sub_i32 s2, s2, s3
+; GFX11-NEXT: s_mul_i32 s3, s3, -7
+; GFX11-NEXT: s_add_i32 s2, s2, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_ge_u32 s2, 7
; GFX11-NEXT: s_cselect_b32 s3, 1, 0
@@ -211,8 +211,8 @@
; GFX6-NEXT: v_readfirstlane_b32 s5, v3
; GFX6-NEXT: s_add_i32 s4, s4, s5
; GFX6-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX6-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_mul_lo_u32 v3, v3, -7
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GFX6-NEXT: v_add_i32_e32 v3, vcc, -7, v2
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -243,8 +243,8 @@
; GFX8-NEXT: v_readfirstlane_b32 s5, v3
; GFX8-NEXT: s_add_i32 s4, s4, s5
; GFX8-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX8-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT: v_mul_lo_u32 v3, v3, -7
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_add_u32_e32 v3, vcc, -7, v2
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -274,8 +274,7 @@
; GFX9-NEXT: s_mul_hi_u32 s5, s4, s5
; GFX9-NEXT: s_add_i32 s4, s4, s5
; GFX9-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX9-NEXT: v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, -7, v[2:3]
; GFX9-NEXT: v_add_u32_e32 v3, -7, v2
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 7, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -305,8 +304,7 @@
; GFX10-NEXT: s_mul_hi_u32 s5, s4, s5
; GFX10-NEXT: s_add_i32 s4, s4, s5
; GFX10-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s4, v3, -7, v[2:3]
; GFX10-NEXT: v_add_nc_u32_e32 v3, -7, v2
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
@@ -339,26 +337,24 @@
; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v3, v2, s0
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v3, v3, 7
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, v2, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v5, -7, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, -7, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
; GFX11-NEXT: v_add_nc_u32_e32 v3, -7, v2
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, -7, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 7, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_sub_nc_u16 v3, 6, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0x7f, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0x7f, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v3, 0x7f, v3
; GFX11-NEXT: v_lshrrev_b16 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b16 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i7 @llvm.fshr.i7(i7 %lhs, i7 %rhs, i7 %amt)
@@ -1526,8 +1522,8 @@
; GFX6-NEXT: v_mov_b32_e32 v0, s3
; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mul_i32 s3, s3, 24
-; GFX6-NEXT: s_sub_i32 s2, s2, s3
+; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX6-NEXT: s_add_i32 s2, s2, s3
; GFX6-NEXT: s_cmp_ge_u32 s2, 24
; GFX6-NEXT: s_cselect_b32 s3, 1, 0
; GFX6-NEXT: s_sub_i32 s4, s2, 24
@@ -1561,8 +1557,8 @@
; GFX8-NEXT: v_mov_b32_e32 v0, s3
; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX8-NEXT: v_readfirstlane_b32 s3, v0
-; GFX8-NEXT: s_mul_i32 s3, s3, 24
-; GFX8-NEXT: s_sub_i32 s2, s2, s3
+; GFX8-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX8-NEXT: s_add_i32 s2, s2, s3
; GFX8-NEXT: s_cmp_ge_u32 s2, 24
; GFX8-NEXT: s_cselect_b32 s3, 1, 0
; GFX8-NEXT: s_sub_i32 s4, s2, 24
@@ -1593,8 +1589,8 @@
; GFX9-NEXT: s_mul_hi_u32 s4, s3, s4
; GFX9-NEXT: s_add_i32 s3, s3, s4
; GFX9-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX9-NEXT: s_mul_i32 s3, s3, 24
-; GFX9-NEXT: s_sub_i32 s2, s2, s3
+; GFX9-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX9-NEXT: s_add_i32 s2, s2, s3
; GFX9-NEXT: s_cmp_ge_u32 s2, 24
; GFX9-NEXT: s_cselect_b32 s3, 1, 0
; GFX9-NEXT: s_sub_i32 s4, s2, 24
@@ -1625,8 +1621,8 @@
; GFX10-NEXT: s_mul_hi_u32 s4, s3, s4
; GFX10-NEXT: s_add_i32 s3, s3, s4
; GFX10-NEXT: s_mul_hi_u32 s3, s2, s3
-; GFX10-NEXT: s_mul_i32 s3, s3, 24
-; GFX10-NEXT: s_sub_i32 s2, s2, s3
+; GFX10-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX10-NEXT: s_add_i32 s2, s2, s3
; GFX10-NEXT: s_cmp_ge_u32 s2, 24
; GFX10-NEXT: s_cselect_b32 s3, 1, 0
; GFX10-NEXT: s_sub_i32 s4, s2, 24
@@ -1662,8 +1658,8 @@
; GFX11-NEXT: s_add_i32 s3, s3, s4
; GFX11-NEXT: s_mul_hi_u32 s3, s2, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s3, s3, 24
-; GFX11-NEXT: s_sub_i32 s2, s2, s3
+; GFX11-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX11-NEXT: s_add_i32 s2, s2, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_ge_u32 s2, 24
; GFX11-NEXT: s_cselect_b32 s3, 1, 0
@@ -1705,8 +1701,8 @@
; GFX6-NEXT: v_readfirstlane_b32 s5, v3
; GFX6-NEXT: s_add_i32 s4, s4, s5
; GFX6-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX6-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_mul_lo_u32 v3, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GFX6-NEXT: v_add_i32_e32 v3, vcc, v2, v4
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -1738,8 +1734,8 @@
; GFX8-NEXT: v_readfirstlane_b32 s5, v3
; GFX8-NEXT: s_add_i32 s4, s4, s5
; GFX8-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX8-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT: v_mul_lo_u32 v3, v3, v4
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v2, v4
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -1760,17 +1756,17 @@
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, 24
; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX9-NEXT: v_not_b32_e32 v4, 23
; GFX9-NEXT: v_and_b32_e32 v1, 0xffffff, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: s_mul_i32 s5, s4, 0xffffffe8
; GFX9-NEXT: s_mul_hi_u32 s5, s4, s5
; GFX9-NEXT: s_add_i32 s4, s4, s5
; GFX9-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX9-NEXT: v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v4, v[2:3]
; GFX9-NEXT: v_add_u32_e32 v3, 0xffffffe8, v2
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
@@ -1799,8 +1795,7 @@
; GFX10-NEXT: s_mul_hi_u32 s5, s4, s5
; GFX10-NEXT: s_add_i32 s4, s4, s5
; GFX10-NEXT: v_mul_hi_u32 v3, v2, s4
-; GFX10-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, v2, v3
+; GFX10-NEXT: v_mad_u64_u32 v[2:3], s4, 0xffffffe8, v3, v[2:3]
; GFX10-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
@@ -1832,24 +1827,23 @@
; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_add_i32 s0, s0, s1
-; GFX11-NEXT: v_mul_hi_u32 v3, v2, s0
+; GFX11-NEXT: v_mul_hi_u32 v5, v2, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_lo_u32 v3, v3, 24
-; GFX11-NEXT: v_sub_nc_u32_e32 v2, v2, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, 0xffffffe8, v5, v[2:3]
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0xffffffe8, v3
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc_lo
; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0xffffffe8, v2
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_sub_nc_u32_e32 v3, 23, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshl_or_b32 v0, v0, v3, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i24 @llvm.fshr.i24(i24 %lhs, i24 %rhs, i24 %amt)
@@ -1921,11 +1915,11 @@
; GFX6-NEXT: s_or_b32 s5, s5, s9
; GFX6-NEXT: v_readfirstlane_b32 s9, v0
; GFX6-NEXT: s_and_b32 s6, s6, 0xff
-; GFX6-NEXT: s_mul_i32 s9, s9, 24
+; GFX6-NEXT: s_mulk_i32 s9, 0xffe8
; GFX6-NEXT: s_and_b32 s8, 0xffff, s8
; GFX6-NEXT: s_and_b32 s6, 0xffff, s6
; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX6-NEXT: s_sub_i32 s3, s3, s9
+; GFX6-NEXT: s_add_i32 s3, s3, s9
; GFX6-NEXT: s_cmp_ge_u32 s3, 24
; GFX6-NEXT: s_cselect_b32 s9, 1, 0
; GFX6-NEXT: s_sub_i32 s10, s3, 24
@@ -1946,13 +1940,13 @@
; GFX6-NEXT: s_lshr_b32 s3, s7, s3
; GFX6-NEXT: s_or_b32 s4, s4, s3
; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_mul_i32 s3, s3, 24
-; GFX6-NEXT: s_sub_i32 s3, s5, s3
-; GFX6-NEXT: s_cmp_ge_u32 s3, 24
-; GFX6-NEXT: s_cselect_b32 s5, 1, 0
-; GFX6-NEXT: s_sub_i32 s6, s3, 24
-; GFX6-NEXT: s_cmp_lg_u32 s5, 0
-; GFX6-NEXT: s_cselect_b32 s3, s6, s3
+; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX6-NEXT: s_add_i32 s5, s5, s3
+; GFX6-NEXT: s_cmp_ge_u32 s5, 24
+; GFX6-NEXT: s_cselect_b32 s3, 1, 0
+; GFX6-NEXT: s_sub_i32 s6, s5, 24
+; GFX6-NEXT: s_cmp_lg_u32 s3, 0
+; GFX6-NEXT: s_cselect_b32 s3, s6, s5
; GFX6-NEXT: s_cmp_ge_u32 s3, 24
; GFX6-NEXT: s_cselect_b32 s5, 1, 0
; GFX6-NEXT: s_sub_i32 s6, s3, 24
@@ -2050,12 +2044,12 @@
; GFX8-NEXT: s_lshl_b32 s9, s9, 16
; GFX8-NEXT: s_or_b32 s5, s5, s9
; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: s_mul_i32 s9, s9, 24
+; GFX8-NEXT: s_mulk_i32 s9, 0xffe8
; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: s_and_b32 s7, 0xffff, s7
-; GFX8-NEXT: s_sub_i32 s4, s4, s9
+; GFX8-NEXT: s_add_i32 s4, s4, s9
; GFX8-NEXT: s_cmp_ge_u32 s4, 24
; GFX8-NEXT: s_cselect_b32 s9, 1, 0
; GFX8-NEXT: s_sub_i32 s10, s4, 24
@@ -2076,13 +2070,13 @@
; GFX8-NEXT: s_lshr_b32 s2, s2, s4
; GFX8-NEXT: s_or_b32 s0, s0, s2
; GFX8-NEXT: v_readfirstlane_b32 s2, v0
-; GFX8-NEXT: s_mul_i32 s2, s2, 24
-; GFX8-NEXT: s_sub_i32 s2, s5, s2
-; GFX8-NEXT: s_cmp_ge_u32 s2, 24
-; GFX8-NEXT: s_cselect_b32 s4, 1, 0
-; GFX8-NEXT: s_sub_i32 s5, s2, 24
-; GFX8-NEXT: s_cmp_lg_u32 s4, 0
-; GFX8-NEXT: s_cselect_b32 s2, s5, s2
+; GFX8-NEXT: s_mulk_i32 s2, 0xffe8
+; GFX8-NEXT: s_add_i32 s5, s5, s2
+; GFX8-NEXT: s_cmp_ge_u32 s5, 24
+; GFX8-NEXT: s_cselect_b32 s2, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s5, 24
+; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s5
; GFX8-NEXT: s_cmp_ge_u32 s2, 24
; GFX8-NEXT: s_cselect_b32 s4, 1, 0
; GFX8-NEXT: s_sub_i32 s5, s2, 24
@@ -2177,12 +2171,12 @@
; GFX9-NEXT: s_mul_hi_u32 s9, s8, s9
; GFX9-NEXT: s_add_i32 s8, s8, s9
; GFX9-NEXT: s_mul_hi_u32 s9, s4, s8
-; GFX9-NEXT: s_mul_i32 s9, s9, 24
+; GFX9-NEXT: s_mulk_i32 s9, 0xffe8
; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
; GFX9-NEXT: s_and_b32 s1, 0xffff, s1
; GFX9-NEXT: s_and_b32 s7, 0xffff, s7
-; GFX9-NEXT: s_sub_i32 s4, s4, s9
+; GFX9-NEXT: s_add_i32 s4, s4, s9
; GFX9-NEXT: s_cmp_ge_u32 s4, 24
; GFX9-NEXT: s_cselect_b32 s9, 1, 0
; GFX9-NEXT: s_sub_i32 s10, s4, 24
@@ -2201,13 +2195,13 @@
; GFX9-NEXT: s_lshr_b32 s2, s2, s4
; GFX9-NEXT: s_or_b32 s0, s0, s2
; GFX9-NEXT: s_mul_hi_u32 s2, s5, s8
-; GFX9-NEXT: s_mul_i32 s2, s2, 24
-; GFX9-NEXT: s_sub_i32 s2, s5, s2
-; GFX9-NEXT: s_cmp_ge_u32 s2, 24
-; GFX9-NEXT: s_cselect_b32 s4, 1, 0
-; GFX9-NEXT: s_sub_i32 s5, s2, 24
-; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
+; GFX9-NEXT: s_mulk_i32 s2, 0xffe8
+; GFX9-NEXT: s_add_i32 s5, s5, s2
+; GFX9-NEXT: s_cmp_ge_u32 s5, 24
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_sub_i32 s4, s5, 24
+; GFX9-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s5
; GFX9-NEXT: s_cmp_ge_u32 s2, 24
; GFX9-NEXT: s_cselect_b32 s4, 1, 0
; GFX9-NEXT: s_sub_i32 s5, s2, 24
@@ -2299,10 +2293,10 @@
; GFX10-NEXT: s_and_b32 s10, s11, 0xff
; GFX10-NEXT: s_mul_hi_u32 s8, s4, s9
; GFX10-NEXT: s_and_b32 s10, 0xffff, s10
-; GFX10-NEXT: s_mul_i32 s8, s8, 24
+; GFX10-NEXT: s_mulk_i32 s8, 0xffe8
; GFX10-NEXT: s_and_b32 s5, 0xffff, s5
; GFX10-NEXT: s_lshl_b32 s10, s10, 16
-; GFX10-NEXT: s_sub_i32 s4, s4, s8
+; GFX10-NEXT: s_add_i32 s4, s4, s8
; GFX10-NEXT: s_and_b32 s0, 0xffff, s0
; GFX10-NEXT: s_and_b32 s6, 0xffff, s6
; GFX10-NEXT: s_and_b32 s1, 0xffff, s1
@@ -2324,15 +2318,15 @@
; GFX10-NEXT: s_or_b32 s0, s6, s0
; GFX10-NEXT: s_mul_hi_u32 s6, s5, s9
; GFX10-NEXT: s_lshl_b32 s0, s0, s8
-; GFX10-NEXT: s_mul_i32 s6, s6, 24
+; GFX10-NEXT: s_mulk_i32 s6, 0xffe8
; GFX10-NEXT: s_lshr_b32 s2, s2, s4
-; GFX10-NEXT: s_sub_i32 s4, s5, s6
+; GFX10-NEXT: s_add_i32 s5, s5, s6
; GFX10-NEXT: s_or_b32 s0, s0, s2
-; GFX10-NEXT: s_cmp_ge_u32 s4, 24
+; GFX10-NEXT: s_cmp_ge_u32 s5, 24
; GFX10-NEXT: s_cselect_b32 s2, 1, 0
-; GFX10-NEXT: s_sub_i32 s5, s4, 24
+; GFX10-NEXT: s_sub_i32 s4, s5, 24
; GFX10-NEXT: s_cmp_lg_u32 s2, 0
-; GFX10-NEXT: s_cselect_b32 s2, s5, s4
+; GFX10-NEXT: s_cselect_b32 s2, s4, s5
; GFX10-NEXT: s_cmp_ge_u32 s2, 24
; GFX10-NEXT: s_cselect_b32 s4, 1, 0
; GFX10-NEXT: s_sub_i32 s5, s2, 24
@@ -2425,10 +2419,10 @@
; GFX11-NEXT: s_and_b32 s10, s11, 0xff
; GFX11-NEXT: s_mul_hi_u32 s8, s4, s9
; GFX11-NEXT: s_and_b32 s10, 0xffff, s10
-; GFX11-NEXT: s_mul_i32 s8, s8, 24
+; GFX11-NEXT: s_mulk_i32 s8, 0xffe8
; GFX11-NEXT: s_and_b32 s5, 0xffff, s5
; GFX11-NEXT: s_lshl_b32 s10, s10, 16
-; GFX11-NEXT: s_sub_i32 s4, s4, s8
+; GFX11-NEXT: s_add_i32 s4, s4, s8
; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-NEXT: s_and_b32 s6, 0xffff, s6
; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
@@ -2451,15 +2445,15 @@
; GFX11-NEXT: s_or_b32 s0, s6, s0
; GFX11-NEXT: s_mul_hi_u32 s6, s5, s9
; GFX11-NEXT: s_lshl_b32 s0, s0, s8
-; GFX11-NEXT: s_mul_i32 s6, s6, 24
+; GFX11-NEXT: s_mulk_i32 s6, 0xffe8
; GFX11-NEXT: s_lshr_b32 s2, s2, s4
-; GFX11-NEXT: s_sub_i32 s4, s5, s6
+; GFX11-NEXT: s_add_i32 s5, s5, s6
; GFX11-NEXT: s_or_b32 s0, s0, s2
-; GFX11-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-NEXT: s_cmp_ge_u32 s5, 24
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-NEXT: s_sub_i32 s5, s4, 24
+; GFX11-NEXT: s_sub_i32 s4, s5, 24
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s2, s5, s4
+; GFX11-NEXT: s_cselect_b32 s2, s4, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_ge_u32 s2, 24
; GFX11-NEXT: s_cselect_b32 s4, 1, 0
@@ -2518,8 +2512,8 @@
; GFX6-NEXT: s_add_i32 s4, s4, s5
; GFX6-NEXT: v_mul_hi_u32 v6, v4, s4
; GFX6-NEXT: v_mul_hi_u32 v8, v5, s4
-; GFX6-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT: v_mul_lo_u32 v6, v6, v7
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6
; GFX6-NEXT: v_add_i32_e32 v6, vcc, v4, v7
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
; GFX6-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
@@ -2529,11 +2523,11 @@
; GFX6-NEXT: v_sub_i32_e32 v6, vcc, 23, v4
; GFX6-NEXT: v_and_b32_e32 v6, 0xffffff, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v0, v6, v0
-; GFX6-NEXT: v_mul_lo_u32 v6, v8, 24
+; GFX6-NEXT: v_mul_lo_u32 v6, v8, v7
; GFX6-NEXT: v_and_b32_e32 v4, 0xffffff, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v2, v4, v2
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v5, v6
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v6
; GFX6-NEXT: v_add_i32_e32 v4, vcc, v2, v7
; GFX6-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
@@ -2569,8 +2563,8 @@
; GFX8-NEXT: s_add_i32 s4, s4, s5
; GFX8-NEXT: v_mul_hi_u32 v6, v4, s4
; GFX8-NEXT: v_mul_hi_u32 v8, v5, s4
-; GFX8-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v4, v6
+; GFX8-NEXT: v_mul_lo_u32 v6, v6, v7
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v6
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v4, v7
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
@@ -2580,11 +2574,11 @@
; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 23, v4
; GFX8-NEXT: v_and_b32_e32 v6, 0xffffff, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v0, v6, v0
-; GFX8-NEXT: v_mul_lo_u32 v6, v8, 24
+; GFX8-NEXT: v_mul_lo_u32 v6, v8, v7
; GFX8-NEXT: v_and_b32_e32 v4, 0xffffff, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v2
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, v5, v6
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v6
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v2, v7
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
@@ -2605,7 +2599,7 @@
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v6
; GFX9-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffffff, v5
+; GFX9-NEXT: v_not_b32_e32 v8, 23
; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v2
; GFX9-NEXT: v_mul_f32_e32 v6, 0x4f7ffffe, v6
; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v6
@@ -2615,27 +2609,26 @@
; GFX9-NEXT: v_readfirstlane_b32 s4, v6
; GFX9-NEXT: s_mul_i32 s5, s4, 0xffffffe8
; GFX9-NEXT: s_mul_hi_u32 s5, s4, s5
-; GFX9-NEXT: s_add_i32 s4, s4, s5
-; GFX9-NEXT: v_mul_hi_u32 v6, v4, s4
-; GFX9-NEXT: v_mul_hi_u32 v7, v5, s4
-; GFX9-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX9-NEXT: v_mul_lo_u32 v7, v7, 24
-; GFX9-NEXT: v_sub_u32_e32 v4, v4, v6
-; GFX9-NEXT: v_add_u32_e32 v6, 0xffffffe8, v4
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT: v_add_u32_e32 v6, 0xffffffe8, v4
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX9-NEXT: v_sub_u32_e32 v6, 23, v4
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX9-NEXT: s_add_i32 s6, s4, s5
+; GFX9-NEXT: v_mul_hi_u32 v6, v4, s6
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v6, v8, v[4:5]
+; GFX9-NEXT: v_and_b32_e32 v4, 0xffffff, v5
+; GFX9-NEXT: v_mul_hi_u32 v5, v4, s6
+; GFX9-NEXT: v_add_u32_e32 v7, 0xffffffe8, v6
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc
+; GFX9-NEXT: v_add_u32_e32 v7, 0xffffffe8, v6
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v6
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v8, v[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc
+; GFX9-NEXT: v_sub_u32_e32 v7, 23, v6
; GFX9-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, v4, v2
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, v6, v2
-; GFX9-NEXT: v_sub_u32_e32 v2, v5, v7
-; GFX9-NEXT: v_add_u32_e32 v4, 0xffffffe8, v2
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX9-NEXT: v_and_b32_e32 v7, 0xffffff, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, v6, v2
+; GFX9-NEXT: v_lshl_or_b32 v0, v0, v7, v2
+; GFX9-NEXT: v_add_u32_e32 v2, 0xffffffe8, v4
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX9-NEXT: v_add_u32_e32 v4, 0xffffffe8, v2
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, 24, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
@@ -2664,33 +2657,31 @@
; GFX10-NEXT: s_mul_hi_u32 s5, s4, s5
; GFX10-NEXT: s_add_i32 s4, s4, s5
; GFX10-NEXT: v_mul_hi_u32 v6, v4, s4
-; GFX10-NEXT: v_mul_hi_u32 v7, v5, s4
-; GFX10-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX10-NEXT: v_mul_lo_u32 v7, v7, 24
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, v4, v6
-; GFX10-NEXT: v_sub_nc_u32_e32 v5, v5, v7
-; GFX10-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
+; GFX10-NEXT: v_mul_hi_u32 v8, v5, s4
+; GFX10-NEXT: v_mad_u64_u32 v[6:7], s4, 0xffffffe8, v6, v[4:5]
+; GFX10-NEXT: v_mad_u64_u32 v[4:5], s4, 0xffffffe8, v8, v[5:6]
+; GFX10-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v6
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v4
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX10-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc_lo
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX10-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX10-NEXT: v_sub_nc_u32_e32 v6, 23, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff, v6
-; GFX10-NEXT: v_sub_nc_u32_e32 v7, 23, v5
+; GFX10-NEXT: v_sub_nc_u32_e32 v6, 23, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX10-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX10-NEXT: v_sub_nc_u32_e32 v7, 23, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v5, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffffff, v7
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, v4, v3
; GFX10-NEXT: v_lshl_or_b32 v0, v0, v6, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, v1, v4, v3
+; GFX10-NEXT: v_lshl_or_b32 v1, v1, v5, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshr_v2i24:
@@ -2698,10 +2689,11 @@
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, 24
; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
; GFX11-NEXT: v_and_b32_e32 v2, 0xffffff, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xffffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX11-NEXT: v_rcp_iflag_f32_e32 v6, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_dual_mul_f32 v6, 0x4f7ffffe, v6 :: v_dual_lshlrev_b32 v1, 1, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2712,41 +2704,37 @@
; GFX11-NEXT: s_mul_hi_u32 s1, s0, s1
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_hi_u32 v6, v4, s0
-; GFX11-NEXT: v_mul_lo_u32 v6, v6, 24
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_nc_u32_e32 v4, v4, v6
+; GFX11-NEXT: v_mul_hi_u32 v8, v4, s0
+; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffffffe8, v8, v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0xffffffe8, v6
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo
+; GFX11-NEXT: v_mul_hi_u32 v9, v5, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, 0xffffffe8, v9, v[5:6]
; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_and_b32 v5, 0xffffff, v5
-; GFX11-NEXT: v_mul_hi_u32 v7, v5, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0xffffffe8, v4
-; GFX11-NEXT: v_mul_lo_u32 v7, v7, 24
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_nc_u32_e32 v5, v5, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0xffffffe8, v5
-; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0xffffffe8, v7
+; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v7
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc_lo
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_dual_cndmask_b32 v4, v4, v6 :: v_dual_add_nc_u32 v7, 0xffffffe8, v5
; GFX11-NEXT: v_cmp_le_u32_e32 vcc_lo, 24, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_sub_nc_u32_e32 v6, 23, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_and_b32 v4, 0xffffff, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v6, 0xffffff, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_sub_nc_u32_e32 v7, 23, v5
; GFX11-NEXT: v_and_b32_e32 v5, 0xffffff, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, v4, v2
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffffff, v7
; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshl_or_b32 v0, v0, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_lshl_or_b32 v1, v1, v4, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call <2 x i24> @llvm.fshr.v2i24(<2 x i24> %lhs, <2 x i24> %rhs, <2 x i24> %amt)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
index 8981b11..68e0ff3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i32.ll
@@ -340,12 +340,12 @@
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; GISEL-NEXT: v_mul_hi_u32 v7, v0, v4
; GISEL-NEXT: v_mul_hi_u32 v4, v1, v4
-; GISEL-NEXT: v_mul_lo_u32 v8, v7, v3
+; GISEL-NEXT: v_mul_lo_u32 v8, v7, v5
; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v7
-; GISEL-NEXT: v_mul_lo_u32 v10, v4, v3
+; GISEL-NEXT: v_mul_lo_u32 v10, v4, v5
; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v4
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v10
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v3
; GISEL-NEXT: v_cndmask_b32_e64 v7, v7, v9, s[4:5]
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v0, v5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
index 1eb8457..a9e471b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll
@@ -343,9 +343,9 @@
; CHECK-NEXT: v_ashrrev_i32_e32 v1, 20, v1
; CHECK-NEXT: v_lshrrev_b32_e32 v2, 31, v1
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
-; CHECK-NEXT: v_mov_b32_e32 v2, 0x12d8fb
+; CHECK-NEXT: v_mov_b32_e32 v2, 0xffed2705
; CHECK-NEXT: v_mul_lo_u32 v1, v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = srem i32 %num, 1235195
ret i32 %result
@@ -372,10 +372,10 @@
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; GISEL-NEXT: v_mul_hi_u32 v7, v0, v4
; GISEL-NEXT: v_mul_hi_u32 v4, v1, v4
-; GISEL-NEXT: v_mul_lo_u32 v7, v7, v3
-; GISEL-NEXT: v_mul_lo_u32 v4, v4, v3
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v7
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v1, v4
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v5
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v4
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v0, v5
; GISEL-NEXT: v_add_i32_e32 v7, vcc, 0xffed2705, v1
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v3
@@ -398,7 +398,7 @@
; CGP: ; %bb.0:
; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CGP-NEXT: v_mov_b32_e32 v2, 0xd9528441
-; CGP-NEXT: v_mov_b32_e32 v3, 0x12d8fb
+; CGP-NEXT: v_mov_b32_e32 v3, 0xffed2705
; CGP-NEXT: v_mul_hi_i32 v4, v0, v2
; CGP-NEXT: v_mul_hi_i32 v2, v1, v2
; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v0
@@ -411,8 +411,8 @@
; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v6
; CGP-NEXT: v_mul_lo_u32 v4, v4, v3
; CGP-NEXT: v_mul_lo_u32 v2, v2, v3
-; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CGP-NEXT: s_setpc_b64 s[30:31]
%result = srem <2 x i32> %num, <i32 1235195, i32 1235195>
ret <2 x i32> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
index 6ae2f56..95a5652 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
@@ -217,9 +217,9 @@
; CHECK-NEXT: v_lshrrev_b32_e32 v2, 1, v2
; CHECK-NEXT: v_add_i32_e32 v1, vcc, v2, v1
; CHECK-NEXT: v_lshrrev_b32_e32 v1, 20, v1
-; CHECK-NEXT: v_mov_b32_e32 v2, 0x12d8fb
+; CHECK-NEXT: v_mov_b32_e32 v2, 0xffed2705
; CHECK-NEXT: v_mul_lo_u32 v1, v1, v2
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = urem i32 %num, 1235195
ret i32 %result
@@ -230,7 +230,7 @@
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v2, 0xb2a50881
-; CHECK-NEXT: v_mov_b32_e32 v3, 0x12d8fb
+; CHECK-NEXT: v_mov_b32_e32 v3, 0xffed2705
; CHECK-NEXT: v_mul_hi_u32 v4, v0, v2
; CHECK-NEXT: v_mul_hi_u32 v2, v1, v2
; CHECK-NEXT: v_sub_i32_e32 v5, vcc, v0, v4
@@ -243,8 +243,8 @@
; CHECK-NEXT: v_lshrrev_b32_e32 v2, 20, v2
; CHECK-NEXT: v_mul_lo_u32 v4, v4, v3
; CHECK-NEXT: v_mul_lo_u32 v2, v2, v3
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; CHECK-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = urem <2 x i32> %num, <i32 1235195, i32 1235195>
ret <2 x i32> %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index be5543b..14226b3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -21,7 +21,7 @@
; CHECK-NEXT: ; %bb.1: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
; CHECK-NEXT: s_cbranch_execnz .LBB0_4
-; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: .LBB0_2: ; %.split
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: s_setpc_b64 s[30:31]
; CHECK-NEXT: .LBB0_3:
@@ -336,7 +336,7 @@
; CHECK-NEXT: v_subrev_i32_e32 v1, vcc, s2, v0
; CHECK-NEXT: v_cmp_le_u32_e32 vcc, s2, v0
; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT: .LBB1_5:
+; CHECK-NEXT: .LBB1_5: ; %.split
; CHECK-NEXT: v_readfirstlane_b32 s0, v0
; CHECK-NEXT: s_mov_b32 s1, s0
; CHECK-NEXT: ; return to shader part epilog
@@ -769,7 +769,7 @@
; CGP-NEXT: v_sub_i32_e32 v2, vcc, v0, v4
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; CGP-NEXT: .LBB2_4:
+; CGP-NEXT: .LBB2_4: ; %.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
; CGP-NEXT: v_or_b32_e32 v3, v9, v7
; CGP-NEXT: v_mov_b32_e32 v2, 0
@@ -782,7 +782,7 @@
; CGP-NEXT: ; %bb.5: ; %Flow
; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
; CGP-NEXT: s_cbranch_execnz .LBB2_8
-; CGP-NEXT: .LBB2_6:
+; CGP-NEXT: .LBB2_6: ; %.split.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
; CGP-NEXT: s_setpc_b64 s[30:31]
; CGP-NEXT: .LBB2_7:
@@ -968,7 +968,7 @@
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v2, 0x1fb03c31
; CHECK-NEXT: v_mov_b32_e32 v3, 0xd9528440
-; CHECK-NEXT: v_mov_b32_e32 v4, 0x12d8fb
+; CHECK-NEXT: v_mov_b32_e32 v4, 0xffed2705
; CHECK-NEXT: v_mul_lo_u32 v5, v1, v2
; CHECK-NEXT: v_mul_lo_u32 v6, v0, v3
; CHECK-NEXT: v_mul_hi_u32 v7, v0, v2
@@ -993,79 +993,146 @@
; CHECK-NEXT: v_lshr_b64 v[2:3], v[2:3], 20
; CHECK-NEXT: v_mul_lo_u32 v5, v2, v4
; CHECK-NEXT: v_mul_lo_u32 v3, v3, v4
-; CHECK-NEXT: v_mul_hi_u32 v2, v2, v4
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; CHECK-NEXT: v_mul_hi_u32 v4, v2, v4
+; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
; CHECK-NEXT: s_setpc_b64 s[30:31]
%result = urem i64 %num, 1235195
ret i64 %result
}
define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
-; CHECK-LABEL: v_urem_v2i64_oddk_denom:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v4, 0x1fb03c31
-; CHECK-NEXT: v_mov_b32_e32 v5, 0xd9528440
-; CHECK-NEXT: v_mov_b32_e32 v8, 0x12d8fb
-; CHECK-NEXT: v_mul_lo_u32 v6, v1, v4
-; CHECK-NEXT: v_mul_lo_u32 v7, v0, v5
-; CHECK-NEXT: v_mul_hi_u32 v9, v0, v4
-; CHECK-NEXT: v_mul_lo_u32 v10, v1, v5
-; CHECK-NEXT: v_mul_hi_u32 v11, v1, v4
-; CHECK-NEXT: v_mul_hi_u32 v12, v0, v5
-; CHECK-NEXT: v_mul_hi_u32 v13, v1, v5
-; CHECK-NEXT: v_mul_lo_u32 v14, v3, v4
-; CHECK-NEXT: v_mul_lo_u32 v15, v2, v5
-; CHECK-NEXT: v_mul_hi_u32 v16, v2, v4
-; CHECK-NEXT: v_mul_lo_u32 v17, v3, v5
-; CHECK-NEXT: v_mul_hi_u32 v4, v3, v4
-; CHECK-NEXT: v_mul_hi_u32 v18, v2, v5
-; CHECK-NEXT: v_mul_hi_u32 v19, v3, v5
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v7
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v10, v11
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v14, v15
-; CHECK-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v17, v4
-; CHECK-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v5, v9
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v7, v12
-; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, v11, v16
-; CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, v4, v18
-; CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v14, v11
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, v15, v4
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v7, v5
-; CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, v12, v6
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v9, v5
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v10, v7
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v13, v5
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, v19, v7
-; CHECK-NEXT: v_lshr_b64 v[4:5], v[4:5], 20
-; CHECK-NEXT: v_lshr_b64 v[6:7], v[6:7], 20
-; CHECK-NEXT: v_mul_lo_u32 v9, v4, v8
-; CHECK-NEXT: v_mul_lo_u32 v5, v5, v8
-; CHECK-NEXT: v_mul_hi_u32 v4, v4, v8
-; CHECK-NEXT: v_mul_lo_u32 v10, v6, v8
-; CHECK-NEXT: v_mul_lo_u32 v7, v7, v8
-; CHECK-NEXT: v_mul_hi_u32 v6, v6, v8
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; CHECK-NEXT: v_add_i32_e32 v5, vcc, v7, v6
-; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v9
-; CHECK-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; CHECK-NEXT: v_sub_i32_e32 v2, vcc, v2, v10
-; CHECK-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GISEL-LABEL: v_urem_v2i64_oddk_denom:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, 0x1fb03c31
+; GISEL-NEXT: v_mov_b32_e32 v5, 0xd9528440
+; GISEL-NEXT: v_mov_b32_e32 v8, 0x12d8fb
+; GISEL-NEXT: v_mul_lo_u32 v6, v1, v4
+; GISEL-NEXT: v_mul_lo_u32 v7, v0, v5
+; GISEL-NEXT: v_mul_hi_u32 v9, v0, v4
+; GISEL-NEXT: v_mul_lo_u32 v10, v1, v5
+; GISEL-NEXT: v_mul_hi_u32 v11, v1, v4
+; GISEL-NEXT: v_mul_hi_u32 v12, v0, v5
+; GISEL-NEXT: v_mul_hi_u32 v13, v1, v5
+; GISEL-NEXT: v_mul_lo_u32 v14, v3, v4
+; GISEL-NEXT: v_mul_lo_u32 v15, v2, v5
+; GISEL-NEXT: v_mul_hi_u32 v16, v2, v4
+; GISEL-NEXT: v_mul_lo_u32 v17, v3, v5
+; GISEL-NEXT: v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT: v_mul_hi_u32 v18, v2, v5
+; GISEL-NEXT: v_mul_hi_u32 v19, v3, v5
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v14, v15
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v17, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v16
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v4, v18
+; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v15, v4
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v7, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v12, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v9, v5
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v13, v5
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v19, v7
+; GISEL-NEXT: v_lshr_b64 v[4:5], v[4:5], 20
+; GISEL-NEXT: v_lshr_b64 v[6:7], v[6:7], 20
+; GISEL-NEXT: v_mul_lo_u32 v9, v4, v8
+; GISEL-NEXT: v_mul_lo_u32 v5, v5, v8
+; GISEL-NEXT: v_mul_hi_u32 v4, v4, v8
+; GISEL-NEXT: v_mul_lo_u32 v10, v6, v8
+; GISEL-NEXT: v_mul_lo_u32 v7, v7, v8
+; GISEL-NEXT: v_mul_hi_u32 v6, v6, v8
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v6
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v9
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64_oddk_denom:
+; CGP: ; %bb.0:
+; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT: v_mov_b32_e32 v4, 0x1fb03c31
+; CGP-NEXT: v_mov_b32_e32 v5, 0xd9528440
+; CGP-NEXT: v_mov_b32_e32 v8, 0xffed2705
+; CGP-NEXT: v_mul_lo_u32 v6, v1, v4
+; CGP-NEXT: v_mul_lo_u32 v7, v0, v5
+; CGP-NEXT: v_mul_hi_u32 v9, v0, v4
+; CGP-NEXT: v_mul_lo_u32 v10, v1, v5
+; CGP-NEXT: v_mul_hi_u32 v11, v1, v4
+; CGP-NEXT: v_mul_hi_u32 v12, v0, v5
+; CGP-NEXT: v_mul_hi_u32 v13, v1, v5
+; CGP-NEXT: v_mul_lo_u32 v14, v3, v4
+; CGP-NEXT: v_mul_lo_u32 v15, v2, v5
+; CGP-NEXT: v_mul_hi_u32 v16, v2, v4
+; CGP-NEXT: v_mul_lo_u32 v17, v3, v5
+; CGP-NEXT: v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT: v_mul_hi_u32 v18, v2, v5
+; CGP-NEXT: v_mul_hi_u32 v19, v3, v5
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v7
+; CGP-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v10, v11
+; CGP-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v14, v15
+; CGP-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v17, v4
+; CGP-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v7, v12
+; CGP-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v12, vcc, v4, v18
+; CGP-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; CGP-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v14, v11
+; CGP-NEXT: v_add_i32_e32 v10, vcc, v15, v4
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v7, v5
+; CGP-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v6, vcc, v12, v6
+; CGP-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v9, v5
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v13, v5
+; CGP-NEXT: v_add_i32_e32 v7, vcc, v19, v7
+; CGP-NEXT: v_lshr_b64 v[4:5], v[4:5], 20
+; CGP-NEXT: v_lshr_b64 v[6:7], v[6:7], 20
+; CGP-NEXT: v_mul_lo_u32 v9, v4, v8
+; CGP-NEXT: v_mul_lo_u32 v5, v5, v8
+; CGP-NEXT: v_mul_hi_u32 v10, v4, v8
+; CGP-NEXT: v_mul_lo_u32 v11, v6, v8
+; CGP-NEXT: v_mul_lo_u32 v7, v7, v8
+; CGP-NEXT: v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT: v_sub_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT: v_sub_i32_e32 v5, vcc, v7, v6
+; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v9
+; CGP-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v11
+; CGP-NEXT: v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CGP-NEXT: s_setpc_b64 s[30:31]
%result = urem <2 x i64> %num, <i64 1235195, i64 1235195>
ret <2 x i64> %result
}
@@ -1090,7 +1157,7 @@
; CHECK-NEXT: ; %bb.1: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
; CHECK-NEXT: s_cbranch_execnz .LBB7_4
-; CHECK-NEXT: .LBB7_2:
+; CHECK-NEXT: .LBB7_2: ; %.split
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: s_setpc_b64 s[30:31]
; CHECK-NEXT: .LBB7_3:
@@ -1673,7 +1740,7 @@
; CGP-NEXT: v_sub_i32_e32 v3, vcc, v0, v2
; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; CGP-NEXT: .LBB8_4:
+; CGP-NEXT: .LBB8_4: ; %.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
; CGP-NEXT: v_or_b32_e32 v3, v7, v10
; CGP-NEXT: v_mov_b32_e32 v2, 0
@@ -1686,7 +1753,7 @@
; CGP-NEXT: ; %bb.5: ; %Flow
; CGP-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
; CGP-NEXT: s_cbranch_execnz .LBB8_8
-; CGP-NEXT: .LBB8_6:
+; CGP-NEXT: .LBB8_6: ; %.split.split
; CGP-NEXT: s_or_b64 exec, exec, s[4:5]
; CGP-NEXT: s_setpc_b64 s[30:31]
; CGP-NEXT: .LBB8_7:
