Merging r288418:
------------------------------------------------------------------------
r288418 | tnorthover | 2016-12-01 13:31:59 -0800 (Thu, 01 Dec 2016) | 13 lines
AArch64: fix 128-bit cmpxchg at -O0 (again, again).
This time the issue is fortunately just a simple mistake rather than a horrible
design spectre. I thought SUBS/SBCS provided sufficient NZCV flags for
comparing two 64-bit values, but they don't.
The fix is slightly clunkier in AArch64 because we can't use conditional
execution to emit a pair of CMPs. Traditionally an "icmp ne i128" would map to
an EOR/EOR/ORR/CBNZ, but that uses more registers so it's easier to go with a
CSET/CINC/CBNZ combination. Slightly less efficient, but this is -O0 anyway.
Thanks to Anton Korobeynikov for pointing out the issue.
------------------------------------------------------------------------
git-svn-id: https://llvm.org/svn/llvm-project/llvm/branches/release_39@288846 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 5e477d3..a1c9825 100644
--- a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -718,13 +718,21 @@
.addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
.addOperand(DesiredLo)
.addImm(0);
- BuildMI(LoadCmpBB, DL, TII->get(AArch64::SBCSXr), AArch64::XZR)
+ BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
+ .addReg(AArch64::WZR)
+ .addReg(AArch64::WZR)
+ .addImm(AArch64CC::EQ);
+ BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
.addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
- .addOperand(DesiredHi);
- BuildMI(LoadCmpBB, DL, TII->get(AArch64::Bcc))
- .addImm(AArch64CC::NE)
- .addMBB(DoneBB)
- .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);
+ .addOperand(DesiredHi)
+ .addImm(0);
+ BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
+ .addReg(StatusReg, RegState::Kill)
+ .addReg(StatusReg, RegState::Kill)
+ .addImm(AArch64CC::EQ);
+ BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW))
+ .addReg(StatusReg, RegState::Kill)
+ .addMBB(DoneBB);
LoadCmpBB->addSuccessor(DoneBB);
LoadCmpBB->addSuccessor(StoreBB);
diff --git a/test/CodeGen/AArch64/cmpxchg-O0.ll b/test/CodeGen/AArch64/cmpxchg-O0.ll
index bd6c63f..aed1aa4 100644
--- a/test/CodeGen/AArch64/cmpxchg-O0.ll
+++ b/test/CodeGen/AArch64/cmpxchg-O0.ll
@@ -65,8 +65,10 @@
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
; CHECK: ldaxp [[OLD_LO:x[0-9]+]], [[OLD_HI:x[0-9]+]], [x0]
; CHECK: cmp [[OLD_LO]], x2
-; CHECK: sbcs xzr, [[OLD_HI]], x3
-; CHECK: b.ne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK: cset [[CMP_TMP:w[0-9]+]], ne
+; CHECK: cmp [[OLD_HI]], x3
+; CHECK: cinc [[CMP:w[0-9]+]], [[CMP_TMP]], ne
+; CHECK: cbnz [[CMP]], [[DONE:.LBB[0-9]+_[0-9]+]]
; CHECK: stlxp [[STATUS:w[0-9]+]], x4, x5, [x0]
; CHECK: cbnz [[STATUS]], [[RETRY]]
; CHECK: [[DONE]]:
@@ -88,8 +90,10 @@
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
; CHECK: ldaxp [[OLD_LO:x[0-9]+]], [[OLD_HI:x[0-9]+]], [x0]
; CHECK: cmp [[OLD_LO]], [[DESIRED_LO]]
-; CHECK: sbcs xzr, [[OLD_HI]], [[DESIRED_HI]]
-; CHECK: b.ne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK: cset [[CMP_TMP:w[0-9]+]], ne
+; CHECK: cmp [[OLD_HI]], [[DESIRED_HI]]
+; CHECK: cinc [[CMP:w[0-9]+]], [[CMP_TMP]], ne
+; CHECK: cbnz [[CMP]], [[DONE:.LBB[0-9]+_[0-9]+]]
; CHECK: stlxp [[STATUS:w[0-9]+]], [[NEW_LO]], [[NEW_HI]], [x0]
; CHECK: cbnz [[STATUS]], [[RETRY]]
; CHECK: [[DONE]]:
