[HotColdSplitting] Refine definition of unlikelyExecuted
The splitting pass uses its 'unlikelyExecuted' predicate to statically
decide which blocks are cold.
- Do not treat noreturn calls as if they are cold unless they are actually
marked cold. This is motivated by functions like exit() and longjmp(), which
are not beneficial to outline.
- Do not treat inline asm as an outlining barrier. In practice asm("") is
frequently used to inhibit basic block merging; enabling outlining in this case
results in substantial memory savings.
- Treat invokes of cold functions as cold.
As a drive-by, remove the 'exceptionHandlingFunctions' predicate, because it's
no longer needed. The pass can identify & outline blocks dominated by EH pads,
so there's no need to special-case __cxa_begin_catch etc.
Differential Revision: https://reviews.llvm.org/D54244
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@348640 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Transforms/IPO/HotColdSplitting.cpp b/lib/Transforms/IPO/HotColdSplitting.cpp
index 93d2b75..704ddbe 100644
--- a/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -26,6 +26,7 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
+#include "llvm/IR/CallSite.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
@@ -98,36 +99,27 @@
return !(isa<ReturnInst>(I) || isa<IndirectBrInst>(I));
}
-static bool exceptionHandlingFunctions(const CallInst *CI) {
- auto F = CI->getCalledFunction();
- if (!F)
- return false;
- auto FName = F->getName();
- return FName == "__cxa_begin_catch" ||
- FName == "__cxa_free_exception" ||
- FName == "__cxa_allocate_exception" ||
- FName == "__cxa_begin_catch" ||
- FName == "__cxa_end_catch";
-}
-
-static bool unlikelyExecuted(const BasicBlock &BB) {
- if (blockEndsInUnreachable(BB))
- return true;
+bool unlikelyExecuted(BasicBlock &BB) {
// Exception handling blocks are unlikely executed.
if (BB.isEHPad())
return true;
- for (const Instruction &I : BB)
- if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
- // The block is cold if it calls functions tagged as cold or noreturn.
- if (CI->hasFnAttr(Attribute::Cold) ||
- CI->hasFnAttr(Attribute::NoReturn) ||
- exceptionHandlingFunctions(CI))
+
+ // The block is cold if it calls/invokes a cold function.
+ for (Instruction &I : BB)
+ if (auto CS = CallSite(&I))
+ if (CS.hasFnAttr(Attribute::Cold))
return true;
- // Assume that inline assembly is hot code.
- if (isa<InlineAsm>(CI->getCalledValue()))
+ // The block is cold if it has an unreachable terminator, unless it's
+ // preceded by a call to a (possibly warm) noreturn call (e.g. longjmp).
+ if (blockEndsInUnreachable(BB)) {
+ if (auto *CI =
+ dyn_cast_or_null<CallInst>(BB.getTerminator()->getPrevNode()))
+ if (CI->hasFnAttr(Attribute::NoReturn))
return false;
- }
+ return true;
+ }
+
return false;
}
diff --git a/test/Transforms/HotColdSplit/eh-pads.ll b/test/Transforms/HotColdSplit/eh-pads.ll
index cf413b4..f9b55f3 100644
--- a/test/Transforms/HotColdSplit/eh-pads.ll
+++ b/test/Transforms/HotColdSplit/eh-pads.ll
@@ -6,12 +6,6 @@
; CHECK-LABEL: define {{.*}}@foo(
; CHECK: landingpad
; CHECK: sideeffect(i32 2)
-
-; CHECK-LABEL: define {{.*}}@foo.cold.1(
-; CHECK: sideeffect(i32 0)
-; CHECK: sideeffect(i32 1)
-; CHECK: sink
-
define void @foo(i32 %cond) personality i8 0 {
entry:
invoke void @llvm.donothing() to label %normal unwind label %exception
@@ -32,6 +26,38 @@
ret void
}
+; CHECK-LABEL: define {{.*}}@bar(
+; CHECK-NOT: landingpad
+define void @bar(i32 %cond) personality i8 0 {
+entry:
+ br i1 undef, label %exit, label %continue
+
+exit:
+ ret void
+
+continue:
+ invoke void @sink() to label %normal unwind label %exception
+
+exception:
+ ; Note: EH pads are not candidates for region entry points.
+ %cleanup = landingpad i8 cleanup
+ ret void
+
+normal:
+ call void @sideeffect(i32 0)
+ call void @sideeffect(i32 1)
+ ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: sideeffect(i32 0)
+; CHECK: sideeffect(i32 1)
+; CHECK: sink
+
+; CHECK-LABEL: define {{.*}}@bar.cold.1(
+; CHECK: sideeffect(i32 0)
+; CHECK: sideeffect(i32 1)
+
declare void @sideeffect(i32)
declare void @sink() cold
diff --git a/test/Transforms/HotColdSplit/noreturn.ll b/test/Transforms/HotColdSplit/noreturn.ll
new file mode 100644
index 0000000..66de934
--- /dev/null
+++ b/test/Transforms/HotColdSplit/noreturn.ll
@@ -0,0 +1,73 @@
+; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+; Don't outline noreturn calls which aren't explicitly marked cold.
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK-NOT: foo.cold.1
+define void @foo(i32, %struct.__jmp_buf_tag*) {
+ %3 = icmp eq i32 %0, 0
+ tail call void @_Z10sideeffectv()
+ br i1 %3, label %5, label %4
+
+; <label>:4: ; preds = %2
+ tail call void @longjmp(%struct.__jmp_buf_tag* %1, i32 0)
+ unreachable
+
+; <label>:5: ; preds = %2
+ ret void
+}
+
+; Do outline noreturn calls marked cold.
+
+; CHECK-LABEL: define {{.*}}@bar(
+; CHECK: call {{.*}}@bar.cold.1(
+define void @bar(i32) {
+ %2 = icmp eq i32 %0, 0
+ tail call void @_Z10sideeffectv()
+ br i1 %2, label %sink, label %exit
+
+sink:
+ tail call void @_Z10sideeffectv()
+ tail call void @_Z10sideeffectv()
+ tail call void @_Z10sideeffectv()
+ call void @llvm.trap()
+ unreachable
+
+exit:
+ ret void
+}
+
+; Do outline noreturn calls preceded by a cold call.
+
+; CHECK-LABEL: define {{.*}}@baz(
+; CHECK: call {{.*}}@baz.cold.1(
+define void @baz(i32, %struct.__jmp_buf_tag*) {
+ %3 = icmp eq i32 %0, 0
+ tail call void @_Z10sideeffectv()
+ br i1 %3, label %5, label %4
+
+; <label>:4: ; preds = %2
+ call void @sink()
+ tail call void @longjmp(%struct.__jmp_buf_tag* %1, i32 0)
+ unreachable
+
+; <label>:5: ; preds = %2
+ ret void
+}
+
+; CHECK-LABEL: define {{.*}}@bar.cold.1(
+; CHECK: call {{.*}}@llvm.trap(
+
+declare void @sink() cold
+
+declare void @llvm.trap() noreturn cold
+
+declare void @_Z10sideeffectv()
+
+declare void @longjmp(%struct.__jmp_buf_tag*, i32) noreturn nounwind
diff --git a/test/Transforms/HotColdSplit/outline-cold-asm.ll b/test/Transforms/HotColdSplit/outline-cold-asm.ll
new file mode 100644
index 0000000..f368503
--- /dev/null
+++ b/test/Transforms/HotColdSplit/outline-cold-asm.ll
@@ -0,0 +1,27 @@
+; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@fun(
+; CHECK: call {{.*}}@fun.cold.1(
+
+; CHECK-LABEL: define {{.*}}@fun.cold.1(
+; CHECK: asm ""
+
+define void @fun() {
+entry:
+ br i1 undef, label %if.then, label %if.else
+
+if.then:
+ ret void
+
+if.else:
+ call void asm "", ""()
+ call void @sink()
+ call void @sink()
+ call void @sink()
+ ret void
+}
+
+declare void @sink() cold