[StaticDataLayout][MemProf] Annotate string literal hotness by making use of data access profiles. (#178333)
The change is gated under a new option
`memprof-annotate-string-literal-section-prefix` so we can flag-gate it
for rollout purposes.
A follow-up PR https://github.com/llvm/llvm-project/pull/178336/changes
updates the codegen pass to reconcile the hotness similar to the
reconciliation for other global variables.
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
index 748c4cb..72d2c4d 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp
@@ -92,6 +92,14 @@
"memprof-annotate-static-data-prefix", cl::init(false), cl::Hidden,
cl::desc("If true, annotate the static data section prefix"));
+// FIXME: This option is added for incremental rollout purposes.
+// After the option, string literal partitioning should be implied by
+// AnnotateStaticDataSectionPrefix above and this option should be cleaned up.
+static cl::opt<bool> AnnotateStringLiteralSectionPrefix(
+ "memprof-annotate-string-literal-section-prefix", cl::init(false),
+ cl::Hidden,
+ cl::desc("If true, annotate the string literal data section prefix"));
+
// Matching statistics
STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");
STATISTIC(NumOfMemProfMismatch,
@@ -233,6 +241,21 @@
<< Reason << ".\n");
}
+// Computes the LLVM version of MD5 hash for the content of a string
+// literal.
+static std::optional<uint64_t>
+getStringContentHash(const GlobalVariable &GVar) {
+ auto *Initializer = GVar.getInitializer();
+ if (!Initializer)
+ return std::nullopt;
+ if (auto *C = dyn_cast<ConstantDataSequential>(Initializer))
+ if (C->isString()) {
+ // Note the hash computed for the literal would include the null byte.
+ return llvm::MD5Hash(C->getAsString());
+ }
+ return std::nullopt;
+}
+
// Structure for tracking info about matched allocation contexts for use with
// -memprof-print-match-info and -memprof-print-matched-alloc-stack.
struct AllocMatchInfo {
@@ -932,19 +955,27 @@
}
StringRef Name = GVar.getName();
+ SymbolHandleRef Handle = SymbolHandleRef(Name);
// Skip string literals as their mangled names don't stay stable across
// binary releases.
- // TODO: Track string content hash in the profiles and compute it inside the
- // compiler to categeorize the hotness string literals.
+ if (!AnnotateStringLiteralSectionPrefix)
+ if (Name.starts_with(".str"))
+ continue;
+
if (Name.starts_with(".str")) {
- LLVM_DEBUG(dbgs() << "Skip annotating string literal " << Name << "\n");
- continue;
+ std::optional<uint64_t> Hash = getStringContentHash(GVar);
+ if (!Hash) {
+ LLVM_DEBUG(dbgs() << "Cannot compute content hash for string literal "
+ << Name << "\n");
+ continue;
+ }
+ Handle = SymbolHandleRef(Hash.value());
}
// DataAccessProfRecord's get* methods will canonicalize the name under the
// hood before looking it up, so optimizer doesn't need to do it.
std::optional<DataAccessProfRecord> Record =
- DataAccessProf->getProfileRecord(Name);
+ DataAccessProf->getProfileRecord(Handle);
// Annotate a global variable as hot if it has non-zero sampled count, and
// annotate it as cold if it's seen in the profiled binary
// file but doesn't have any access sample.
@@ -956,7 +987,7 @@
Changed |= GVar.setSectionPrefix("hot");
LLVM_DEBUG(dbgs() << "Global variable " << Name
<< " is annotated as hot\n");
- } else if (DataAccessProf->isKnownColdSymbol(Name)) {
+ } else if (DataAccessProf->isKnownColdSymbol(Handle)) {
++NumOfMemProfColdGlobalVars;
Changed |= GVar.setSectionPrefix("unlikely");
Changed = true;
diff --git a/llvm/test/Transforms/PGOProfile/data-access-profile.ll b/llvm/test/Transforms/PGOProfile/data-access-profile.ll
index 205184b..7c062a9 100644
--- a/llvm/test/Transforms/PGOProfile/data-access-profile.ll
+++ b/llvm/test/Transforms/PGOProfile/data-access-profile.ll
@@ -1,5 +1,5 @@
; REQUIRES: asserts
-; asserts are required for -debug-only=<pass-name>
+; asserts are required for -debug-only=<pass-name> and -stats
; RUN: rm -rf %t && split-file %s %t && cd %t
@@ -10,11 +10,21 @@
;; Run optimizer pass on an IR module without IR functions, and test that global
;; variables in the module could be annotated (i.e., no early return),
; RUN: opt -passes='memprof-use<profile-filename=memprof.profdata>' -memprof-annotate-static-data-prefix \
-; RUN: -debug-only=memprof -stats -S funcless-module.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOG,IR,STAT
+; RUN: -debug-only=memprof -stats -S funcless-module.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOGCOMMON,IRCOMMON,IR,STAT
+
+;; Add '-memprof-annotate-string-literal-section-prefix' to RUN command above.
+; RUN: opt -passes='memprof-use<profile-filename=memprof.profdata>' -memprof-annotate-static-data-prefix \
+; RUN: -memprof-annotate-string-literal-section-prefix \
+; RUN: -debug-only=memprof -stats -S funcless-module.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOGCOMMON,LOGSTR,IRCOMMON,IRSTR,STRSTAT
;; Run optimizer pass on the IR, and check the section prefix.
; RUN: opt -passes='memprof-use<profile-filename=memprof.profdata>' -memprof-annotate-static-data-prefix \
-; RUN: -debug-only=memprof -stats -S input.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOG,IR,STAT
+; RUN: -debug-only=memprof -stats -S input.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOGCOMMON,IR,STAT
+
+;; Add '-memprof-annotate-string-literal-section-prefix' to RUN command above.
+; RUN: opt -passes='memprof-use<profile-filename=memprof.profdata>' -memprof-annotate-static-data-prefix \
+; RUN: -memprof-annotate-string-literal-section-prefix \
+; RUN: -debug-only=memprof -stats -S input.ll -o - 2>&1 | FileCheck %s --check-prefixes=LOGCOMMON,LOGSTR,IRSTR,STRSTAT
;; Run memprof without providing memprof data. Test that IR has module flag
;; `EnableDataAccessProf` as 0.
@@ -26,40 +36,59 @@
; RUN: opt -passes='memprof-use<profile-filename=memprof.profdata>' \
; RUN: -debug-only=memprof -stats -S input.ll -o - | FileCheck %s --check-prefix=FLAGLESS --implicit-check-not="section_prefix"
-; LOG: Skip annotating string literal .str
-; LOG: Global variable var1 is annotated as hot
-; LOG: Global variable var2.llvm.125 is annotated as hot
-; LOG: Global variable bar is not annotated
-; LOG: Global variable foo is annotated as unlikely
-; LOG: Skip annotation for var3 due to explicit section name.
-; LOG: Skip annotation for var4 due to explicit section name.
-; LOG: Skip annotation for llvm.fake_var due to name starts with `llvm.`.
-; LOG: Skip annotation for qux due to linker declaration.
+; LOGSTR: Global variable .str is annotated as hot
+
+;; Common log lines
+; LOGCOMMON: Global variable var1 is annotated as hot
+; LOGCOMMON: Global variable var2.llvm.125 is annotated as hot
+; LOGCOMMON: Global variable bar is not annotated
+; LOGCOMMON: Global variable foo is annotated as unlikely
+; LOGCOMMON: Skip annotation for var3 due to explicit section name.
+; LOGCOMMON: Skip annotation for var4 due to explicit section name.
+; LOGCOMMON: Skip annotation for llvm.fake_var due to name starts with `llvm.`.
+; LOGCOMMON: Skip annotation for qux due to linker declaration.
+
+; LOGSTR: Global variable .str.llvm.98765 is annotated as unlikely
+; LOGSTR: Global variable .str.2 is not annotated
;; String literals are not annotated.
; IR: @.str = unnamed_addr constant [5 x i8] c"abcde"
; IR-NOT: section_prefix
-; IR: @var1 = global i32 123, !section_prefix !0
+; IR-SAME: {{$}}
+
+;; .str is hot
+; IRSTR: @.str = unnamed_addr constant [5 x i8] c"abcde", !section_prefix !0
+
+; IRCOMMON: @var1 = global i32 123, !section_prefix !0
;; @var.llvm.125 will be canonicalized to @var2 for profile look-up.
-; IR-NEXT: @var2.llvm.125 = global i64 0, !section_prefix !0
+; IRCOMMON-NEXT: @var2.llvm.125 = global i64 0, !section_prefix !0
;; @bar is not seen in hot symbol or known symbol set, so it won't get a section
;; prefix. Test this by testing that there is no section_prefix between @bar and
;; @foo.
-; IR-NEXT: @bar = global i16 3
-; IR-NOT: !section_prefix
+; IRCOMMON-NEXT: @bar = global i16 3
+; IRCOMMON-NOT: !section_prefix
+; IRCOMMON-SAME: {{$}}
;; @foo is unlikely.
-; IR-NEXT: @foo = global i8 2, !section_prefix !1
+; IRCOMMON-NEXT: @foo = global i8 2, !section_prefix !1
-; IR-NEXT: @var3 = constant [2 x i32] [i32 12345, i32 6789], section "sec1"
-; IR-NEXT: @var4 = constant [1 x i64] [i64 98765] #0
+; IRCOMMON-NEXT: @var3 = constant [2 x i32] [i32 12345, i32 6789], section "sec1"
+; IRCOMMON-NEXT: @var4 = constant [1 x i64] [i64 98765] #0
-; IR: @llvm.fake_var = global i32 123
-; IR-NOT: !section_prefix
-; IR: @qux = external global i64
-; IR-NOT: !section_prefix
+; IRCOMMON: @llvm.fake_var = global i32 123
+; IRCOMMON-NOT: !section_prefix
+; IRCOMMON-SAME: {{$}}
+; IRCOMMON: @qux = external global i64
+; IRCOMMON-NOT: !section_prefix
+; IRCOMMON-SAME: {{$}}
+
+;; @.str.llvm.98765 is unlikely and @.str.2 has no section prefix.
+; IRSTR: @.str.llvm.98765 = constant [5 x i8] c"Joins", align 1, !section_prefix !1
+; IRSTR: @.str.2 = constant [15 x i8] c"*ptr == nullptr", align 1
+; IRSTR-NOT: section_prefix
+; IRSTR-SAME: {{$}}
; IR: attributes #0 = { "rodata-section"="sec2" }
@@ -75,6 +104,11 @@
; STAT: 2 memprof - Number of global vars annotated with 'hot' section prefix.
; STAT: 1 memprof - Number of global vars with unknown hotness (no section prefix).
+; STRSTAT: 2 memprof - Number of global vars annotated with 'unlikely' section prefix.
+; STRSTAT: 2 memprof - Number of global vars with user-specified section (not annotated).
+; STRSTAT: 3 memprof - Number of global vars annotated with 'hot' section prefix.
+; STRSTAT: 2 memprof - Number of global vars with unknown hotness (no section prefix).
+
;--- memprof.yaml
---
DataAccessProfiles:
@@ -83,11 +117,14 @@
AccessCount: 1000
- Symbol: var2
AccessCount: 5
- - Hash: 101010
+ # 4211217683648370347 is the hash for 'abcde'
+ - Hash: 4211217683648370347
AccessCount: 145
KnownColdSymbols:
- foo
- KnownColdStrHashes: [ 999, 1001 ]
+ # 1698928196410683810 is the hash for 'Joins'
+ # 9184168289490324740 is the hash for 'ptr = nullptr'. This string doesn't exist in the IR.
+ KnownColdStrHashes: [ 1698928196410683810, 9184168289490324740 ]
...
;--- memprof-no-dap.yaml
---
@@ -114,13 +151,15 @@
@.str = unnamed_addr constant [5 x i8] c"abcde"
@var1 = global i32 123
-@var2.llvm.125 = global i64 0
+@var2.llvm.125 = global i64 0
@bar = global i16 3
@foo = global i8 2
@var3 = constant [2 x i32][i32 12345, i32 6789], section "sec1"
@var4 = constant [1 x i64][i64 98765] #0
@llvm.fake_var = global i32 123
@qux = external global i64
+@.str.llvm.98765 = constant [5 x i8] c"Joins", align 1
+@.str.2 = constant [15 x i8] c"*ptr == nullptr", align 1
define i32 @func() {
%a = load i32, ptr @var1
@@ -148,6 +187,8 @@
@var4 = constant [1 x i64][i64 98765] #0
@llvm.fake_var = global i32 123
@qux = external global i64
+@.str.llvm.98765 = constant [5 x i8] c"Joins", align 1
+@.str.2 = constant [15 x i8] c"*ptr == nullptr", align 1
attributes #0 = { "rodata-section"="sec2" }
