Skip to content

Commit 87476bb

Browse files
mtrofinmtrofin
committed
[PGO][profcheck] ignore explicitly cold functions
1 parent c5459a0 commit 87476bb

File tree

6 files changed

+103
-19
lines changed

6 files changed

+103
-19
lines changed

‎llvm/lib/Transforms/Utils/ProfileVerify.cpp

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@
2020
#include "llvm/IR/MDBuilder.h"
2121
#include "llvm/IR/ProfDataUtils.h"
2222
#include "llvm/Support/BranchProbability.h"
23+
#include "llvm/Support/CommandLine.h"
2324

2425
using namespace llvm;
26+
static cl::opt<int64_t>
27+
DefaultFunctionEntryCount("profcheck-default-function-entry-count",
28+
cl::init(1000));
2529
namespace {
2630
class ProfileInjector {
2731
Function &F;
@@ -63,6 +67,19 @@ bool ProfileInjector::inject() {
6367
// will get the same BPI it does if the injector wasn't running.
6468
auto &BPI = FAM.getResult<BranchProbabilityAnalysis>(F);
6569

70+
// Inject a function count if there's none. It's reasonable for a pass to
71+
// want to clear the MD_prof of a function with zero entry count. If the
72+
// original profile (iFDO or AFDO) is empty for a function, it's simpler to
73+
// require assigning it the 0-entry count explicitly than to mark every branch
74+
// as cold (we do want some explicit information in the spirit of what this
75+
// verifier wants to achieve - make dropping / corrupting MD_prof
76+
// unit-testable)
77+
if (!F.getEntryCount(/*AllowSynthetic=*/true))
78+
F.setEntryCount(DefaultFunctionEntryCount);
79+
// If there is an entry count that's 0, then don't bother injecting. We won't
80+
// verify these either.
81+
if (F.getEntryCount(/*AllowSynthetic=*/true)->getCount() == 0)
82+
return false;
6683
bool Changed = false;
6784
for (auto &BB : F) {
6885
auto *Term = getTerminatorBenefitingFromMDProf(BB);
@@ -119,11 +136,20 @@ PreservedAnalyses ProfileInjectorPass::run(Function &F,
119136

120137
PreservedAnalyses ProfileVerifierPass::run(Function &F,
121138
FunctionAnalysisManager &FAM) {
139+
const auto EntryCount = F.getEntryCount(/*AllowSynthetic=*/true);
140+
if (!EntryCount) {
141+
F.getContext().emitError("Profile verification failed: function entry "
142+
"count missing (set to 0 if cold)");
143+
return PreservedAnalyses::all();
144+
}
145+
if (EntryCount->getCount() == 0)
146+
return PreservedAnalyses::all();
122147
for (const auto &BB : F)
123148
if (const auto *Term =
124149
ProfileInjector::getTerminatorBenefitingFromMDProf(BB))
125150
if (!Term->getMetadata(LLVMContext::MD_prof))
126-
F.getContext().emitError("Profile verification failed");
151+
F.getContext().emitError(
152+
"Profile verification failed: branch annotation missing");
127153

128-
return PreservedAnalyses::none();
154+
return PreservedAnalyses::all();
129155
}

‎llvm/test/Transforms/PGOProfile/prof-inject-existing.ll

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
; Test that prof-inject does not modify existing metadata (incl. "unknown")
2+
3+
; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
4+
5+
define void @foo(i32 %i) {
6+
%c = icmp eq i32 %i, 0
7+
br i1 %c, label %yes, label %no, !prof !0
8+
yes:
9+
br i1 %c, label %yes2, label %no, !prof !1
10+
yes2:
11+
ret void
12+
no:
13+
ret void
14+
}
15+
16+
!0 = !{!"branch_weights", i32 1, i32 2}
17+
!1 = !{!"unknown"}
18+
; CHECK: define void @foo(i32 %i) !prof !0
19+
; CHECK: br i1 %c, label %yes, label %no, !prof !1
20+
; CHECK: !0 = !{!"function_entry_count", i64 1000}
21+
; CHECK: !1 = !{!"branch_weights", i32 1, i32 2}
22+
; CHECK: !2 = !{!"unknown"}

‎llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
; Test that prof-inject only injects missing metadata
22

3-
; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
3+
; RUN: opt -passes=prof-inject -profcheck-default-function-entry-count=10 %s -S -o - | FileCheck %s
44

55
define void @foo(i32 %i) {
66
%c = icmp eq i32 %i, 0
@@ -13,8 +13,26 @@ no:
1313
ret void
1414
}
1515

16+
define void @cold(i32 %i) !prof !1 {
17+
%c = icmp eq i32 %i, 0
18+
br i1 %c, label %yes, label %no
19+
yes:
20+
br i1 %c, label %yes2, label %no
21+
yes2:
22+
ret void
23+
no:
24+
ret void
25+
}
1626
!0 = !{!"branch_weights", i32 1, i32 2}
17-
; CHECK: br i1 %c, label %yes, label %no, !prof !0
18-
; CHECK: br i1 %c, label %yes2, label %no, !prof !1
19-
; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
20-
; CHECK: !1 = !{!"branch_weights", i32 3, i32 5}
27+
!1 = !{!"function_entry_count", i32 0}
28+
29+
; CHECK-LABEL: @foo
30+
; CHECK: br i1 %c, label %yes, label %no, !prof !1
31+
; CHECK: br i1 %c, label %yes2, label %no, !prof !2
32+
; CHECK-LABEL: @cold
33+
; CHECK: br i1 %c, label %yes, label %no{{$}}
34+
; CHECK: br i1 %c, label %yes2, label %no{{$}}
35+
; CHECK: !0 = !{!"function_entry_count", i64 10}
36+
; CHECK: !1 = !{!"branch_weights", i32 1, i32 2}
37+
; CHECK: !2 = !{!"branch_weights", i32 3, i32 5}
38+
; CHECK: !3 = !{!"function_entry_count", i32 0}

‎llvm/test/Transforms/PGOProfile/prof-verify-existing.ll

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
; Test that prof-inject does not modify existing metadata (incl. "unknown")
22

3-
; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
43
; RUN: opt -passes=prof-verify %s -S --disable-output
54

6-
define void @foo(i32 %i) {
5+
define void @foo(i32 %i) !prof !0 {
76
%c = icmp eq i32 %i, 0
8-
br i1 %c, label %yes, label %no, !prof !0
7+
br i1 %c, label %yes, label %no, !prof !1
98
yes:
10-
br i1 %c, label %yes2, label %no, !prof !1
9+
br i1 %c, label %yes2, label %no, !prof !2
1110
yes2:
1211
ret void
1312
no:
1413
ret void
1514
}
1615

17-
!0 = !{!"branch_weights", i32 1, i32 2}
18-
!1 = !{!"unknown"}
19-
; CHECK: br i1 %c, label %yes, label %no, !prof !0
20-
; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
21-
; CHECK: !1 = !{!"unknown"}
16+
!0 = !{!"function_entry_count", i32 1}
17+
!1 = !{!"branch_weights", i32 1, i32 2}
18+
!2 = !{!"unknown"}
19+
; CHECK: define void @foo(i32 %i) !prof !0
20+
; CHECK: br i1 %c, label %yes, label %no, !prof !1
21+
; CHECK: !0 = !{!"function_entry_count", i64 1}
22+
; CHECK: !1 = !{!"branch_weights", i32 1, i32 2}
23+
; CHECK: !2 = !{!"unknown"}

‎llvm/test/Transforms/PGOProfile/prof-verify-known-cold.ll

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
; Test prof-verify for functions explicitly marked as cold
2+
3+
; RUN: opt -passes=prof-inject,prof-verify %s -o - 2>&1 | FileCheck %s
4+
5+
define void @foo(i32 %i) !prof !0 {
6+
%c = icmp eq i32 %i, 0
7+
br i1 %c, label %yes, label %no
8+
yes:
9+
ret void
10+
no:
11+
ret void
12+
}
13+
!0 = !{!"function_entry_count", i32 0}
14+
15+
; CHECK-NOT: Profile verification failed

‎llvm/test/Transforms/PGOProfile/prof-verify.ll

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,17 @@
55
; RUN: opt -passes=prof-inject,prof-verify %s --disable-output
66
; RUN: opt -enable-profcheck %s -S -o - | FileCheck %s --check-prefix=INJECT
77

8-
define void @foo(i32 %i) {
8+
define void @foo(i32 %i) !prof !0 {
99
%c = icmp eq i32 %i, 0
1010
br i1 %c, label %yes, label %no
1111
yes:
1212
ret void
1313
no:
1414
ret void
1515
}
16+
!0 = !{!"function_entry_count", i32 1}
1617

17-
; INJECT: br i1 %c, label %yes, label %no, !prof !0
18-
; INJECT: !0 = !{!"branch_weights", i32 3, i32 5}
18+
; INJECT: br i1 %c, label %yes, label %no, !prof !1
19+
; INJECT: !1 = !{!"branch_weights", i32 3, i32 5}
1920

2021
; VERIFY: Profile verification failed

0 commit comments

Comments
 (0)