[PGO][profcheck] ignore explicitly cold functions

Author: mtrofin

llvm/lib/Transforms/Utils/ProfileVerify.cpp

@@ -20,8 +20,12 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/ProfDataUtils.h"
 #include "llvm/Support/BranchProbability.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
+static cl::opt<int64_t>
+    DefaultFunctionEntryCount("profcheck-default-fct-entry-count",
+                              cl::init(1000));
 namespace {
 class ProfileInjector {
   Function &F;
@@ -63,6 +67,17 @@ bool ProfileInjector::inject() {
   // will get the same BPI it does if the injector wasn't running.
   auto &BPI = FAM.getResult<BranchProbabilityAnalysis>(F);
 
+  // Inject a function count if there's none. It's reasonable for a pass to
+  // want to clear the MD_prof of a function with zero entry count. From a
+  // metadata economy perspective, if a profile comes empty for a function, it's
+  // cheaper to assign it the 0-entry count explicitly than to mark every branch
+  // as cold.
+  if (!F.getEntryCount(true))
+    F.setEntryCount(DefaultFunctionEntryCount);
+  // If there is an entry count that's 0, then don't bother injecting. We won't
+  // verify these either.
+  if (F.getEntryCount(true)->getCount() == 0)
+    return false;
   bool Changed = false;
   for (auto &BB : F) {
     auto *Term = getTerminatorBenefitingFromMDProf(BB);
@@ -119,11 +134,13 @@ PreservedAnalyses ProfileInjectorPass::run(Function &F,
 
 PreservedAnalyses ProfileVerifierPass::run(Function &F,
                                            FunctionAnalysisManager &FAM) {
+  if (!F.getEntryCount(true) || F.getEntryCount(true)->getCount() == 0)
+    return PreservedAnalyses::all();
   for (const auto &BB : F)
     if (const auto *Term =
             ProfileInjector::getTerminatorBenefitingFromMDProf(BB))
       if (!Term->getMetadata(LLVMContext::MD_prof))
         F.getContext().emitError("Profile verification failed");
 
-  return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
 }

llvm/test/Transforms/PGOProfile/prof-verify-as-needed.ll

@@ -1,6 +1,6 @@
 ; Test that prof-inject only injects missing metadata
 
-; RUN: opt -passes=prof-inject %s -S -o - | FileCheck %s
+; RUN: opt -passes=prof-inject -profcheck-default-fct-entry-count=10 %s -S -o - | FileCheck %s
 
 define void @foo(i32 %i) {
   %c = icmp eq i32 %i, 0
@@ -13,8 +13,26 @@ no:
   ret void
 }
 
+define void @cold(i32 %i) !prof !1 {
+  %c = icmp eq i32 %i, 0
+  br i1 %c, label %yes, label %no
+yes:
+  br i1 %c, label %yes2, label %no
+yes2:
+  ret void
+no:
+  ret void
+}
 !0 = !{!"branch_weights", i32 1, i32 2}
-; CHECK: br i1 %c, label %yes, label %no, !prof !0
-; CHECK: br i1 %c, label %yes2, label %no, !prof !1
-; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
-; CHECK: !1 = !{!"branch_weights", i32 3, i32 5}
+!1 = !{!"function_entry_count", i32 0}
+
+; CHECK-LABEL: @foo
+; CHECK: br i1 %c, label %yes, label %no, !prof !1
+; CHECK: br i1 %c, label %yes2, label %no, !prof !2
+; CHECK-LABEL: @cold
+; CHECK: br i1 %c, label %yes, label %no{{$}}
+; CHECK: br i1 %c, label %yes2, label %no{{$}}
+; CHECK: !0 = !{!"function_entry_count", i64 10}
+; CHECK: !1 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !2 = !{!"branch_weights", i32 3, i32 5}
+; CHECK: !3 = !{!"function_entry_count", i32 0}

llvm/test/Transforms/PGOProfile/prof-verify-existing.ll

@@ -16,6 +16,8 @@ no:
 
 !0 = !{!"branch_weights", i32 1, i32 2}
 !1 = !{!"unknown"}
-; CHECK: br i1 %c, label %yes, label %no, !prof !0
-; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
-; CHECK: !1 = !{!"unknown"}
+; CHECK: define void @foo(i32 %i) !prof !0
+; CHECK: br i1 %c, label %yes, label %no, !prof !1
+; CHECK: !0 = !{!"function_entry_count", i64 1000}
+; CHECK: !1 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !2 = !{!"unknown"}

llvm/test/Transforms/PGOProfile/prof-verify-known-cold.ll

@@ -0,0 +1,15 @@
+; Test prof-verify for functions explicitly marked as cold
+
+; RUN: opt -passes=prof-inject,prof-verify %s -o - 2>&1 | FileCheck %s
+
+define void @foo(i32 %i) !prof !0 {
+  %c = icmp eq i32 %i, 0
+  br i1 %c, label %yes, label %no
+yes:
+  ret void
+no:
+  ret void
+}
+!0 = !{!"function_entry_count", i32 0}
+
+; CHECK-NOT: Profile verification failed

llvm/test/Transforms/PGOProfile/prof-verify.ll

@@ -5,16 +5,17 @@
 ; RUN: opt -passes=prof-inject,prof-verify %s --disable-output
 ; RUN: opt -enable-profcheck %s -S -o - | FileCheck %s --check-prefix=INJECT
 
-define void @foo(i32 %i) {
+define void @foo(i32 %i) !prof !0 {
   %c = icmp eq i32 %i, 0
   br i1 %c, label %yes, label %no
 yes:
   ret void
 no:
   ret void
 }
+!0 = !{!"function_entry_count", i32 1}
 
-; INJECT: br i1 %c, label %yes, label %no, !prof !0
-; INJECT: !0 = !{!"branch_weights", i32 3, i32 5}
+; INJECT: br i1 %c, label %yes, label %no, !prof !1
+; INJECT: !1 = !{!"branch_weights", i32 3, i32 5}
 
 ; VERIFY: Profile verification failed