llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp

//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
//
//===----------------------------------------------------------------------===//

#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/PostDominators.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/ValueSymbolTable.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/SpecialCaseList.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/EscapeEnumerator.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"

using namespace llvm;

#define DEBUG_TYPE "sancov"

const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
const char SanCovTracePCEntryName[] = "__sanitizer_cov_trace_pc_entry";
const char SanCovTracePCExitName[] = "__sanitizer_cov_trace_pc_exit";
const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
const char SanCovLoad1[] = "__sanitizer_cov_load1";
const char SanCovLoad2[] = "__sanitizer_cov_load2";
const char SanCovLoad4[] = "__sanitizer_cov_load4";
const char SanCovLoad8[] = "__sanitizer_cov_load8";
const char SanCovLoad16[] = "__sanitizer_cov_load16";
const char SanCovStore1[] = "__sanitizer_cov_store1";
const char SanCovStore2[] = "__sanitizer_cov_store2";
const char SanCovStore4[] = "__sanitizer_cov_store4";
const char SanCovStore8[] = "__sanitizer_cov_store8";
const char SanCovStore16[] = "__sanitizer_cov_store16";
const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
const char SanCovModuleCtorTracePcGuardName[] =
    "sancov.module_ctor_trace_pc_guard";
const char SanCovModuleCtor8bitCountersName[] =
    "sancov.module_ctor_8bit_counters";
const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
static const uint64_t SanCtorAndDtorPriority = 2;

const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";

const char SanCovGuardsSectionName[] = "sancov_guards";
const char SanCovCountersSectionName[] = "sancov_cntrs";
const char SanCovBoolFlagSectionName[] = "sancov_bools";
const char SanCovPCsSectionName[] = "sancov_pcs";
const char SanCovCFsSectionName[] = "sancov_cfs";
const char SanCovCallbackGateSectionName[] = "sancov_gate";

const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";
const char SanCovLowestStackName[] = "__sancov_lowest_stack";
const char SanCovCallbackGateName[] = "__sancov_should_track";

static cl::opt<int> ClCoverageLevel(
    "sanitizer-coverage-level",
    cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
             "3: all blocks and critical edges"),
    cl::Hidden);

static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",
                               cl::desc("Experimental pc tracing"), cl::Hidden);

static cl::opt<bool> ClTracePCEntryExit(
    "sanitizer-coverage-trace-pc-entry-exit",
    cl::desc("pc tracing with separate entry/exit callbacks"), cl::Hidden);

static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",
                                    cl::desc("pc tracing with a guard"),
                                    cl::Hidden);

// If true, we create a global variable that contains PCs of all instrumented
// BBs, put this global into a named section, and pass this section's bounds
// to __sanitizer_cov_pcs_init.
// This way the coverage instrumentation does not need to acquire the PCs
// at run-time. Works with trace-pc-guard, inline-8bit-counters, and
// inline-bool-flag.
static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",
                                     cl::desc("create a static PC table"),
                                     cl::Hidden);

static cl::opt<bool>
    ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",
                         cl::desc("increments 8-bit counter for every edge"),
                         cl::Hidden);

static cl::opt<bool>
    ClSancovDropCtors("sanitizer-coverage-drop-ctors",
                      cl::desc("do not emit module ctors for global counters"),
                      cl::Hidden);

static cl::opt<bool>
    ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",
                     cl::desc("sets a boolean flag for every edge"),
                     cl::Hidden);

static cl::opt<bool>
    ClCMPTracing("sanitizer-coverage-trace-compares",
                 cl::desc("Tracing of CMP and similar instructions"),
                 cl::Hidden);

static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",
                                  cl::desc("Tracing of DIV instructions"),
                                  cl::Hidden);

static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",
                                   cl::desc("Tracing of load instructions"),
                                   cl::Hidden);

static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",
                                    cl::desc("Tracing of store instructions"),
                                    cl::Hidden);

static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",
                                  cl::desc("Tracing of GEP instructions"),
                                  cl::Hidden);

static cl::opt<bool>
    ClPruneBlocks("sanitizer-coverage-prune-blocks",
                  cl::desc("Reduce the number of instrumented blocks"),
                  cl::Hidden, cl::init(true));

static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",
                                  cl::desc("max stack depth tracing"),
                                  cl::Hidden);

static cl::opt<int> ClStackDepthCallbackMin(
    "sanitizer-coverage-stack-depth-callback-min",
    cl::desc("max stack depth tracing should use callback and only when "
             "stack depth more than specified"),
    cl::Hidden);

static cl::opt<bool>
    ClCollectCF("sanitizer-coverage-control-flow",
                cl::desc("collect control flow for each function"), cl::Hidden);

static cl::opt<bool> ClGatedCallbacks(
    "sanitizer-coverage-gated-trace-callbacks",
    cl::desc("Gate the invocation of the tracing callbacks on a global variable"
             ". Currently only supported for trace-pc-guard and trace-cmp."),
    cl::Hidden, cl::init(false));

namespace {

SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
  SanitizerCoverageOptions Res;
  switch (LegacyCoverageLevel) {
  case 0:
    Res.CoverageType = SanitizerCoverageOptions::SCK_None;
    break;
  case 1:
    Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
    break;
  case 2:
    Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
    break;
  case 3:
    Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
    break;
  case 4:
    Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
    Res.IndirectCalls = true;
    break;
  }
  return Res;
}

SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
  // Sets CoverageType and IndirectCalls.
  SanitizerCoverageOptions CLOpts = getOptions(ClCoverageLevel);
  Options.CoverageType = std::max(Options.CoverageType, CLOpts.CoverageType);
  Options.IndirectCalls |= CLOpts.IndirectCalls;
  Options.TraceCmp |= ClCMPTracing;
  Options.TraceDiv |= ClDIVTracing;
  Options.TraceGep |= ClGEPTracing;
  Options.TracePC |= ClTracePC;
  Options.TracePCEntryExit |= ClTracePCEntryExit;
  Options.TracePCGuard |= ClTracePCGuard;
  Options.Inline8bitCounters |= ClInline8bitCounters;
  Options.InlineBoolFlag |= ClInlineBoolFlag;
  Options.PCTable |= ClCreatePCTable;
  Options.NoPrune |= !ClPruneBlocks;
  Options.StackDepth |= ClStackDepth;
  Options.StackDepthCallbackMin = std::max(Options.StackDepthCallbackMin,
                                           ClStackDepthCallbackMin.getValue());
  Options.TraceLoads |= ClLoadTracing;
  Options.TraceStores |= ClStoreTracing;
  Options.GatedCallbacks |= ClGatedCallbacks;
  if (!Options.TracePCGuard && !Options.TracePC && !Options.TracePCEntryExit &&
      !Options.Inline8bitCounters && !Options.StackDepth &&
      !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
    Options.TracePCGuard = true; // TracePCGuard is default.
  Options.CollectControlFlow |= ClCollectCF;
  return Options;
}

class ModuleSanitizerCoverage {
public:
  using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;
  using PostDomTreeCallback =
      function_ref<const PostDominatorTree &(Function &F)>;

  ModuleSanitizerCoverage(Module &M, DomTreeCallback DTCallback,
                          PostDomTreeCallback PDTCallback,
                          const SanitizerCoverageOptions &Options,
                          const SpecialCaseList *Allowlist,
                          const SpecialCaseList *Blocklist)
      : M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),
        Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}

  bool instrumentModule();

private:
  void createFunctionControlFlow(Function &F);
  void instrumentFunction(Function &F);
  void InjectCoverageForIndirectCalls(Function &F,
                                      ArrayRef<Instruction *> IndirCalls);
  void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,
                         Value *&FunctionGateCmp);
  void InjectTraceForDiv(Function &F,
                         ArrayRef<BinaryOperator *> DivTraceTargets);
  void InjectTraceForGep(Function &F,
                         ArrayRef<GetElementPtrInst *> GepTraceTargets);
  void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
                                    ArrayRef<StoreInst *> Stores);
  void InjectTraceForExits(Function &F);
  void InjectTraceForSwitch(Function &F,
                            ArrayRef<Instruction *> SwitchTraceTargets,
                            Value *&FunctionGateCmp);
  bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
                      Value *&FunctionGateCmp, bool IsLeafFunc);
  GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
                                                    Function &F, Type *Ty,
                                                    const char *Section);
  GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
  void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
  Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,
                                Instruction *I);
  Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);
  void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
                             Value *&FunctionGateCmp, bool IsLeafFunc);
  Function *CreateInitCallsForSections(Module &M, const char *CtorName,
                                       const char *InitFunctionName, Type *Ty,
                                       const char *Section);
  std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
                                                Type *Ty);

  std::string getSectionName(const std::string &Section) const;
  std::string getSectionStart(const std::string &Section) const;
  std::string getSectionEnd(const std::string &Section) const;

  Module &M;
  DomTreeCallback DTCallback;
  PostDomTreeCallback PDTCallback;

  FunctionCallee SanCovStackDepthCallback;
  FunctionCallee SanCovTracePCIndir;
  FunctionCallee SanCovTracePC, SanCovTracePCGuard;
  FunctionCallee SanCovTracePCEntry, SanCovTracePCExit;
  std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
  std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
  std::array<FunctionCallee, 5> SanCovLoadFunction;
  std::array<FunctionCallee, 5> SanCovStoreFunction;
  std::array<FunctionCallee, 2> SanCovTraceDivFunction;
  FunctionCallee SanCovTraceGepFunction;
  FunctionCallee SanCovTraceSwitchFunction;
  GlobalVariable *SanCovLowestStack;
  GlobalVariable *SanCovCallbackGate;
  Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
  Module *CurModule;
  Triple TargetTriple;
  LLVMContext *C;
  const DataLayout *DL;

  GlobalVariable *FunctionGuardArray;       // for trace-pc-guard.
  GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
  GlobalVariable *FunctionBoolArray;        // for inline-bool-flag.
  GlobalVariable *FunctionPCsArray;         // for pc-table.
  GlobalVariable *FunctionCFsArray;         // for control flow table
  SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
  SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;

  SanitizerCoverageOptions Options;

  const SpecialCaseList *Allowlist;
  const SpecialCaseList *Blocklist;
};
} // namespace

SanitizerCoveragePass::SanitizerCoveragePass(
    SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
    const std::vector<std::string> &AllowlistFiles,
    const std::vector<std::string> &BlocklistFiles)
    : Options(std::move(Options)),
      VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {
  if (AllowlistFiles.size() > 0)
    Allowlist = SpecialCaseList::createOrDie(AllowlistFiles, *this->VFS);
  if (BlocklistFiles.size() > 0)
    Blocklist = SpecialCaseList::createOrDie(BlocklistFiles, *this->VFS);
}

PreservedAnalyses SanitizerCoveragePass::run(Module &M,
                                             ModuleAnalysisManager &MAM) {
  auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
  auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {
    return FAM.getResult<DominatorTreeAnalysis>(F);
  };
  auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {
    return FAM.getResult<PostDominatorTreeAnalysis>(F);
  };
  ModuleSanitizerCoverage ModuleSancov(M, DTCallback, PDTCallback,
                                       OverrideFromCL(Options), Allowlist.get(),
                                       Blocklist.get());
  if (!ModuleSancov.instrumentModule())
    return PreservedAnalyses::all();

  PreservedAnalyses PA = PreservedAnalyses::none();
  // GlobalsAA is considered stateless and does not get invalidated unless
  // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
  // make changes that require GlobalsAA to be invalidated.
  PA.abandon<GlobalsAA>();
  return PA;
}

std::pair<Value *, Value *>
ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
                                           Type *Ty) {
  // Use ExternalWeak so that if all sections are discarded due to section
  // garbage collection, the linker will not report undefined symbol errors.
  // Windows defines the start/stop symbols in compiler-rt so no need for
  // ExternalWeak.
  GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
                                          ? GlobalVariable::ExternalLinkage
                                          : GlobalVariable::ExternalWeakLinkage;
  GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,
                                                getSectionStart(Section));
  SecStart->setVisibility(GlobalValue::HiddenVisibility);
  GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,
                                              getSectionEnd(Section));
  SecEnd->setVisibility(GlobalValue::HiddenVisibility);
  if (!TargetTriple.isOSBinFormatCOFF())
    return std::make_pair(SecStart, SecEnd);

  // Account for the fact that on windows-msvc __start_* symbols actually
  // point to a uint64_t before the start of the array.
  auto *GEP = ConstantExpr::getPtrAdd(
      SecStart, ConstantInt::get(IntptrTy, sizeof(uint64_t)));
  return std::make_pair(GEP, SecEnd);
}

Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
    Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
    const char *Section) {
  if (ClSancovDropCtors)
    return nullptr;
  auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
  auto SecStart = SecStartEnd.first;
  auto SecEnd = SecStartEnd.second;
  Function *CtorFunc;
  std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(
      M, CtorName, InitFunctionName, {PtrTy, PtrTy}, {SecStart, SecEnd});
  assert(CtorFunc->getName() == CtorName);

  if (TargetTriple.supportsCOMDAT()) {
    // Use comdat to dedup CtorFunc.
    CtorFunc->setComdat(M.getOrInsertComdat(CtorName));
    appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority, CtorFunc);
  } else {
    appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority);
  }

  if (TargetTriple.isOSBinFormatCOFF()) {
    // In COFF files, if the contructors are set as COMDAT (they are because
    // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
    // functions and data) is used, the constructors get stripped. To prevent
    // this, give the constructors weak ODR linkage and ensure the linker knows
    // to include the sancov constructor. This way the linker can deduplicate
    // the constructors but always leave one copy.
    CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
  }
  return CtorFunc;
}

bool ModuleSanitizerCoverage::instrumentModule() {
  if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
    return false;
  if (Allowlist &&
      !Allowlist->inSection("coverage", "src", M.getSourceFileName()))
    return false;
  if (Blocklist &&
      Blocklist->inSection("coverage", "src", M.getSourceFileName()))
    return false;
  C = &(M.getContext());
  DL = &M.getDataLayout();
  CurModule = &M;
  TargetTriple = M.getTargetTriple();
  FunctionGuardArray = nullptr;
  Function8bitCounterArray = nullptr;
  FunctionBoolArray = nullptr;
  FunctionPCsArray = nullptr;
  FunctionCFsArray = nullptr;
  IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
  PtrTy = PointerType::getUnqual(*C);
  Type *VoidTy = Type::getVoidTy(*C);
  IRBuilder<> IRB(*C);
  Int64Ty = IRB.getInt64Ty();
  Int32Ty = IRB.getInt32Ty();
  Int16Ty = IRB.getInt16Ty();
  Int8Ty = IRB.getInt8Ty();
  Int1Ty = IRB.getInt1Ty();

  SanCovTracePCIndir =
      M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy);
  // Make sure smaller parameters are zero-extended to i64 if required by the
  // target ABI.
  AttributeList SanCovTraceCmpZeroExtAL;
  SanCovTraceCmpZeroExtAL =
      SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);
  SanCovTraceCmpZeroExtAL =
      SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);

  SanCovTraceCmpFunction[0] =
      M.getOrInsertFunction(SanCovTraceCmp1, SanCovTraceCmpZeroExtAL, VoidTy,
                            IRB.getInt8Ty(), IRB.getInt8Ty());
  SanCovTraceCmpFunction[1] =
      M.getOrInsertFunction(SanCovTraceCmp2, SanCovTraceCmpZeroExtAL, VoidTy,
                            IRB.getInt16Ty(), IRB.getInt16Ty());
  SanCovTraceCmpFunction[2] =
      M.getOrInsertFunction(SanCovTraceCmp4, SanCovTraceCmpZeroExtAL, VoidTy,
                            IRB.getInt32Ty(), IRB.getInt32Ty());
  SanCovTraceCmpFunction[3] =
      M.getOrInsertFunction(SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty);

  SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
      SanCovTraceConstCmp1, SanCovTraceCmpZeroExtAL, VoidTy, Int8Ty, Int8Ty);
  SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
      SanCovTraceConstCmp2, SanCovTraceCmpZeroExtAL, VoidTy, Int16Ty, Int16Ty);
  SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
      SanCovTraceConstCmp4, SanCovTraceCmpZeroExtAL, VoidTy, Int32Ty, Int32Ty);
  SanCovTraceConstCmpFunction[3] =
      M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty);

  // Loads.
  SanCovLoadFunction[0] = M.getOrInsertFunction(SanCovLoad1, VoidTy, PtrTy);
  SanCovLoadFunction[1] = M.getOrInsertFunction(SanCovLoad2, VoidTy, PtrTy);
  SanCovLoadFunction[2] = M.getOrInsertFunction(SanCovLoad4, VoidTy, PtrTy);
  SanCovLoadFunction[3] = M.getOrInsertFunction(SanCovLoad8, VoidTy, PtrTy);
  SanCovLoadFunction[4] = M.getOrInsertFunction(SanCovLoad16, VoidTy, PtrTy);
  // Stores.
  SanCovStoreFunction[0] = M.getOrInsertFunction(SanCovStore1, VoidTy, PtrTy);
  SanCovStoreFunction[1] = M.getOrInsertFunction(SanCovStore2, VoidTy, PtrTy);
  SanCovStoreFunction[2] = M.getOrInsertFunction(SanCovStore4, VoidTy, PtrTy);
  SanCovStoreFunction[3] = M.getOrInsertFunction(SanCovStore8, VoidTy, PtrTy);
  SanCovStoreFunction[4] = M.getOrInsertFunction(SanCovStore16, VoidTy, PtrTy);

  {
    AttributeList AL;
    AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);
    SanCovTraceDivFunction[0] =
        M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty());
  }
  SanCovTraceDivFunction[1] =
      M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty);
  SanCovTraceGepFunction =
      M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy);
  SanCovTraceSwitchFunction =
      M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, PtrTy);

  SanCovLowestStack = M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy);
  if (SanCovLowestStack->getValueType() != IntptrTy) {
    C->emitError(StringRef("'") + SanCovLowestStackName +
                 "' should not be declared by the user");
    return true;
  }
  SanCovLowestStack->setThreadLocalMode(
      GlobalValue::ThreadLocalMode::InitialExecTLSModel);
  if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
    SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy));

  if (Options.GatedCallbacks) {
    if (!Options.TracePCGuard && !Options.TraceCmp) {
      C->emitError(StringRef("'") + ClGatedCallbacks.ArgStr +
                   "' is only supported with trace-pc-guard or trace-cmp");
      return true;
    }

    SanCovCallbackGate = cast<GlobalVariable>(
        M.getOrInsertGlobal(SanCovCallbackGateName, Int64Ty));
    SanCovCallbackGate->setSection(
        getSectionName(SanCovCallbackGateSectionName));
    SanCovCallbackGate->setInitializer(Constant::getNullValue(Int64Ty));
    SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);
    SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);
    appendToCompilerUsed(M, SanCovCallbackGate);
  }

  SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy);
  SanCovTracePCEntry = M.getOrInsertFunction(SanCovTracePCEntryName, VoidTy);
  SanCovTracePCExit = M.getOrInsertFunction(SanCovTracePCExitName, VoidTy);
  SanCovTracePCGuard =
      M.getOrInsertFunction(SanCovTracePCGuardName, VoidTy, PtrTy);

  SanCovStackDepthCallback =
      M.getOrInsertFunction(SanCovStackDepthCallbackName, VoidTy);

  for (auto &F : M)
    instrumentFunction(F);

  Function *Ctor = nullptr;

  if (FunctionGuardArray)
    Ctor = CreateInitCallsForSections(M, SanCovModuleCtorTracePcGuardName,
                                      SanCovTracePCGuardInitName, Int32Ty,
                                      SanCovGuardsSectionName);
  if (Function8bitCounterArray)
    Ctor = CreateInitCallsForSections(M, SanCovModuleCtor8bitCountersName,
                                      SanCov8bitCountersInitName, Int8Ty,
                                      SanCovCountersSectionName);
  if (FunctionBoolArray) {
    Ctor = CreateInitCallsForSections(M, SanCovModuleCtorBoolFlagName,
                                      SanCovBoolFlagInitName, Int1Ty,
                                      SanCovBoolFlagSectionName);
  }
  if (Ctor && Options.PCTable) {
    auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrTy);
    FunctionCallee InitFunction =
        declareSanitizerInitFunction(M, SanCovPCsInitName, {PtrTy, PtrTy});
    IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
    IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});
  }

  if (Ctor && Options.CollectControlFlow) {
    auto SecStartEnd = CreateSecStartEnd(M, SanCovCFsSectionName, IntptrTy);
    FunctionCallee InitFunction =
        declareSanitizerInitFunction(M, SanCovCFsInitName, {PtrTy, PtrTy});
    IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
    IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});
  }

  appendToUsed(M, GlobalsToAppendToUsed);
  appendToCompilerUsed(M, GlobalsToAppendToCompilerUsed);
  return true;
}

// True if block has successors and it dominates all of them.
static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {
  if (succ_empty(BB))
    return false;

  return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) {
    return DT.dominates(BB, SUCC);
  });
}

// True if block has predecessors and it postdominates all of them.
static bool isFullPostDominator(const BasicBlock *BB,
                                const PostDominatorTree &PDT) {
  if (pred_empty(BB))
    return false;

  return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) {
    return PDT.dominates(BB, PRED);
  });
}

static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
                                  const DominatorTree &DT,
                                  const PostDominatorTree &PDT,
                                  const SanitizerCoverageOptions &Options) {
  // Don't insert coverage for blocks containing nothing but unreachable: we
  // will never call __sanitizer_cov() for them, so counting them in
  // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
  // percentage. Also, unreachable instructions frequently have no debug
  // locations.
  if (isa<UnreachableInst>(BB->getFirstNonPHIOrDbgOrLifetime()))
    return false;

  // Don't insert coverage into blocks without a valid insertion point
  // (catchswitch blocks).
  if (BB->getFirstInsertionPt() == BB->end())
    return false;

  if (Options.NoPrune || &F.getEntryBlock() == BB)
    return true;

  if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
      &F.getEntryBlock() != BB)
    return false;

  // Do not instrument full dominators, or full post-dominators with multiple
  // predecessors.
  return !isFullDominator(BB, DT) &&
         !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
}

// Returns true iff From->To is a backedge.
// A twist here is that we treat From->To as a backedge if
//   * To dominates From or
//   * To->UniqueSuccessor dominates From
static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
                       const DominatorTree &DT) {
  if (DT.dominates(To, From))
    return true;
  if (auto Next = To->getUniqueSuccessor())
    if (DT.dominates(Next, From))
      return true;
  return false;
}

// Prunes uninteresting Cmp instrumentation:
//   * CMP instructions that feed into loop backedge branch.
//
// Note that Cmp pruning is controlled by the same flag as the
// BB pruning.
static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,
                             const SanitizerCoverageOptions &Options) {
  if (!Options.NoPrune)
    if (CMP->hasOneUse())
      if (auto BR = dyn_cast<CondBrInst>(CMP->user_back()))
        for (BasicBlock *B : BR->successors())
          if (IsBackEdge(BR->getParent(), B, DT))
            return false;
  return true;
}

void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
  if (F.empty())
    return;
  if (F.getName().contains(".module_ctor"))
    return; // Should not instrument sanitizer init functions.
  if (F.getName().starts_with("__sanitizer_"))
    return; // Don't instrument __sanitizer_* callbacks.
  // Don't touch available_externally functions, their actual body is elewhere.
  if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
    return;
  // Don't instrument MSVC CRT configuration helpers. They may run before normal
  // initialization.
  if (F.getName() == "__local_stdio_printf_options" ||
      F.getName() == "__local_stdio_scanf_options")
    return;
  if (isa<UnreachableInst>(F.getEntryBlock().getTerminator()))
    return;
  // Don't instrument functions using SEH for now. Splitting basic blocks like
  // we do for coverage breaks WinEHPrepare.
  // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
  if (F.hasPersonalityFn() &&
      isAsynchronousEHPersonality(classifyEHPersonality(F.getPersonalityFn())))
    return;
  if (Allowlist && !Allowlist->inSection("coverage", "fun", F.getName()))
    return;
  if (Blocklist && Blocklist->inSection("coverage", "fun", F.getName()))
    return;
  // Do not apply any instrumentation for naked functions.
  if (F.hasFnAttribute(Attribute::Naked))
    return;
  if (F.hasFnAttribute(Attribute::NoSanitizeCoverage))
    return;
  if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
    return;
  if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
    SplitAllCriticalEdges(
        F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
  }
  SmallVector<Instruction *, 8> IndirCalls;
  SmallVector<BasicBlock *, 16> BlocksToInstrument;
  SmallVector<Instruction *, 8> CmpTraceTargets;
  SmallVector<Instruction *, 8> SwitchTraceTargets;
  SmallVector<BinaryOperator *, 8> DivTraceTargets;
  SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
  SmallVector<LoadInst *, 8> Loads;
  SmallVector<StoreInst *, 8> Stores;

  const DominatorTree &DT = DTCallback(F);
  const PostDominatorTree &PDT = PDTCallback(F);
  bool IsLeafFunc = true;

  for (auto &BB : F) {
    if (shouldInstrumentBlock(F, &BB, DT, PDT, Options))
      BlocksToInstrument.push_back(&BB);
    for (auto &Inst : BB) {
      if (Options.IndirectCalls) {
        CallBase *CB = dyn_cast<CallBase>(&Inst);
        if (CB && CB->isIndirectCall())
          IndirCalls.push_back(&Inst);
      }
      if (Options.TraceCmp) {
        if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst))
          if (IsInterestingCmp(CMP, DT, Options))
            CmpTraceTargets.push_back(&Inst);
        if (isa<SwitchInst>(&Inst))
          SwitchTraceTargets.push_back(&Inst);
      }
      if (Options.TraceDiv)
        if (BinaryOperator *BO = dyn_cast<BinaryOperator>(&Inst))
          if (BO->getOpcode() == Instruction::SDiv ||
              BO->getOpcode() == Instruction::UDiv)
            DivTraceTargets.push_back(BO);
      if (Options.TraceGep)
        if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&Inst))
          GepTraceTargets.push_back(GEP);
      if (Options.TraceLoads)
        if (LoadInst *LI = dyn_cast<LoadInst>(&Inst))
          Loads.push_back(LI);
      if (Options.TraceStores)
        if (StoreInst *SI = dyn_cast<StoreInst>(&Inst))
          Stores.push_back(SI);
      if (Options.StackDepth)
        if (isa<InvokeInst>(Inst) ||
            (isa<CallInst>(Inst) && !isa<IntrinsicInst>(Inst)))
          IsLeafFunc = false;
    }
  }

  if (Options.CollectControlFlow)
    createFunctionControlFlow(F);

  Value *FunctionGateCmp = nullptr;
  InjectCoverage(F, BlocksToInstrument, FunctionGateCmp, IsLeafFunc);
  InjectCoverageForIndirectCalls(F, IndirCalls);
  InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);
  InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);
  InjectTraceForDiv(F, DivTraceTargets);
  InjectTraceForGep(F, GepTraceTargets);
  InjectTraceForLoadsAndStores(F, Loads, Stores);

  if (Options.TracePCEntryExit)
    InjectTraceForExits(F);
}

GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
    size_t NumElements, Function &F, Type *Ty, const char *Section) {
  ArrayType *ArrayTy = ArrayType::get(Ty, NumElements);
  auto Array = new GlobalVariable(
      *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
      Constant::getNullValue(ArrayTy), "__sancov_gen_");

  if (TargetTriple.supportsCOMDAT() &&
      (F.hasComdat() || TargetTriple.isOSBinFormatELF() || !F.isInterposable()))
    if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple))
      Array->setComdat(Comdat);
  Array->setSection(getSectionName(Section));
  Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));

  // sancov_pcs parallels the other metadata section(s). Optimizers (e.g.
  // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other
  // section(s) as a unit, so we conservatively retain all unconditionally in
  // the compiler.
  //
  // With comdat (COFF/ELF), the linker can guarantee the associated sections
  // will be retained or discarded as a unit, so llvm.compiler.used is
  // sufficient. Otherwise, conservatively make all of them retained by the
  // linker.
  if (Array->hasComdat())
    GlobalsToAppendToCompilerUsed.push_back(Array);
  else
    GlobalsToAppendToUsed.push_back(Array);

  return Array;
}

GlobalVariable *
ModuleSanitizerCoverage::CreatePCArray(Function &F,
                                       ArrayRef<BasicBlock *> AllBlocks) {
  size_t N = AllBlocks.size();
  assert(N);
  SmallVector<Constant *, 32> PCs;
  IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
  for (size_t i = 0; i < N; i++) {
    if (&F.getEntryBlock() == AllBlocks[i]) {
      PCs.push_back((Constant *)IRB.CreatePointerCast(&F, PtrTy));
      PCs.push_back(
          (Constant *)IRB.CreateIntToPtr(ConstantInt::get(IntptrTy, 1), PtrTy));
    } else {
      PCs.push_back((Constant *)IRB.CreatePointerCast(
          BlockAddress::get(AllBlocks[i]), PtrTy));
      PCs.push_back(Constant::getNullValue(PtrTy));
    }
  }
  auto *PCArray =
      CreateFunctionLocalArrayInSection(N * 2, F, PtrTy, SanCovPCsSectionName);
  PCArray->setInitializer(
      ConstantArray::get(ArrayType::get(PtrTy, N * 2), PCs));
  PCArray->setConstant(true);

  return PCArray;
}

void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
    Function &F, ArrayRef<BasicBlock *> AllBlocks) {
  if (Options.TracePCGuard)
    FunctionGuardArray = CreateFunctionLocalArrayInSection(
        AllBlocks.size(), F, Int32Ty, SanCovGuardsSectionName);

  if (Options.Inline8bitCounters)
    Function8bitCounterArray = CreateFunctionLocalArrayInSection(
        AllBlocks.size(), F, Int8Ty, SanCovCountersSectionName);
  if (Options.InlineBoolFlag)
    FunctionBoolArray = CreateFunctionLocalArrayInSection(
        AllBlocks.size(), F, Int1Ty, SanCovBoolFlagSectionName);

  if (Options.PCTable)
    FunctionPCsArray = CreatePCArray(F, AllBlocks);
}

Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {
  auto Load = IRB.CreateLoad(Int64Ty, SanCovCallbackGate);
  Load->setNoSanitizeMetadata();
  auto Cmp = IRB.CreateIsNotNull(Load);
  Cmp->setName("sancov gate cmp");
  return Cmp;
}

Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,
                                                       Value *&FunctionGateCmp,
                                                       Instruction *IP) {
  if (!FunctionGateCmp) {
    // Create this in the entry block
    BasicBlock &BB = F.getEntryBlock();
    BasicBlock::iterator IP = BB.getFirstInsertionPt();
    IP = PrepareToSplitEntryBlock(BB, IP);
    IRBuilder<> EntryIRB(&*IP);
    FunctionGateCmp = CreateFunctionLocalGateCmp(EntryIRB);
  }
  // Set the branch weights in order to minimize the price paid when the
  // gate is turned off, allowing the default enablement of this
  // instrumentation with as little of a performance cost as possible
  auto Weights = MDBuilder(*C).createBranchWeights(1, 100000);
  return SplitBlockAndInsertIfThen(FunctionGateCmp, IP, false, Weights);
}

bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
                                             ArrayRef<BasicBlock *> AllBlocks,
                                             Value *&FunctionGateCmp,
                                             bool IsLeafFunc) {
  if (AllBlocks.empty())
    return false;
  CreateFunctionLocalArrays(F, AllBlocks);
  for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
    InjectCoverageAtBlock(F, *AllBlocks[i], i, FunctionGateCmp, IsLeafFunc);

  return true;
}

// On every indirect call we call a run-time function
// __sanitizer_cov_indir_call* with two parameters:
//   - callee address,
//   - global cache array that contains CacheSize pointers (zero-initialized).
//     The cache is used to speed up recording the caller-callee pairs.
// The address of the caller is passed implicitly via caller PC.
// CacheSize is encoded in the name of the run-time function.
void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
    Function &F, ArrayRef<Instruction *> IndirCalls) {
  if (IndirCalls.empty())
    return;
  assert(Options.TracePC || Options.TracePCEntryExit || Options.TracePCGuard ||
         Options.Inline8bitCounters || Options.InlineBoolFlag);
  for (auto *I : IndirCalls) {
    InstrumentationIRBuilder IRB(I);
    CallBase &CB = cast<CallBase>(*I);
    Value *Callee = CB.getCalledOperand();
    if (isa<InlineAsm>(Callee))
      continue;
    IRB.CreateCall(SanCovTracePCIndir, IRB.CreatePointerCast(Callee, IntptrTy));
  }
}

// For every switch statement we insert a call:
// __sanitizer_cov_trace_switch(CondValue,
//      {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })

void ModuleSanitizerCoverage::InjectTraceForSwitch(
    Function &F, ArrayRef<Instruction *> SwitchTraceTargets,
    Value *&FunctionGateCmp) {
  for (auto *I : SwitchTraceTargets) {
    if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
      InstrumentationIRBuilder IRB(I);
      SmallVector<Constant *, 16> Initializers;
      Value *Cond = SI->getCondition();
      if (Cond->getType()->getScalarSizeInBits() >
          Int64Ty->getScalarSizeInBits())
        continue;
      Initializers.push_back(ConstantInt::get(Int64Ty, SI->getNumCases()));
      Initializers.push_back(
          ConstantInt::get(Int64Ty, Cond->getType()->getScalarSizeInBits()));
      if (Cond->getType()->getScalarSizeInBits() <
          Int64Ty->getScalarSizeInBits())
        Cond = IRB.CreateIntCast(Cond, Int64Ty, false);
      for (auto It : SI->cases()) {
        ConstantInt *C = It.getCaseValue();
        if (C->getType()->getScalarSizeInBits() < 64)
          C = ConstantInt::get(C->getContext(), C->getValue().zext(64));
        Initializers.push_back(C);
      }
      llvm::sort(drop_begin(Initializers, 2),
                 [](const Constant *A, const Constant *B) {
                   return cast<ConstantInt>(A)->getLimitedValue() <
                          cast<ConstantInt>(B)->getLimitedValue();
                 });
      ArrayType *ArrayOfInt64Ty = ArrayType::get(Int64Ty, Initializers.size());
      GlobalVariable *GV = new GlobalVariable(
          *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
          ConstantArray::get(ArrayOfInt64Ty, Initializers),
          "__sancov_gen_cov_switch_values");
      if (Options.GatedCallbacks) {
        auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);
        IRBuilder<> GateIRB(GateBranch);
        GateIRB.CreateCall(SanCovTraceSwitchFunction, {Cond, GV});
      } else {
        IRB.CreateCall(SanCovTraceSwitchFunction, {Cond, GV});
      }
    }
  }
}

void ModuleSanitizerCoverage::InjectTraceForDiv(
    Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
  for (auto *BO : DivTraceTargets) {
    InstrumentationIRBuilder IRB(BO);
    Value *A1 = BO->getOperand(1);
    if (isa<ConstantInt>(A1))
      continue;
    if (!A1->getType()->isIntegerTy())
      continue;
    uint64_t TypeSize = DL->getTypeStoreSizeInBits(A1->getType());
    int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;
    if (CallbackIdx < 0)
      continue;
    auto Ty = Type::getIntNTy(*C, TypeSize);
    IRB.CreateCall(SanCovTraceDivFunction[CallbackIdx],
                   {IRB.CreateIntCast(A1, Ty, true)});
  }
}

void ModuleSanitizerCoverage::InjectTraceForGep(
    Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
  for (auto *GEP : GepTraceTargets) {
    InstrumentationIRBuilder IRB(GEP);
    for (Use &Idx : GEP->indices())
      if (!isa<ConstantInt>(Idx) && Idx->getType()->isIntegerTy())
        IRB.CreateCall(SanCovTraceGepFunction,
                       {IRB.CreateIntCast(Idx, IntptrTy, true)});
  }
}

void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
    Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
  auto CallbackIdx = [&](Type *ElementTy) -> int {
    uint64_t TypeSize = DL->getTypeStoreSizeInBits(ElementTy);
    return TypeSize == 8     ? 0
           : TypeSize == 16  ? 1
           : TypeSize == 32  ? 2
           : TypeSize == 64  ? 3
           : TypeSize == 128 ? 4
                             : -1;
  };
  for (auto *LI : Loads) {
    InstrumentationIRBuilder IRB(LI);
    auto Ptr = LI->getPointerOperand();
    int Idx = CallbackIdx(LI->getType());
    if (Idx < 0)
      continue;
    IRB.CreateCall(SanCovLoadFunction[Idx], Ptr);
  }
  for (auto *SI : Stores) {
    InstrumentationIRBuilder IRB(SI);
    auto Ptr = SI->getPointerOperand();
    int Idx = CallbackIdx(SI->getValueOperand()->getType());
    if (Idx < 0)
      continue;
    IRB.CreateCall(SanCovStoreFunction[Idx], Ptr);
  }
}

void ModuleSanitizerCoverage::InjectTraceForExits(Function &F) {
  EscapeEnumerator EE(F, "sancov_exit");
  while (IRBuilder<> *AtExit = EE.Next()) {
    InstrumentationIRBuilder::ensureDebugInfo(*AtExit, F);
    AtExit->CreateCall(SanCovTracePCExit, {})
        ->setTailCallKind(CallInst::TCK_NoTail);
  }
}

void ModuleSanitizerCoverage::InjectTraceForCmp(
    Function &F, ArrayRef<Instruction *> CmpTraceTargets,
    Value *&FunctionGateCmp) {
  for (auto *I : CmpTraceTargets) {
    if (ICmpInst *ICMP = dyn_cast<ICmpInst>(I)) {
      InstrumentationIRBuilder IRB(ICMP);
      Value *A0 = ICMP->getOperand(0);
      Value *A1 = ICMP->getOperand(1);
      if (!A0->getType()->isIntegerTy())
        continue;
      uint64_t TypeSize = DL->getTypeStoreSizeInBits(A0->getType());
      int CallbackIdx = TypeSize == 8    ? 0
                        : TypeSize == 16 ? 1
                        : TypeSize == 32 ? 2
                        : TypeSize == 64 ? 3
                                         : -1;
      if (CallbackIdx < 0)
        continue;
      // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
      auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
      bool FirstIsConst = isa<ConstantInt>(A0);
      bool SecondIsConst = isa<ConstantInt>(A1);
      // If both are const, then we don't need such a comparison.
      if (FirstIsConst && SecondIsConst)
        continue;
      // If only one is const, then make it the first callback argument.
      if (FirstIsConst || SecondIsConst) {
        CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
        if (SecondIsConst)
          std::swap(A0, A1);
      }

      auto Ty = Type::getIntNTy(*C, TypeSize);
      if (Options.GatedCallbacks) {
        auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);
        IRBuilder<> GateIRB(GateBranch);
        GateIRB.CreateCall(CallbackFunc, {GateIRB.CreateIntCast(A0, Ty, true),
                                          GateIRB.CreateIntCast(A1, Ty, true)});
      } else {
        IRB.CreateCall(CallbackFunc, {IRB.CreateIntCast(A0, Ty, true),
                                      IRB.CreateIntCast(A1, Ty, true)});
      }
    }
  }
}

void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
                                                    size_t Idx,
                                                    Value *&FunctionGateCmp,
                                                    bool IsLeafFunc) {
  BasicBlock::iterator IP = BB.getFirstInsertionPt();
  bool IsEntryBB = &BB == &F.getEntryBlock();
  DebugLoc EntryLoc;
  if (IsEntryBB) {
    if (auto SP = F.getSubprogram())
      EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
    // Keep static allocas and llvm.localescape calls in the entry block.  Even
    // if we aren't splitting the block, it's nice for allocas to be before
    // calls.
    IP = PrepareToSplitEntryBlock(BB, IP);
  }

  InstrumentationIRBuilder IRB(&*IP);
  if (EntryLoc)
    IRB.SetCurrentDebugLocation(EntryLoc);
  if (Options.TracePC || (IsEntryBB && Options.TracePCEntryExit)) {
    FunctionCallee Callee = IsEntryBB && Options.TracePCEntryExit
                                ? SanCovTracePCEntry
                                : SanCovTracePC;
    IRB.CreateCall(Callee)
        ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
  }
  if (Options.TracePCGuard) {
    auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
        FunctionGuardArray->getValueType(), FunctionGuardArray, 0, Idx);
    if (Options.GatedCallbacks) {
      Instruction *I = &*IP;
      auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);
      IRBuilder<> GateIRB(GateBranch);
      GateIRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge();
    } else {
      IRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge();
    }
  }
  if (Options.Inline8bitCounters) {
    auto CounterPtr = IRB.CreateGEP(
        Function8bitCounterArray->getValueType(), Function8bitCounterArray,
        {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});
    auto Load = IRB.CreateLoad(Int8Ty, CounterPtr);
    auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1));
    auto Store = IRB.CreateStore(Inc, CounterPtr);
    Load->setNoSanitizeMetadata();
    Store->setNoSanitizeMetadata();
  }
  if (Options.InlineBoolFlag) {
    auto FlagPtr = IRB.CreateGEP(
        FunctionBoolArray->getValueType(), FunctionBoolArray,
        {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});
    auto Load = IRB.CreateLoad(Int1Ty, FlagPtr);
    auto ThenTerm = SplitBlockAndInsertIfThen(
        IRB.CreateIsNull(Load), &*IP, false,
        MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
    InstrumentationIRBuilder ThenIRB(ThenTerm);
    auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr);
    if (EntryLoc)
      Store->setDebugLoc(EntryLoc);
    Load->setNoSanitizeMetadata();
    Store->setNoSanitizeMetadata();
  }
  if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
    Module *M = F.getParent();
    const DataLayout &DL = M->getDataLayout();

    if (Options.StackDepthCallbackMin) {
      // In callback mode, only add call when stack depth reaches minimum.
      int EstimatedStackSize = 0;
      // If dynamic alloca found, always add call.
      bool HasDynamicAlloc = false;
      // Find an insertion point after last "alloca".
      llvm::Instruction *InsertBefore = nullptr;

      // Examine all allocas in the basic block. since we're too early
      // to have results from Intrinsic::frameaddress, we have to manually
      // estimate the stack size.
      for (auto &I : BB) {
        if (auto *AI = dyn_cast<AllocaInst>(&I)) {
          // Move potential insertion point past the "alloca".
          InsertBefore = AI->getNextNode();

          // Make an estimate on the stack usage.
          if (auto AllocaSize = AI->getAllocationSize(DL)) {
            if (AllocaSize->isFixed())
              EstimatedStackSize += AllocaSize->getFixedValue();
            else
              HasDynamicAlloc = true;
          } else {
            HasDynamicAlloc = true;
          }
        }
      }

      if (HasDynamicAlloc ||
          EstimatedStackSize >= Options.StackDepthCallbackMin) {
        if (InsertBefore)
          IRB.SetInsertPoint(InsertBefore);
        auto Call = IRB.CreateCall(SanCovStackDepthCallback);
        if (EntryLoc)
          Call->setDebugLoc(EntryLoc);
        Call->setCannotMerge();
      }
    } else {
      // Check stack depth.  If it's the deepest so far, record it.
      auto FrameAddrPtr = IRB.CreateIntrinsic(
          Intrinsic::frameaddress, IRB.getPtrTy(DL.getAllocaAddrSpace()),
          {Constant::getNullValue(Int32Ty)});
      auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy);
      auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack);
      auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack);
      auto ThenTerm = SplitBlockAndInsertIfThen(
          IsStackLower, &*IP, false,
          MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
      InstrumentationIRBuilder ThenIRB(ThenTerm);
      auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack);
      if (EntryLoc)
        Store->setDebugLoc(EntryLoc);
      LowestStack->setNoSanitizeMetadata();
      Store->setNoSanitizeMetadata();
    }
  }
}

std::string
ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
  if (TargetTriple.isOSBinFormatCOFF()) {
    if (Section == SanCovCountersSectionName)
      return ".SCOV$CM";
    if (Section == SanCovBoolFlagSectionName)
      return ".SCOV$BM";
    if (Section == SanCovPCsSectionName)
      return ".SCOVP$M";
    return ".SCOV$GM"; // For SanCovGuardsSectionName.
  }
  if (TargetTriple.isOSBinFormatMachO())
    return "__DATA,__" + Section;
  return "__" + Section;
}

std::string
ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
  if (TargetTriple.isOSBinFormatMachO())
    return "\1section$start$__DATA$__" + Section;
  return "__start___" + Section;
}

std::string
ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
  if (TargetTriple.isOSBinFormatMachO())
    return "\1section$end$__DATA$__" + Section;
  return "__stop___" + Section;
}

void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
  SmallVector<Constant *, 32> CFs;
  IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());

  for (auto &BB : F) {
    // blockaddress can not be used on function's entry block.
    if (&BB == &F.getEntryBlock())
      CFs.push_back((Constant *)IRB.CreatePointerCast(&F, PtrTy));
    else
      CFs.push_back(
          (Constant *)IRB.CreatePointerCast(BlockAddress::get(&BB), PtrTy));

    for (auto SuccBB : successors(&BB)) {
      assert(SuccBB != &F.getEntryBlock());
      CFs.push_back(
          (Constant *)IRB.CreatePointerCast(BlockAddress::get(SuccBB), PtrTy));
    }

    CFs.push_back((Constant *)Constant::getNullValue(PtrTy));

    for (auto &Inst : BB) {
      if (CallBase *CB = dyn_cast<CallBase>(&Inst)) {
        if (CB->isIndirectCall()) {
          // TODO(navidem): handle indirect calls, for now mark its existence.
          CFs.push_back((Constant *)IRB.CreateIntToPtr(
              ConstantInt::getAllOnesValue(IntptrTy), PtrTy));
        } else {
          auto CalledF = CB->getCalledFunction();
          if (CalledF && !CalledF->isIntrinsic())
            CFs.push_back((Constant *)IRB.CreatePointerCast(CalledF, PtrTy));
        }
      }
    }

    CFs.push_back((Constant *)Constant::getNullValue(PtrTy));
  }

  FunctionCFsArray = CreateFunctionLocalArrayInSection(CFs.size(), F, PtrTy,
                                                       SanCovCFsSectionName);
  FunctionCFsArray->setInitializer(
      ConstantArray::get(ArrayType::get(PtrTy, CFs.size()), CFs));
  FunctionCFsArray->setConstant(true);
}