llvm/lib/Transforms/IPO/MergeFunctions.cpp

//===- MergeFunctions.cpp - Merge identical functions ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass looks for equivalent functions that are mergable and folds them.
//
// Order relation is defined on set of functions. It was made through
// special function comparison procedure that returns
// 0 when functions are equal,
// -1 when Left function is less than right function, and
// 1 for opposite case. We need total-ordering, so we need to maintain
// four properties on the functions set:
// a <= a (reflexivity)
// if a <= b and b <= a then a = b (antisymmetry)
// if a <= b and b <= c then a <= c (transitivity).
// for all a and b: a <= b or b <= a (totality).
//
// Comparison iterates through each instruction in each basic block.
// Functions are kept on binary tree. For each new function F we perform
// lookup in binary tree.
// In practice it works the following way:
// -- We define Function* container class with custom "operator<" (FunctionPtr).
// -- "FunctionPtr" instances are stored in std::set collection, so every
//    std::set::insert operation will give you result in log(N) time.
//
// As an optimization, a hash of the function structure is calculated first, and
// two functions are only compared if they have the same hash. This hash is
// cheap to compute, and has the property that if function F == G according to
// the comparison function, then hash(F) == hash(G). This consistency property
// is critical to ensuring all possible merging opportunities are exploited.
// Collisions in the hash affect the speed of the pass but not the correctness
// or determinism of the resulting transformation.
//
// When a match is found the functions are folded. If both functions are
// overridable, we move the functionality into a new internal function and
// leave two overridable thunks to it.
//
//===----------------------------------------------------------------------===//
//
// Future work:
//
// * virtual functions.
//
// Many functions have their address taken by the virtual function table for
// the object they belong to. However, as long as it's only used for a lookup
// and call, this is irrelevant, and we'd like to fold such functions.
//
// * be smarter about bitcasts.
//
// In order to fold functions, we will sometimes add either bitcast instructions
// or bitcast constant expressions. Unfortunately, this can confound further
// analysis since the two functions differ where one has a bitcast and the
// other doesn't. We should learn to look through bitcasts.
//
// * Compare complex types with pointer types inside.
// * Compare cross-reference cases.
// * Compare complex expressions.
//
// All the three issues above could be described as ability to prove that
// fA == fB == fC == fE == fF == fG in example below:
//
//  void fA() {
//    fB();
//  }
//  void fB() {
//    fA();
//  }
//
//  void fE() {
//    fF();
//  }
//  void fF() {
//    fG();
//  }
//  void fG() {
//    fE();
//  }
//
// Simplest cross-reference case (fA <--> fB) was implemented in previous
// versions of MergeFunctions, though it presented only in two function pairs
// in test-suite (that counts >50k functions)
// Though possibility to detect complex cross-referencing (e.g.: A->B->C->D->A)
// could cover much more cases.
//
//===----------------------------------------------------------------------===//

#include "llvm/Transforms/IPO/MergeFunctions.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/StructuralHash.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/Utils/FunctionComparator.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <optional>
#include <set>
#include <utility>
#include <vector>

using namespace llvm;

#define DEBUG_TYPE "mergefunc"

STATISTIC(NumFunctionsMerged, "Number of functions merged");
STATISTIC(NumThunksWritten, "Number of thunks generated");
STATISTIC(NumAliasesWritten, "Number of aliases generated");
STATISTIC(NumDoubleWeak, "Number of new functions created");

static cl::opt<unsigned> NumFunctionsForVerificationCheck(
    "mergefunc-verify",
    cl::desc("How many functions in a module could be used for "
             "MergeFunctions to pass a basic correctness check. "
             "'0' disables this check. Works only with '-debug' key."),
    cl::init(0), cl::Hidden);

// Under option -mergefunc-preserve-debug-info we:
// - Do not create a new function for a thunk.
// - Retain the debug info for a thunk's parameters (and associated
//   instructions for the debug info) from the entry block.
//   Note: -debug will display the algorithm at work.
// - Create debug-info for the call (to the shared implementation) made by
//   a thunk and its return value.
// - Erase the rest of the function, retaining the (minimally sized) entry
//   block to create a thunk.
// - Preserve a thunk's call site to point to the thunk even when both occur
//   within the same translation unit, to aid debugability. Note that this
//   behaviour differs from the underlying -mergefunc implementation which
//   modifies the thunk's call site to point to the shared implementation
//   when both occur within the same translation unit.
static cl::opt<bool>
    MergeFunctionsPDI("mergefunc-preserve-debug-info", cl::Hidden,
                      cl::init(false),
                      cl::desc("Preserve debug info in thunk when mergefunc "
                               "transformations are made."));

static cl::opt<bool>
    MergeFunctionsAliases("mergefunc-use-aliases", cl::Hidden,
                          cl::init(false),
                          cl::desc("Allow mergefunc to create aliases"));

namespace {

class FunctionNode {
  mutable AssertingVH<Function> F;
  stable_hash Hash;

public:
  // Note the hash is recalculated potentially multiple times, but it is cheap.
  FunctionNode(Function *F) : F(F), Hash(StructuralHash(*F)) {}

  Function *getFunc() const { return F; }
  stable_hash getHash() const { return Hash; }

  /// Replace the reference to the function F by the function G, assuming their
  /// implementations are equal.
  void replaceBy(Function *G) const {
    F = G;
  }
};

/// MergeFunctions finds functions which will generate identical machine code,
/// by considering all pointer types to be equivalent. Once identified,
/// MergeFunctions will fold them by replacing a call to one to a call to a
/// bitcast of the other.
class MergeFunctions {
public:
  explicit MergeFunctions(FunctionAnalysisManager &FAM)
      : FnTree(FunctionNodeCmp(&GlobalNumbers)), FAM(FAM) {}

  template <typename FuncContainer> bool run(FuncContainer &Functions);
  DenseMap<Function *, Function *> runOnFunctions(ArrayRef<Function *> Funcs);

  SmallPtrSet<GlobalValue *, 4> &getUsed();

private:
  // The function comparison operator is provided here so that FunctionNodes do
  // not need to become larger with another pointer.
  class FunctionNodeCmp {
    GlobalNumberState* GlobalNumbers;

  public:
    FunctionNodeCmp(GlobalNumberState* GN) : GlobalNumbers(GN) {}

    bool operator()(const FunctionNode &LHS, const FunctionNode &RHS) const {
      // Order first by hashes, then full function comparison.
      if (LHS.getHash() != RHS.getHash())
        return LHS.getHash() < RHS.getHash();
      FunctionComparator FCmp(LHS.getFunc(), RHS.getFunc(), GlobalNumbers);
      return FCmp.compare() < 0;
    }
  };
  using FnTreeType = std::set<FunctionNode, FunctionNodeCmp>;

  GlobalNumberState GlobalNumbers;

  /// A work queue of functions that may have been modified and should be
  /// analyzed again.
  std::vector<WeakTrackingVH> Deferred;

  /// Set of values marked as used in llvm.used and llvm.compiler.used.
  SmallPtrSet<GlobalValue *, 4> Used;

#ifndef NDEBUG
  /// Checks the rules of order relation introduced among functions set.
  /// Returns true, if check has been passed, and false if failed.
  bool doFunctionalCheck(std::vector<WeakTrackingVH> &Worklist);
#endif

  /// Insert a ComparableFunction into the FnTree, or merge it away if it's
  /// equal to one that's already present.
  bool insert(Function *NewFunction);

  /// Remove a Function from the FnTree and queue it up for a second sweep of
  /// analysis.
  void remove(Function *F);

  /// Find the functions that use this Value and remove them from FnTree and
  /// queue the functions.
  void removeUsers(Value *V);

  /// Replace all direct calls of Old with calls of New. Will bitcast New if
  /// necessary to make types match.
  void replaceDirectCallers(Function *Old, Function *New);

  /// Merge two equivalent functions. Upon completion, G may be deleted, or may
  /// be converted into a thunk. In either case, it should never be visited
  /// again.
  void mergeTwoFunctions(Function *F, Function *G);

  /// Merge \p Src's instruction-level annotations into the corresponding
  /// instructions of \p Dst. \p Dst is the surviving function; \p Src will be
  /// erased or rewritten after this call.
  /// Both functions must be structurally identical.
  void mergeInstrAnnotations(Function *Dst, Function *Src);

  /// Fill PDIUnrelatedWL with instructions from the entry block that are
  /// unrelated to parameter related debug info.
  /// \param PDVRUnrelatedWL The equivalent non-intrinsic debug records.
  void
  filterInstsUnrelatedToPDI(BasicBlock *GEntryBlock,
                            std::vector<Instruction *> &PDIUnrelatedWL,
                            std::vector<DbgVariableRecord *> &PDVRUnrelatedWL);

  /// Erase the rest of the CFG (i.e. barring the entry block).
  void eraseTail(Function *G);

  /// Erase the instructions in PDIUnrelatedWL as they are unrelated to the
  /// parameter debug info, from the entry block.
  /// \param PDVRUnrelatedWL contains the equivalent set of non-instruction
  /// debug-info records.
  void
  eraseInstsUnrelatedToPDI(std::vector<Instruction *> &PDIUnrelatedWL,
                           std::vector<DbgVariableRecord *> &PDVRUnrelatedWL);

  /// Replace G with a simple tail call to bitcast(F). Also (unless
  /// MergeFunctionsPDI holds) replace direct uses of G with bitcast(F),
  /// delete G.
  void writeThunk(Function *F, Function *G);

  // Replace G with an alias to F (deleting function G)
  void writeAlias(Function *F, Function *G);

  // If needed, replace G with an alias to F if possible, or a thunk to F if
  // profitable. Returns false if neither is the case. If \p G is not needed
  // (i.e. it is discardable and not used), \p G is removed directly.
  bool writeThunkOrAliasIfNeeded(Function *F, Function *G);

  /// Replace function F with function G in the function tree.
  void replaceFunctionInTree(const FunctionNode &FN, Function *G);

  /// The set of all distinct functions. Use the insert() and remove() methods
  /// to modify it. The map allows efficient lookup and deferring of Functions.
  FnTreeType FnTree;

  // Map functions to the iterators of the FunctionNode which contains them
  // in the FnTree. This must be updated carefully whenever the FnTree is
  // modified, i.e. in insert(), remove(), and replaceFunctionInTree(), to avoid
  // dangling iterators into FnTree. The invariant that preserves this is that
  // there is exactly one mapping F -> FN for each FunctionNode FN in FnTree.
  DenseMap<AssertingVH<Function>, FnTreeType::iterator> FNodesInTree;

  /// Deleted-New functions mapping
  DenseMap<Function *, Function *> DelToNewMap;

  FunctionAnalysisManager &FAM;
};
} // end anonymous namespace

PreservedAnalyses MergeFunctionsPass::run(Module &M,
                                          ModuleAnalysisManager &AM) {
  if (!MergeFunctionsPass::runOnModule(M, AM))
    return PreservedAnalyses::all();
  return PreservedAnalyses::none();
}

SmallPtrSet<GlobalValue *, 4> &MergeFunctions::getUsed() { return Used; }

bool MergeFunctionsPass::runOnModule(Module &M, ModuleAnalysisManager &AM) {
  auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
  MergeFunctions MF(FAM);
  SmallVector<GlobalValue *, 4> UsedV;
  collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/false);
  collectUsedGlobalVariables(M, UsedV, /*CompilerUsed=*/true);
  MF.getUsed().insert_range(UsedV);
  return MF.run(M);
}

DenseMap<Function *, Function *>
MergeFunctionsPass::runOnFunctions(ArrayRef<Function *> Funcs,
                                   ModuleAnalysisManager &AM) {
  if (Funcs.empty())
    return DenseMap<Function *, Function *>();

  Module &M = *Funcs.front()->getParent();
  auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
  MergeFunctions MF(FAM);
  return MF.runOnFunctions(Funcs);
}

#ifndef NDEBUG
bool MergeFunctions::doFunctionalCheck(std::vector<WeakTrackingVH> &Worklist) {
  if (const unsigned Max = NumFunctionsForVerificationCheck) {
    unsigned TripleNumber = 0;
    bool Valid = true;

    dbgs() << "MERGEFUNC-VERIFY: Started for first " << Max << " functions.\n";

    unsigned i = 0;
    for (std::vector<WeakTrackingVH>::iterator I = Worklist.begin(),
                                               E = Worklist.end();
         I != E && i < Max; ++I, ++i) {
      unsigned j = i;
      for (std::vector<WeakTrackingVH>::iterator J = I; J != E && j < Max;
           ++J, ++j) {
        Function *F1 = cast<Function>(*I);
        Function *F2 = cast<Function>(*J);
        int Res1 = FunctionComparator(F1, F2, &GlobalNumbers).compare();
        int Res2 = FunctionComparator(F2, F1, &GlobalNumbers).compare();

        // If F1 <= F2, then F2 >= F1, otherwise report failure.
        if (Res1 != -Res2) {
          dbgs() << "MERGEFUNC-VERIFY: Non-symmetric; triple: " << TripleNumber
                 << "\n";
          dbgs() << *F1 << '\n' << *F2 << '\n';
          Valid = false;
        }

        if (Res1 == 0)
          continue;

        unsigned k = j;
        for (std::vector<WeakTrackingVH>::iterator K = J; K != E && k < Max;
             ++k, ++K, ++TripleNumber) {
          if (K == J)
            continue;

          Function *F3 = cast<Function>(*K);
          int Res3 = FunctionComparator(F1, F3, &GlobalNumbers).compare();
          int Res4 = FunctionComparator(F2, F3, &GlobalNumbers).compare();

          bool Transitive = true;

          if (Res1 != 0 && Res1 == Res4) {
            // F1 > F2, F2 > F3 => F1 > F3
            Transitive = Res3 == Res1;
          } else if (Res3 != 0 && Res3 == -Res4) {
            // F1 > F3, F3 > F2 => F1 > F2
            Transitive = Res3 == Res1;
          } else if (Res4 != 0 && -Res3 == Res4) {
            // F2 > F3, F3 > F1 => F2 > F1
            Transitive = Res4 == -Res1;
          }

          if (!Transitive) {
            dbgs() << "MERGEFUNC-VERIFY: Non-transitive; triple: "
                   << TripleNumber << "\n";
            dbgs() << "Res1, Res3, Res4: " << Res1 << ", " << Res3 << ", "
                   << Res4 << "\n";
            dbgs() << *F1 << '\n' << *F2 << '\n' << *F3 << '\n';
            Valid = false;
          }
        }
      }
    }

    dbgs() << "MERGEFUNC-VERIFY: " << (Valid ? "Passed." : "Failed.") << "\n";
    return Valid;
  }
  return true;
}
#endif

/// Check whether \p F has an intrinsic which references
/// distinct metadata as an operand. The most common
/// instance of this would be CFI checks for function-local types.
static bool hasDistinctMetadataIntrinsic(const Function &F) {
  for (const BasicBlock &BB : F) {
    for (const Instruction &I : BB) {
      if (!isa<IntrinsicInst>(&I))
        continue;

      for (MetadataAsValue *MDL :
           make_isa_range<MetadataAsValue>(I.operands())) {
        if (MDNode *N = dyn_cast<MDNode>(MDL->getMetadata()))
          if (N->isDistinct())
            return true;
      }
    }
  }
  return false;
}

/// Check whether \p F is eligible for function merging.
static bool isEligibleForMerging(Function &F) {
  // Oracle functions of llvm.speculative.load may only be referenced by
  // llvm.speculative.load, so they can neither become thunks nor be called.
  return !F.isDeclaration() && !F.hasAvailableExternallyLinkage() &&
         !F.hasFnAttribute(Attribute::NoIPA) &&
         !F.hasFnAttribute("oracle-function") &&
         !hasDistinctMetadataIntrinsic(F);
}

inline Function *asPtr(Function *Fn) { return Fn; }
inline Function *asPtr(Function &Fn) { return &Fn; }

template <typename FuncContainer> bool MergeFunctions::run(FuncContainer &M) {
  bool Changed = false;

  // All functions in the module, ordered by hash. Functions with a unique
  // hash value are easily eliminated.
  std::vector<std::pair<stable_hash, Function *>> HashedFuncs;
  for (auto &Func : M) {
    Function *FuncPtr = asPtr(Func);
    if (isEligibleForMerging(*FuncPtr)) {
      HashedFuncs.push_back({StructuralHash(*FuncPtr), FuncPtr});
    }
  }

  llvm::stable_sort(HashedFuncs, less_first());

  auto S = HashedFuncs.begin();
  for (auto I = HashedFuncs.begin(), IE = HashedFuncs.end(); I != IE; ++I) {
    // If the hash value matches the previous value or the next one, we must
    // consider merging it. Otherwise it is dropped and never considered again.
    if ((I != S && std::prev(I)->first == I->first) ||
        (std::next(I) != IE && std::next(I)->first == I->first)) {
      Deferred.push_back(WeakTrackingVH(I->second));
    }
  }

  do {
    std::vector<WeakTrackingVH> Worklist;
    Deferred.swap(Worklist);

    LLVM_DEBUG(doFunctionalCheck(Worklist));

    LLVM_DEBUG(dbgs() << "size of module: " << M.size() << '\n');
    LLVM_DEBUG(dbgs() << "size of worklist: " << Worklist.size() << '\n');

    // Insert functions and merge them.
    for (WeakTrackingVH &I : Worklist) {
      if (!I)
        continue;
      Function *F = cast<Function>(I);
      if (!F->isDeclaration() && !F->hasAvailableExternallyLinkage() &&
          !F->hasFnAttribute(Attribute::NoIPA)) {
        Changed |= insert(F);
      }
    }
    LLVM_DEBUG(dbgs() << "size of FnTree: " << FnTree.size() << '\n');
  } while (!Deferred.empty());

  FnTree.clear();
  FNodesInTree.clear();
  GlobalNumbers.clear();
  Used.clear();

  return Changed;
}

DenseMap<Function *, Function *>
MergeFunctions::runOnFunctions(ArrayRef<Function *> Funcs) {
  [[maybe_unused]] bool MergeResult = this->run(Funcs);
  assert(MergeResult == !DelToNewMap.empty());
  return this->DelToNewMap;
}

// Direct calls to Old are about to call New. The verifier requires a location
// on a call from a function with debug info to a function with debug info, but
// a call to Old need not have one if Old has no debug info. Give such calls a
// line 0 location in the caller.
static void addMissingCallLocations(Function *Old, Function *New) {
  if (!New->getSubprogram())
    return;
  for (Use &U : Old->uses()) {
    auto *CB = dyn_cast<CallBase>(U.getUser());
    if (!CB || !CB->isCallee(&U) || CB->getDebugLoc())
      continue;
    if (DISubprogram *SP = CB->getFunction()->getSubprogram())
      CB->setDebugLoc(DILocation::get(CB->getContext(), 0, 0, SP));
  }
}

// Replace direct callers of Old with New.
void MergeFunctions::replaceDirectCallers(Function *Old, Function *New) {
  addMissingCallLocations(Old, New);
  for (Use &U : make_early_inc_range(Old->uses())) {
    CallBase *CB = dyn_cast<CallBase>(U.getUser());
    if (CB && CB->isCallee(&U)) {
      // Do not copy attributes from the called function to the call-site.
      // Function comparison ensures that the attributes are the same up to
      // type congruences in byval(), in which case we need to keep the byval
      // type of the call-site, not the callee function.
      remove(CB->getFunction());
      U.set(New);
    }
  }
}

// Erase the instructions in PDIUnrelatedWL as they are unrelated to the
// parameter debug info, from the entry block.
void MergeFunctions::eraseInstsUnrelatedToPDI(
    std::vector<Instruction *> &PDIUnrelatedWL,
    std::vector<DbgVariableRecord *> &PDVRUnrelatedWL) {
  LLVM_DEBUG(
      dbgs() << " Erasing instructions (in reverse order of appearance in "
                "entry block) unrelated to parameter debug info from entry "
                "block: {\n");
  while (!PDIUnrelatedWL.empty()) {
    Instruction *I = PDIUnrelatedWL.back();
    LLVM_DEBUG(dbgs() << "  Deleting Instruction: ");
    LLVM_DEBUG(I->print(dbgs()));
    LLVM_DEBUG(dbgs() << "\n");
    I->eraseFromParent();
    PDIUnrelatedWL.pop_back();
  }

  while (!PDVRUnrelatedWL.empty()) {
    DbgVariableRecord *DVR = PDVRUnrelatedWL.back();
    LLVM_DEBUG(dbgs() << "  Deleting DbgVariableRecord ");
    LLVM_DEBUG(DVR->print(dbgs()));
    LLVM_DEBUG(dbgs() << "\n");
    DVR->eraseFromParent();
    PDVRUnrelatedWL.pop_back();
  }

  LLVM_DEBUG(dbgs() << " } // Done erasing instructions unrelated to parameter "
                       "debug info from entry block. \n");
}

// Reduce G to its entry block.
void MergeFunctions::eraseTail(Function *G) {
  std::vector<BasicBlock *> WorklistBB;
  for (BasicBlock &BB : drop_begin(*G)) {
    BB.dropAllReferences();
    WorklistBB.push_back(&BB);
  }
  while (!WorklistBB.empty()) {
    BasicBlock *BB = WorklistBB.back();
    BB->eraseFromParent();
    WorklistBB.pop_back();
  }
}

// We are interested in the following instructions from the entry block as being
// related to parameter debug info:
// - @llvm.dbg.declare
// - stores from the incoming parameters to locations on the stack-frame
// - allocas that create these locations on the stack-frame
// - @llvm.dbg.value
// - the entry block's terminator
// The rest are unrelated to debug info for the parameters; fill up
// PDIUnrelatedWL with such instructions.
void MergeFunctions::filterInstsUnrelatedToPDI(
    BasicBlock *GEntryBlock, std::vector<Instruction *> &PDIUnrelatedWL,
    std::vector<DbgVariableRecord *> &PDVRUnrelatedWL) {
  std::set<Instruction *> PDIRelated;
  std::set<DbgVariableRecord *> PDVRRelated;

  // Work out whether a dbg.value intrinsic or an equivalent DbgVariableRecord
  // is a parameter to be preserved.
  auto ExamineDbgValue = [&PDVRRelated](DbgVariableRecord *DbgVal) {
    LLVM_DEBUG(dbgs() << " Deciding: ");
    LLVM_DEBUG(DbgVal->print(dbgs()));
    LLVM_DEBUG(dbgs() << "\n");
    DILocalVariable *DILocVar = DbgVal->getVariable();
    if (DILocVar->isParameter()) {
      LLVM_DEBUG(dbgs() << "  Include (parameter): ");
      LLVM_DEBUG(DbgVal->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
      PDVRRelated.insert(DbgVal);
    } else {
      LLVM_DEBUG(dbgs() << "  Delete (!parameter): ");
      LLVM_DEBUG(DbgVal->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
    }
  };

  auto ExamineDbgDeclare = [&PDIRelated,
                            &PDVRRelated](DbgVariableRecord *DbgDecl) {
    LLVM_DEBUG(dbgs() << " Deciding: ");
    LLVM_DEBUG(DbgDecl->print(dbgs()));
    LLVM_DEBUG(dbgs() << "\n");
    DILocalVariable *DILocVar = DbgDecl->getVariable();
    if (DILocVar->isParameter()) {
      LLVM_DEBUG(dbgs() << "  Parameter: ");
      LLVM_DEBUG(DILocVar->print(dbgs()));
      AllocaInst *AI = dyn_cast_or_null<AllocaInst>(DbgDecl->getAddress());
      if (AI) {
        LLVM_DEBUG(dbgs() << "  Processing alloca users: ");
        LLVM_DEBUG(dbgs() << "\n");
        for (User *U : AI->users()) {
          if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
            if (Value *Arg = SI->getValueOperand()) {
              if (isa<Argument>(Arg)) {
                LLVM_DEBUG(dbgs() << "  Include: ");
                LLVM_DEBUG(AI->print(dbgs()));
                LLVM_DEBUG(dbgs() << "\n");
                PDIRelated.insert(AI);
                LLVM_DEBUG(dbgs() << "   Include (parameter): ");
                LLVM_DEBUG(SI->print(dbgs()));
                LLVM_DEBUG(dbgs() << "\n");
                PDIRelated.insert(SI);
                LLVM_DEBUG(dbgs() << "  Include: ");
                LLVM_DEBUG(DbgDecl->print(dbgs()));
                LLVM_DEBUG(dbgs() << "\n");
                PDVRRelated.insert(DbgDecl);
              } else {
                LLVM_DEBUG(dbgs() << "   Delete (!parameter): ");
                LLVM_DEBUG(SI->print(dbgs()));
                LLVM_DEBUG(dbgs() << "\n");
              }
            }
          } else {
            LLVM_DEBUG(dbgs() << "   Defer: ");
            LLVM_DEBUG(U->print(dbgs()));
            LLVM_DEBUG(dbgs() << "\n");
          }
        }
      } else {
        LLVM_DEBUG(dbgs() << "  Delete (alloca NULL): ");
        LLVM_DEBUG(DbgDecl->print(dbgs()));
        LLVM_DEBUG(dbgs() << "\n");
      }
    } else {
      LLVM_DEBUG(dbgs() << "  Delete (!parameter): ");
      LLVM_DEBUG(DbgDecl->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
    }
  };

  for (BasicBlock::iterator BI = GEntryBlock->begin(), BIE = GEntryBlock->end();
       BI != BIE; ++BI) {
    // Examine DbgVariableRecords as they happen "before" the instruction. Are
    // they connected to parameters?
    for (DbgVariableRecord &DVR : filterDbgVars(BI->getDbgRecordRange())) {
      if (DVR.isDbgValue() || DVR.isDbgAssign()) {
        ExamineDbgValue(&DVR);
      } else {
        assert(DVR.isDbgDeclare());
        ExamineDbgDeclare(&DVR);
      }
    }

    if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
      LLVM_DEBUG(dbgs() << " Will Include Terminator: ");
      LLVM_DEBUG(BI->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
      PDIRelated.insert(&*BI);
    } else {
      LLVM_DEBUG(dbgs() << " Defer: ");
      LLVM_DEBUG(BI->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
    }
  }
  LLVM_DEBUG(
      dbgs()
      << " Report parameter debug info related/related instructions: {\n");

  auto IsPDIRelated = [](auto *Rec, auto &Container, auto &UnrelatedCont) {
    if (Container.find(Rec) == Container.end()) {
      LLVM_DEBUG(dbgs() << "  !PDIRelated: ");
      LLVM_DEBUG(Rec->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
      UnrelatedCont.push_back(Rec);
    } else {
      LLVM_DEBUG(dbgs() << "   PDIRelated: ");
      LLVM_DEBUG(Rec->print(dbgs()));
      LLVM_DEBUG(dbgs() << "\n");
    }
  };

  // Collect the set of unrelated instructions and debug records.
  for (Instruction &I : *GEntryBlock) {
    for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
      IsPDIRelated(&DVR, PDVRRelated, PDVRUnrelatedWL);
    IsPDIRelated(&I, PDIRelated, PDIUnrelatedWL);
  }
  LLVM_DEBUG(dbgs() << " }\n");
}

/// Whether this function may be replaced by a forwarding thunk.
static bool canCreateThunkFor(Function *F) {
  if (F->isVarArg())
    return false;

  if (F->hasKernelCallingConv())
    return false;

  // Don't merge tiny functions using a thunk, since it can just end up
  // making the function larger.
  if (F->size() == 1) {
    if (F->front().size() < 2) {
      LLVM_DEBUG(dbgs() << "canCreateThunkFor: " << F->getName()
                        << " is too small to bother creating a thunk for\n");
      return false;
    }
  }
  return true;
}

/// Copy all metadata of a specific kind from one function to another.
static void copyMetadataIfPresent(Function *From, Function *To,
                                  StringRef Kind) {
  SmallVector<MDNode *, 4> MDs;
  From->getMetadata(Kind, MDs);
  for (MDNode *MD : MDs)
    To->addMetadata(Kind, *MD);
}

// Replace G with a simple tail call to bitcast(F). Also (unless
// MergeFunctionsPDI holds) replace direct uses of G with bitcast(F),
// delete G. Under MergeFunctionsPDI, we use G itself for creating
// the thunk as we preserve the debug info (and associated instructions)
// from G's entry block pertaining to G's incoming arguments which are
// passed on as corresponding arguments in the call that G makes to F.
// For better debugability, under MergeFunctionsPDI, we do not modify G's
// call sites to point to F even when within the same translation unit.
void MergeFunctions::writeThunk(Function *F, Function *G) {
  std::optional<uint64_t> GEntryCount = G->getEntryCount();
  BasicBlock *GEntryBlock = nullptr;
  std::vector<Instruction *> PDIUnrelatedWL;
  std::vector<DbgVariableRecord *> PDVRUnrelatedWL;
  BasicBlock *BB = nullptr;
  Function *NewG = nullptr;
  if (MergeFunctionsPDI) {
    LLVM_DEBUG(dbgs() << "writeThunk: (MergeFunctionsPDI) Do not create a new "
                         "function as thunk; retain original: "
                      << G->getName() << "()\n");
    GEntryBlock = &G->getEntryBlock();
    LLVM_DEBUG(
        dbgs() << "writeThunk: (MergeFunctionsPDI) filter parameter related "
                  "debug info for "
               << G->getName() << "() {\n");
    filterInstsUnrelatedToPDI(GEntryBlock, PDIUnrelatedWL, PDVRUnrelatedWL);
    GEntryBlock->getTerminator()->eraseFromParent();
    BB = GEntryBlock;
  } else {
    NewG = Function::Create(G->getFunctionType(), G->getLinkage(),
                            G->getAddressSpace(), "", G->getParent());
    NewG->setComdat(G->getComdat());
    BB = BasicBlock::Create(F->getContext(), "", NewG);
  }

  IRBuilder<> Builder(BB);
  Function *H = MergeFunctionsPDI ? G : NewG;
  SmallVector<Value *, 16> Args;
  unsigned i = 0;
  FunctionType *FFTy = F->getFunctionType();
  for (Argument &AI : H->args()) {
    Args.push_back(Builder.CreateAggregateCast(&AI, FFTy->getParamType(i)));
    ++i;
  }

  CallInst *CI = Builder.CreateCall(F, Args);
  ReturnInst *RI = nullptr;
  bool isSwiftTailCall = F->getCallingConv() == CallingConv::SwiftTail &&
                         G->getCallingConv() == CallingConv::SwiftTail;
  CI->setTailCallKind(isSwiftTailCall ? CallInst::TCK_MustTail
                                      : CallInst::TCK_Tail);
  CI->setCallingConv(F->getCallingConv());
  CI->setAttributes(F->getAttributes());
  if (H->getReturnType()->isVoidTy()) {
    RI = Builder.CreateRetVoid();
  } else {
    RI = Builder.CreateRet(Builder.CreateAggregateCast(CI, H->getReturnType()));
  }

  if (MergeFunctionsPDI) {
    DISubprogram *DIS = G->getSubprogram();
    if (DIS) {
      DebugLoc CIDbgLoc =
          DILocation::get(DIS->getContext(), DIS->getScopeLine(), 0, DIS);
      DebugLoc RIDbgLoc =
          DILocation::get(DIS->getContext(), DIS->getScopeLine(), 0, DIS);
      CI->setDebugLoc(CIDbgLoc);
      RI->setDebugLoc(RIDbgLoc);
    } else {
      LLVM_DEBUG(
          dbgs() << "writeThunk: (MergeFunctionsPDI) No DISubprogram for "
                 << G->getName() << "()\n");
    }
    eraseTail(G);
    eraseInstsUnrelatedToPDI(PDIUnrelatedWL, PDVRUnrelatedWL);
    LLVM_DEBUG(
        dbgs() << "} // End of parameter related debug info filtering for: "
               << G->getName() << "()\n");
  } else {
    NewG->copyAttributesFrom(G);
    if (GEntryCount)
      NewG->setEntryCount(*GEntryCount);
    NewG->takeName(G);
    // Ensure CFI type metadata is propagated to the new function.
    copyMetadataIfPresent(G, NewG, "type");
    copyMetadataIfPresent(G, NewG, "kcfi_type");
    copyMetadataIfPresent(G, NewG, "callgraph");
    removeUsers(G);
    G->replaceAllUsesWith(NewG);
    G->eraseFromParent();
  }

  LLVM_DEBUG(dbgs() << "writeThunk: " << H->getName() << '\n');
  ++NumThunksWritten;
}

// Whether this function may be replaced by an alias
static bool canCreateAliasFor(Function *F) {
  if (!MergeFunctionsAliases || !F->hasGlobalUnnamedAddr())
    return false;

  // We should only see linkages supported by aliases here
  assert(F->hasLocalLinkage() || F->hasExternalLinkage()
      || F->hasWeakLinkage() || F->hasLinkOnceLinkage());
  return true;
}

static bool hasNonLocalAlias(const Function *F) {
  for (const GlobalAlias &GA : F->getParent()->aliases())
    if (!GA.hasLocalLinkage() && GA.getAliaseeObject() == F)
      return true;
  return false;
}

/// A COFF weak external must name its target, and a local symbol has no name
/// the linker can agree on across objects (LNK1227).
static bool canBeAliasee(const Function *F) {
  if (!F->getParent()->getTargetTriple().isOSBinFormatCOFF())
    return true;
  return F->hasName() && !F->hasLocalLinkage();
}

// Replace G with an alias to F (deleting function G)
void MergeFunctions::writeAlias(Function *F, Function *G) {
  PointerType *PtrType = G->getType();
  auto *GA =
      GlobalAlias::create(G->getFunctionType(), PtrType->getAddressSpace(),
                          G->getLinkage(), "", F, G->getParent());

  const MaybeAlign FAlign = F->getAlign();
  const MaybeAlign GAlign = G->getAlign();
  if (FAlign || GAlign)
    F->setAlignment(std::max(FAlign.valueOrOne(), GAlign.valueOrOne()));
  else
    F->setAlignment(std::nullopt);
  GA->takeName(G);
  GA->setVisibility(G->getVisibility());
  GA->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);

  removeUsers(G);
  G->replaceAllUsesWith(GA);
  G->eraseFromParent();

  LLVM_DEBUG(dbgs() << "writeAlias: " << GA->getName() << '\n');
  ++NumAliasesWritten;
}

static DenseSet<GlobalValue::GUID> unionImportGUIDs(const Function &F,
                                                    const Function &G) {
  DenseSet<GlobalValue::GUID> AllImports = F.getImportGUIDs();
  DenseSet<GlobalValue::GUID> GImports = G.getImportGUIDs();
  AllImports.insert(GImports.begin(), GImports.end());
  return AllImports;
}

static void mergeEntryCountsAndImportsInto(Function &F, Function &G) {
  std::optional<uint64_t> FEntryCount = F.getEntryCount();
  std::optional<uint64_t> GEntryCount = G.getEntryCount();
  DenseSet<GlobalValue::GUID> AllImports = unionImportGUIDs(F, G);
  if (!FEntryCount && !GEntryCount && AllImports.empty())
    return;

  // -1 is a safe placeholder here, getEntryCount() already treats it as
  // "unknown" (same sentinel SamplePGO uses for no-sample functions), so
  // it won't look hot to anyone reading the count back.
  uint64_t Sum = static_cast<uint64_t>(-1);
  if (FEntryCount || GEntryCount)
    Sum = SaturatingAdd(FEntryCount ? *FEntryCount : uint64_t{0},
                        GEntryCount ? *GEntryCount : uint64_t{0});
  F.setEntryCount(Sum, AllImports.empty() ? nullptr : &AllImports);
}

bool MergeFunctions::writeThunkOrAliasIfNeeded(Function *F, Function *G) {
  bool ShouldErase =
      G->isDiscardableIfUnused() && G->use_empty() && !MergeFunctionsPDI;
  bool ShouldAlias = canCreateAliasFor(G) && canBeAliasee(F);
  bool ShouldThunk = canCreateThunkFor(F);

  if (!ShouldErase && !ShouldAlias && !ShouldThunk)
    return false;

  if (ShouldErase) {
    G->eraseFromParent();
    return true;
  }

  if (ShouldAlias) {
    writeAlias(F, G);
    return true;
  }
  if (ShouldThunk) {
    writeThunk(F, G);
    return true;
  }

  llvm_unreachable("Erase, alias or thunk must apply");
}

/// Returns true if \p F is either weak_odr or linkonce_odr.
static bool isODR(const Function *F) {
  return F->hasWeakODRLinkage() || F->hasLinkOnceODRLinkage();
}

static uint64_t getBlockCountForMerging(const BlockFrequencyInfo &BFI,
                                        const BasicBlock *BB) {
  if (auto Count = BFI.getBlockProfileCount(BB))
    return *Count;
  return 1;
}

// The branch weights are relative within a function. Before merging we
// normalize these to absolute counts.
// (weight * BlockCount / TotalWeight)
static uint64_t scaleToBlockCount(uint64_t Weight, uint64_t TotalWeight,
                                  uint64_t BlockCount) {
  if (Weight == 0 || TotalWeight == 0 || BlockCount == 0)
    return 0;
  APInt Num(128, BlockCount);
  Num *= APInt(128, Weight);
  APInt Den(128, TotalWeight);
  Num = (Num + Den.lshr(1)).udiv(Den);
  assert(Num.getActiveBits() <= 64 &&
         "scaleToBlockCount: result exceeds uint64_t; Weight > TotalWeight?");
  return Num.getLimitedValue();
}

// Combine the scaled branch_weights of corresponding instructions of F and G.
static void mergeBranchWeightsOnInstructions(Instruction *DstI,
                                             const Instruction *SrcI,
                                             const BlockFrequencyInfo &DstBFI,
                                             const BlockFrequencyInfo &SrcBFI) {
  SmallVector<uint32_t, 8> DstWeights, SrcWeights;
  bool HasDst = extractBranchWeights(*DstI, DstWeights);
  bool HasSrc = extractBranchWeights(*SrcI, SrcWeights);
  if (!HasDst && !HasSrc)
    return;

  uint64_t DstBlockCount = getBlockCountForMerging(DstBFI, DstI->getParent());
  uint64_t SrcBlockCount = getBlockCountForMerging(SrcBFI, SrcI->getParent());

  uint64_t DstTotal = 0, SrcTotal = 0;
  if (HasDst)
    extractProfTotalWeight(*DstI, DstTotal);
  if (HasSrc)
    extractProfTotalWeight(*SrcI, SrcTotal);

  assert((!HasDst || !HasSrc || DstWeights.size() == SrcWeights.size()) &&
         "equivalent branch/select instructions must have matching weight "
         "arity");
  size_t NumWeights = HasDst ? DstWeights.size() : SrcWeights.size();
  SmallVector<uint64_t, 8> MergedWeights;
  MergedWeights.reserve(NumWeights);
  for (size_t I = 0; I < NumWeights; ++I) {
    uint64_t DstW = HasDst ? DstWeights[I] : 0;
    uint64_t SrcW = HasSrc ? SrcWeights[I] : 0;
    uint64_t DstAbs = scaleToBlockCount(DstW, DstTotal, DstBlockCount);
    uint64_t SrcAbs = scaleToBlockCount(SrcW, SrcTotal, SrcBlockCount);
    MergedWeights.push_back(SaturatingAdd(DstAbs, SrcAbs));
  }

  bool IsExpected =
      hasBranchWeightOrigin(*DstI) && hasBranchWeightOrigin(*SrcI);
  setFittedBranchWeights(*DstI, MergedWeights, IsExpected);
}

// Accumulate value profile counts of Instruction I into Merged. Value profile
// counts are absolute, not relative branch-style weights.
static void addValueProfile(const Instruction &I, InstrProfValueKind Kind,
                            DenseMap<uint64_t, uint64_t> &Merged) {
  uint64_t Total = 0;
  SmallVector<InstrProfValueData, 4> VDs =
      getValueProfDataFromInst(I, Kind, /*MaxNumValueData=*/UINT32_MAX, Total);
  if (VDs.empty())
    return;
  for (const InstrProfValueData &VD : VDs)
    Merged[VD.Value] = SaturatingAdd(Merged[VD.Value], VD.Count);
}

// Merge (union) value profiles of Dst and Src.
static void mergeValueProfileOnInstructions(Instruction *DstI,
                                            const Instruction *SrcI) {
  MDNode *DstProf = DstI->getMetadata(LLVMContext::MD_prof);
  MDNode *SrcProf = SrcI->getMetadata(LLVMContext::MD_prof);
  bool HasDst = DstProf && isValueProfileMD(DstProf);
  bool HasSrc = SrcProf && isValueProfileMD(SrcProf);
  if (!HasDst && !HasSrc)
    return;

  auto *DstKind =
      HasDst ? mdconst::dyn_extract<ConstantInt>(DstProf->getOperand(1))
             : nullptr;
  auto *SrcKind =
      HasSrc ? mdconst::dyn_extract<ConstantInt>(SrcProf->getOperand(1))
             : nullptr;
  if (HasDst && HasSrc && DstKind && SrcKind &&
      DstKind->getZExtValue() != SrcKind->getZExtValue()) {
    DstI->setMetadata(LLVMContext::MD_prof, nullptr);
    return;
  }

  const ConstantInt *KindCI = DstKind ? DstKind : SrcKind;
  if (!KindCI) {
    DstI->setMetadata(LLVMContext::MD_prof, nullptr);
    return;
  }

  InstrProfValueKind Kind =
      static_cast<InstrProfValueKind>(KindCI->getZExtValue());

  DenseMap<uint64_t, uint64_t> Merged;
  if (HasDst)
    addValueProfile(*DstI, Kind, Merged);
  if (HasSrc)
    addValueProfile(*SrcI, Kind, Merged);

  if (Merged.empty())
    return;

  SmallVector<InstrProfValueData, 8> VDs;
  VDs.reserve(Merged.size());
  uint64_t Sum = 0;
  for (auto &[Value, Count] : Merged) {
    VDs.push_back({Value, Count});
    Sum = SaturatingAdd(Sum, Count);
  }
  llvm::sort(VDs, [](const InstrProfValueData &A, const InstrProfValueData &B) {
    return A.Count > B.Count;
  });
  annotateValueSite(*DstI->getFunction()->getParent(), *DstI, VDs, Sum, Kind,
                    VDs.size());
}

void MergeFunctions::mergeInstrAnnotations(Function *Dst, Function *Src) {
  const BlockFrequencyInfo &DstBFI =
      FAM.getResult<BlockFrequencyAnalysis>(*Dst);
  const BlockFrequencyInfo &SrcBFI =
      FAM.getResult<BlockFrequencyAnalysis>(*Src);

  // FunctionComparator guarantees identical CFG topology and instruction
  // ordering. Walk the CFGs in RPO rather than function block-list order, as
  // equivalent functions need not store their basic blocks in the same order.
  ReversePostOrderTraversal<Function *> DstRPOT(Dst);
  ReversePostOrderTraversal<Function *> SrcRPOT(Src);
  for (auto [DstBB, SrcBB] : llvm::zip_equal(DstRPOT, SrcRPOT)) {
    for (auto [DstI, SrcI] : llvm::zip_equal(*DstBB, *SrcBB)) {
      // Merge poison-generating flags.
      DstI.andIRFlags(&SrcI);

      MDNode *DstProf = DstI.getMetadata(LLVMContext::MD_prof);
      MDNode *SrcProf = SrcI.getMetadata(LLVMContext::MD_prof);
      if ((DstProf && isValueProfileMD(DstProf)) ||
          (SrcProf && isValueProfileMD(SrcProf)))
        mergeValueProfileOnInstructions(&DstI, &SrcI);

      // Handle branch weights on SelectInsts here. Terminators are handled
      // separately below, outside the instruction loop.
      if (isa<SelectInst>(DstI))
        mergeBranchWeightsOnInstructions(&DstI, &SrcI, DstBFI, SrcBFI);
    }
    Instruction *DstTerm = DstBB->getTerminator();
    const Instruction *SrcTerm = SrcBB->getTerminator();
    mergeBranchWeightsOnInstructions(DstTerm, SrcTerm, DstBFI, SrcBFI);
  }

  PreservedAnalyses PA = PreservedAnalyses::all();
  PA.abandon<BranchProbabilityAnalysis>();
  PA.abandon<BlockFrequencyAnalysis>();
  FAM.invalidate(*Dst, PA);
}

// Merge two equivalent functions. Upon completion, Function G is deleted.
void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {

  std::optional<uint64_t> FEntryCount = F->getEntryCount();

  // Create a new thunk that both F and G can call, if F cannot call G directly.
  // That is the case if F is either interposable or if G is either weak_odr or
  // linkonce_odr.
  if (F->isInterposable() || (isODR(F) && isODR(G))) {
    assert((!isODR(G) || isODR(F)) &&
           "if G is ODR, F must also be ODR due to ordering");

    // Both writeThunkOrAliasIfNeeded() calls below must succeed, either because
    // we can create aliases for G and NewF, or because a thunk for F is
    // profitable. F here has the same signature as NewF below, so that's what
    // we check.
    if (!canCreateThunkFor(F) &&
        (!canCreateAliasFor(F) || !canCreateAliasFor(G)))
      return;

    // Make them both thunks to the same internal function.
    Function *NewF = Function::Create(F->getFunctionType(), F->getLinkage(),
                                      F->getAddressSpace(), "", F->getParent());
    NewF->copyAttributesFrom(F);
    NewF->takeName(F);
    NewF->setComdat(F->getComdat());
    F->setComdat(nullptr);
    // Ensure CFI type metadata is propagated to the new function.
    copyMetadataIfPresent(F, NewF, "type");
    copyMetadataIfPresent(F, NewF, "kcfi_type");
    copyMetadataIfPresent(F, NewF, "callgraph");
    removeUsers(F);
    F->replaceAllUsesWith(NewF);

    // If G or NewF are (weak|linkonce)_odr, update all callers to call the
    // thunk.
    if (isODR(G))
      replaceDirectCallers(G, F);
    if (isODR(F))
      replaceDirectCallers(NewF, F);

    // We collect alignment before writeThunkOrAliasIfNeeded that overwrites
    // NewF and G's content.
    const MaybeAlign NewFAlign = NewF->getAlign();
    const MaybeAlign GAlign = G->getAlign();

    // Merge annotations, while G still has its body.
    mergeInstrAnnotations(F, G);
    mergeEntryCountsAndImportsInto(*F, *G);

    writeThunkOrAliasIfNeeded(F, G);
    if (FEntryCount)
      NewF->setEntryCount(*FEntryCount);
    // NewF becomes thunk/alias to the shared body F, it has no annotations to
    // be merged.
    writeThunkOrAliasIfNeeded(F, NewF);

    if (NewFAlign || GAlign)
      F->setAlignment(std::max(NewFAlign.valueOrOne(), GAlign.valueOrOne()));
    else
      F->setAlignment(std::nullopt);
    F->setLinkage(GlobalValue::PrivateLinkage);
    ++NumDoubleWeak;
    ++NumFunctionsMerged;
  } else {
    // For better debugability, under MergeFunctionsPDI, we do not modify G's
    // call sites to point to F even when within the same translation unit.
    if (!G->isInterposable() && !MergeFunctionsPDI) {
      // Functions referred to by llvm.used/llvm.compiler.used are special:
      // there are uses of the symbol name that are not visible to LLVM,
      // usually from inline asm.
      // Replacing G also retargets G's aliases at F.
      if (G->hasGlobalUnnamedAddr() && !Used.contains(G) &&
          (!hasNonLocalAlias(G) || canBeAliasee(F))) {
        // G might have been a key in our GlobalNumberState, and it's illegal
        // to replace a key in ValueMap<GlobalValue *> with a non-global.
        GlobalNumbers.erase(G);
        // If G's address is not significant, replace it entirely.
        removeUsers(G);
        addMissingCallLocations(G, F);
        G->replaceAllUsesWith(F);
      } else {
        // Redirect direct callers of G to F. (See note on MergeFunctionsPDI
        // above).
        replaceDirectCallers(G, F);
      }
    }

    mergeInstrAnnotations(F, G);
    mergeEntryCountsAndImportsInto(*F, *G);

    // If G was internal then we may have replaced all uses of G with F. If so,
    // stop here and delete G. There's no need for a thunk. (See note on
    // MergeFunctionsPDI above).
    if (G->isDiscardableIfUnused() && G->use_empty() && !MergeFunctionsPDI) {
      G->eraseFromParent();
      ++NumFunctionsMerged;
      return;
    }

    if (writeThunkOrAliasIfNeeded(F, G))
      ++NumFunctionsMerged;
  }
}

/// Replace function F by function G.
void MergeFunctions::replaceFunctionInTree(const FunctionNode &FN,
                                           Function *G) {
  Function *F = FN.getFunc();
  assert(FunctionComparator(F, G, &GlobalNumbers).compare() == 0 &&
         "The two functions must be equal");

  auto I = FNodesInTree.find(F);
  assert(I != FNodesInTree.end() && "F should be in FNodesInTree");
  assert(FNodesInTree.count(G) == 0 && "FNodesInTree should not contain G");

  FnTreeType::iterator IterToFNInFnTree = I->second;
  assert(&(*IterToFNInFnTree) == &FN && "F should map to FN in FNodesInTree.");
  // Remove F -> FN and insert G -> FN
  FNodesInTree.erase(I);
  FNodesInTree.insert({G, IterToFNInFnTree});
  // Replace F with G in FN, which is stored inside the FnTree.
  FN.replaceBy(G);
}

// Ordering for functions that are equal under FunctionComparator
static bool isFuncOrderCorrect(const Function *F, const Function *G) {
  if (isODR(F) != isODR(G)) {
    // ODR functions before non-ODR functions. A ODR function can call a non-ODR
    // function if it is not interposable, but not the other way around.
    return isODR(G);
  }

  if (F->isInterposable() != G->isInterposable()) {
    // Strong before weak, because the weak function may call the strong
    // one, but not the other way around.
    return !F->isInterposable();
  }

  if (F->hasLocalLinkage() != G->hasLocalLinkage()) {
    // External before local, because we definitely have to keep the external
    // function, but may be able to drop the local one.
    return !F->hasLocalLinkage();
  }

  // Impose a total order (by name) on the replacement of functions. This is
  // important when operating on more than one module independently to prevent
  // cycles of thunks calling each other when the modules are linked together.
  return F->getName() <= G->getName();
}

// Insert a ComparableFunction into the FnTree, or merge it away if equal to one
// that was already inserted.
bool MergeFunctions::insert(Function *NewFunction) {
  std::pair<FnTreeType::iterator, bool> Result =
      FnTree.insert(FunctionNode(NewFunction));

  if (Result.second) {
    assert(FNodesInTree.count(NewFunction) == 0);
    FNodesInTree.insert({NewFunction, Result.first});
    LLVM_DEBUG(dbgs() << "Inserting as unique: " << NewFunction->getName()
                      << '\n');
    return false;
  }

  const FunctionNode &OldF = *Result.first;

  if (!isFuncOrderCorrect(OldF.getFunc(), NewFunction)) {
    // Swap the two functions.
    Function *F = OldF.getFunc();
    replaceFunctionInTree(*Result.first, NewFunction);
    NewFunction = F;
    assert(OldF.getFunc() != F && "Must have swapped the functions.");
  }

  // Capture the Function pointer before mergeTwoFunctions, which may invalidate
  // OldF by erasing it from FnTree via removeUsers().
  Function *OldFunc = OldF.getFunc();

  LLVM_DEBUG(dbgs() << "  " << OldFunc->getName()
                    << " == " << NewFunction->getName() << '\n');

  Function *DeleteF = NewFunction;
  mergeTwoFunctions(OldFunc, DeleteF);
  this->DelToNewMap.insert({DeleteF, OldFunc});
  return true;
}

// Remove a function from FnTree. If it was already in FnTree, add
// it to Deferred so that we'll look at it in the next round.
void MergeFunctions::remove(Function *F) {
  auto I = FNodesInTree.find(F);
  if (I != FNodesInTree.end()) {
    LLVM_DEBUG(dbgs() << "Deferred " << F->getName() << ".\n");
    FnTree.erase(I->second);
    // I->second has been invalidated, remove it from the FNodesInTree map to
    // preserve the invariant.
    FNodesInTree.erase(I);
    Deferred.emplace_back(F);
  }
}

// For each instruction used by the value, remove() the function that contains
// the instruction. This should happen right before a call to RAUW.
void MergeFunctions::removeUsers(Value *V) {
  for (User *U : V->users())
    if (auto *I = dyn_cast<Instruction>(U))
      remove(I->getFunction());
}