//===---------- PPCTLSDynamicCall.cpp - TLS Dynamic Call Fixup ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass expands ADDItls{ld,gd}LADDR[32] machine instructions into
// separate ADDItls[gd]L[32] and GETtlsADDR[32] instructions, both of
// which define GPR3.  A copy is added from GPR3 to the target virtual
// register of the original instruction.  The GETtlsADDR[32] is really
// a call instruction, so its target register is constrained to be GPR3.
// This is not true of ADDItls[gd]L[32], but there is a legacy linker
// optimization bug that requires the target register of the addi of
// a local- or general-dynamic TLS access sequence to be GPR3.
//
// This is done in a late pass so that TLS variable accesses can be
// fully commoned by MachineCSE.
//
//===----------------------------------------------------------------------===//

#include "PPC.h"
#include "PPCInstrInfo.h"
#include "PPCTargetMachine.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

using namespace llvm;

#define DEBUG_TYPE "ppc-tls-dynamic-call"

namespace {
  struct PPCTLSDynamicCall : public MachineFunctionPass {
    static char ID;
    PPCTLSDynamicCall() : MachineFunctionPass(ID) {}

    const PPCInstrInfo *TII;

protected:
    bool processBlock(MachineBasicBlock &MBB) {
      bool Changed = false;
      bool NeedFence = true;
      const PPCSubtarget &Subtarget =
          MBB.getParent()->getSubtarget<PPCSubtarget>();
      bool Is64Bit = Subtarget.isPPC64();
      bool IsAIX = Subtarget.isAIXABI();
      bool IsLargeModel =
          Subtarget.getTargetMachine().getCodeModel() == CodeModel::Large;
      bool IsPCREL = false;
      MachineFunction *MF = MBB.getParent();
      MachineRegisterInfo &RegInfo = MF->getRegInfo();

      for (MachineBasicBlock::iterator I = MBB.begin(), IE = MBB.end();
           I != IE;) {
        MachineInstr &MI = *I;
        IsPCREL = isPCREL(MI);
        // There are a number of slight differences in code generation
        // when we call .__get_tpointer (32-bit AIX TLS).
        bool IsTLSTPRelMI = MI.getOpcode() == PPC::GETtlsTpointer32AIX;
        bool IsTLSLDAIXMI = (MI.getOpcode() == PPC::TLSLDAIX8 ||
                             MI.getOpcode() == PPC::TLSLDAIX);

        if (MI.getOpcode() != PPC::ADDItlsgdLADDR &&
            MI.getOpcode() != PPC::ADDItlsldLADDR &&
            MI.getOpcode() != PPC::ADDItlsgdLADDR32 &&
            MI.getOpcode() != PPC::ADDItlsldLADDR32 &&
            MI.getOpcode() != PPC::TLSGDAIX &&
            MI.getOpcode() != PPC::TLSGDAIX8 && !IsTLSTPRelMI && !IsPCREL &&
            !IsTLSLDAIXMI) {
          // Although we create ADJCALLSTACKDOWN and ADJCALLSTACKUP
          // as scheduling fences, we skip creating fences if we already
          // have existing ADJCALLSTACKDOWN/UP to avoid nesting,
          // which causes verification error with -verify-machineinstrs.
          if (MI.getOpcode() == PPC::ADJCALLSTACKDOWN)
            NeedFence = false;
          else if (MI.getOpcode() == PPC::ADJCALLSTACKUP)
            NeedFence = true;

          ++I;
          continue;
        }

        LLVM_DEBUG(dbgs() << "TLS Dynamic Call Fixup:\n    " << MI);

        Register OutReg = MI.getOperand(0).getReg();
        Register InReg = PPC::NoRegister;
        Register GPR3 = Is64Bit ? PPC::X3 : PPC::R3;
        Register GPR4 = Is64Bit ? PPC::X4 : PPC::R4;
        if (!IsPCREL && !IsTLSTPRelMI)
          InReg = MI.getOperand(1).getReg();
        DebugLoc DL = MI.getDebugLoc();

        unsigned Opc1, Opc2;
        switch (MI.getOpcode()) {
        default:
          llvm_unreachable("Opcode inconsistency error");
        case PPC::ADDItlsgdLADDR:
          Opc1 = PPC::ADDItlsgdL;
          Opc2 = PPC::GETtlsADDR;
          break;
        case PPC::ADDItlsldLADDR:
          Opc1 = PPC::ADDItlsldL;
          Opc2 = PPC::GETtlsldADDR;
          break;
        case PPC::ADDItlsgdLADDR32:
          Opc1 = PPC::ADDItlsgdL32;
          Opc2 = PPC::GETtlsADDR32;
          break;
        case PPC::ADDItlsldLADDR32:
          Opc1 = PPC::ADDItlsldL32;
          Opc2 = PPC::GETtlsldADDR32;
          break;
        case PPC::TLSLDAIX:
          // TLSLDAIX is expanded to one copy and GET_TLS_MOD, so we only set
          // Opc2 here.
          Opc2 = PPC::GETtlsMOD32AIX;
          break;
        case PPC::TLSLDAIX8:
          // TLSLDAIX8 is expanded to one copy and GET_TLS_MOD, so we only set
          // Opc2 here.
          Opc2 = PPC::GETtlsMOD64AIX;
          break;
        case PPC::TLSGDAIX8:
          // TLSGDAIX8 is expanded to two copies and GET_TLS_ADDR, so we only
          // set Opc2 here.
          Opc2 = PPC::GETtlsADDR64AIX;
          break;
        case PPC::TLSGDAIX:
          // TLSGDAIX is expanded to two copies and GET_TLS_ADDR, so we only
          // set Opc2 here.
          Opc2 = PPC::GETtlsADDR32AIX;
          break;
        case PPC::GETtlsTpointer32AIX:
          // GETtlsTpointer32AIX is expanded to a call to GET_TPOINTER on AIX
          // 32-bit mode within PPCAsmPrinter. This instruction does not need
          // to change, so Opc2 is set to the same instruction opcode.
          Opc2 = PPC::GETtlsTpointer32AIX;
          break;
        case PPC::PADDI8pc:
          assert(IsPCREL && "Expecting General/Local Dynamic PCRel");
          Opc1 = PPC::PADDI8pc;
          Opc2 = MI.getOperand(2).getTargetFlags() ==
                         PPCII::MO_GOT_TLSGD_PCREL_FLAG
                     ? PPC::GETtlsADDRPCREL
                     : PPC::GETtlsldADDRPCREL;
        }

        // We create ADJCALLSTACKUP and ADJCALLSTACKDOWN around _tls_get_addr
        // as scheduling fence to avoid it is scheduled before
        // mflr in the prologue and the address in LR is clobbered (PR25839).
        // We don't really need to save data to the stack - the clobbered
        // registers are already saved when the SDNode (e.g. PPCaddiTlsgdLAddr)
        // gets translated to the pseudo instruction (e.g. ADDItlsgdLADDR).
        if (NeedFence) {
          MBB.getParent()->getFrameInfo().setAdjustsStack(true);
          BuildMI(MBB, I, DL, TII->get(PPC::ADJCALLSTACKDOWN)).addImm(0)
                                                              .addImm(0);
        }

        MachineInstr *Call;
        if (IsAIX) {
          if (IsTLSLDAIXMI) {
            // The relative order between the node that loads the variable
            // offset from the TOC, and the .__tls_get_mod node is being tuned
            // here. It is better to put the variable offset TOC load after the
            // call, since this node can use clobbers r4/r5.
            // Search for the pattern of the two nodes that load from the TOC
            // (either for the variable offset or for the module handle), and
            // then move the variable offset TOC load right before the node that
            // uses the OutReg of the .__tls_get_mod node.
            unsigned LDTocOp =
                Is64Bit ? (IsLargeModel ? PPC::LDtocL : PPC::LDtoc)
                        : (IsLargeModel ? PPC::LWZtocL : PPC::LWZtoc);
            if (!RegInfo.use_empty(OutReg)) {
              std::set<MachineInstr *> Uses;
              // Collect all instructions that use the OutReg.
              for (MachineInstr &UseMI : RegInfo.use_instructions(OutReg))
                Uses.insert(&UseMI);
              // Find the first user (e.g.: lwax/stfdx) of the OutReg within the
              // current BB.
              MachineBasicBlock::iterator UseIter = MBB.begin();
              for (MachineBasicBlock::iterator IE = MBB.end(); UseIter != IE;
                   ++UseIter)
                if (Uses.count(&*UseIter))
                  break;

              // Additional handling is required when UserIter (the first user
              // of OutReg) is pointing to a valid node that loads from the TOC.
              // Check the pattern and do the movement if the pattern matches.
              if (UseIter != MBB.end()) {
                // Collect all associated nodes that load from the TOC. Use
                // hasOneDef() to guard against unexpected scenarios.
                std::set<MachineInstr *> LoadFromTocs;
                for (MachineOperand &MO : UseIter->operands())
                  if (MO.isReg() && MO.isUse()) {
                    Register MOReg = MO.getReg();
                    if (RegInfo.hasOneDef(MOReg)) {
                      MachineInstr *Temp =
                          RegInfo.getOneDef(MOReg)->getParent();
                      // For the current TLSLDAIX node, get the corresponding
                      // node that loads from the TOC for the InReg. Otherwise,
                      // Temp probably pointed to the variable offset TOC load
                      // we would like to move.
                      if (Temp == &MI && RegInfo.hasOneDef(InReg))
                        Temp = RegInfo.getOneDef(InReg)->getParent();
                      if (Temp->getOpcode() == LDTocOp)
                        LoadFromTocs.insert(Temp);
                    } else {
                      // FIXME: analyze this scenario if there is one.
                      LoadFromTocs.clear();
                      break;
                    }
                  }

                // Check the two nodes that loaded from the TOC: one should be
                // "_$TLSML", and the other will be moved before the node that
                // uses the OutReg of the .__tls_get_mod node.
                if (LoadFromTocs.size() == 2) {
                  MachineBasicBlock::iterator TLSMLIter = MBB.end();
                  MachineBasicBlock::iterator OffsetIter = MBB.end();
                  // Make sure the two nodes that loaded from the TOC are within
                  // the current BB, and that one of them is from the "_$TLSML"
                  // pseudo symbol, while the other is from the variable.
                  for (MachineBasicBlock::iterator I = MBB.begin(),
                                                   IE = MBB.end();
                       I != IE; ++I)
                    if (LoadFromTocs.count(&*I)) {
                      MachineOperand MO = I->getOperand(1);
                      if (MO.isGlobal() && MO.getGlobal()->hasName() &&
                          MO.getGlobal()->getName() == "_$TLSML")
                        TLSMLIter = I;
                      else
                        OffsetIter = I;
                    }
                  // Perform the movement when the desired scenario has been
                  // identified, which should be when both of the iterators are
                  // valid.
                  if (TLSMLIter != MBB.end() && OffsetIter != MBB.end())
                    OffsetIter->moveBefore(&*UseIter);
                }
              }
            }
            // The module-handle is copied into r3. The copy is followed by
            // GETtlsMOD32AIX/GETtlsMOD64AIX.
            BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR3)
                .addReg(InReg);
            // The call to .__tls_get_mod.
            Call = BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3);
          } else if (!IsTLSTPRelMI) {
            // The variable offset and region handle (for TLSGD) are copied in
            // r4 and r3. The copies are followed by
            // GETtlsADDR32AIX/GETtlsADDR64AIX.
            BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR4)
                .addReg(MI.getOperand(1).getReg());
            BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), GPR3)
                .addReg(MI.getOperand(2).getReg());
            Call = BuildMI(MBB, I, DL, TII->get(Opc2), GPR3)
                       .addReg(GPR3)
                       .addReg(GPR4);
          } else
            // The opcode of GETtlsTpointer32AIX does not change, because later
            // this instruction will be expanded into a call to .__get_tpointer,
            // which will return the thread pointer into r3.
            Call = BuildMI(MBB, I, DL, TII->get(Opc2), GPR3);
        } else {
          MachineInstr *Addi;
          if (IsPCREL) {
            Addi = BuildMI(MBB, I, DL, TII->get(Opc1), GPR3).addImm(0);
          } else {
            // Expand into two ops built prior to the existing instruction.
            assert(InReg != PPC::NoRegister && "Operand must be a register");
            Addi = BuildMI(MBB, I, DL, TII->get(Opc1), GPR3).addReg(InReg);
          }

          Addi->addOperand(MI.getOperand(2));

          Call = BuildMI(MBB, I, DL, TII->get(Opc2), GPR3).addReg(GPR3);
          if (IsPCREL)
            Call->addOperand(MI.getOperand(2));
          else
            Call->addOperand(MI.getOperand(3));
        }

        Call->setPhysRegsDeadExcept(GPR3, *Subtarget.getRegisterInfo());

        if (NeedFence)
          BuildMI(MBB, I, DL, TII->get(PPC::ADJCALLSTACKUP)).addImm(0).addImm(0);

        BuildMI(MBB, I, DL, TII->get(TargetOpcode::COPY), OutReg)
          .addReg(GPR3);

        // Move past the original instruction and remove it.
        ++I;
        MI.removeFromParent();

        Changed = true;
      }

      return Changed;
    }

public:
  bool isPCREL(const MachineInstr &MI) {
    return (MI.getOpcode() == PPC::PADDI8pc) &&
           (MI.getOperand(2).getTargetFlags() ==
                PPCII::MO_GOT_TLSGD_PCREL_FLAG ||
            MI.getOperand(2).getTargetFlags() ==
                PPCII::MO_GOT_TLSLD_PCREL_FLAG);
  }

    bool runOnMachineFunction(MachineFunction &MF) override {
      TII = MF.getSubtarget<PPCSubtarget>().getInstrInfo();

      bool Changed = false;

      for (MachineBasicBlock &B : llvm::make_early_inc_range(MF))
        if (processBlock(B))
          Changed = true;

      return Changed;
    }

    void getAnalysisUsage(AnalysisUsage &AU) const override {
      AU.addRequired<LiveIntervalsWrapperPass>();
      AU.addRequired<SlotIndexesWrapperPass>();
      AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
      MachineFunctionPass::getAnalysisUsage(AU);
    }
  };
}

INITIALIZE_PASS_BEGIN(PPCTLSDynamicCall, DEBUG_TYPE,
                      "PowerPC TLS Dynamic Call Fixup", false, false)
INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
INITIALIZE_PASS_END(PPCTLSDynamicCall, DEBUG_TYPE,
                    "PowerPC TLS Dynamic Call Fixup", false, false)

char PPCTLSDynamicCall::ID = 0;
FunctionPass*
llvm::createPPCTLSDynamicCallPass() { return new PPCTLSDynamicCall(); }