llvm/lib/ExecutionEngine/JITLink/aarch32.cpp

//===--------- aarch32.cpp - Generic JITLink arm/thumb utilities ----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Generic utilities for graphs representing arm/thumb objects.
//
//===----------------------------------------------------------------------===//

#include "llvm/ExecutionEngine/JITLink/aarch32.h"

#include "llvm/ADT/StringExtras.h"
#include "llvm/ExecutionEngine/JITLink/JITLink.h"
#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/MathExtras.h"

#define DEBUG_TYPE "jitlink"

namespace llvm {
namespace jitlink {
namespace aarch32 {

/// Check whether the given target flags are set for this Symbol.
bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) {
  return Sym.getTargetFlags() & Flags;
}

/// Encode 22-bit immediate value for branch instructions without J1J2 range
/// extension (formats B T4, BL T1 and BLX T2).
///
///   00000:Imm11H:Imm11L:0 -> [ 00000:Imm11H, 00000:Imm11L ]
///                                            J1^ ^J2 will always be 1
///
HalfWords encodeImmBT4BlT1BlxT2(int64_t Value) {
  constexpr uint32_t J1J2 = 0x2800;
  uint32_t Imm11H = (Value >> 12) & 0x07ff;
  uint32_t Imm11L = (Value >> 1) & 0x07ff;
  return HalfWords{Imm11H, Imm11L | J1J2};
}

/// Decode 22-bit immediate value for branch instructions without J1J2 range
/// extension (formats B T4, BL T1 and BLX T2).
///
///   [ 00000:Imm11H, 00000:Imm11L ] -> 00000:Imm11H:Imm11L:0
///                   J1^ ^J2 will always be 1
///
int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) {
  uint32_t Imm11H = Hi & 0x07ff;
  uint32_t Imm11L = Lo & 0x07ff;
  return SignExtend64<22>(Imm11H << 12 | Imm11L << 1);
}

/// Encode 25-bit immediate value for branch instructions with J1J2 range
/// extension (formats B T4, BL T1 and BLX T2).
///
///   S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ]
///
LLVM_ABI HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
  uint32_t S = (Value >> 14) & 0x0400;
  uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000);
  uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800);
  uint32_t Imm10 = (Value >> 12) & 0x03ff;
  uint32_t Imm11 = (Value >> 1) & 0x07ff;
  return HalfWords{S | Imm10, J1 | J2 | Imm11};
}

/// Decode 25-bit immediate value for branch instructions with J1J2 range
/// extension (formats B T4, BL T1 and BLX T2).
///
///   [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:0
///
LLVM_ABI int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
  uint32_t S = Hi & 0x0400;
  uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000;
  uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000;
  uint32_t Imm10 = Hi & 0x03ff;
  uint32_t Imm11 = Lo & 0x07ff;
  return SignExtend64<25>(S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1);
}

/// Encode 26-bit immediate value for branch instructions
/// (formats B A1, BL A1 and BLX A2).
///
///   Imm24:00 ->  00000000:Imm24
///
LLVM_ABI uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
  return (Value >> 2) & 0x00ffffff;
}

/// Decode 26-bit immediate value for branch instructions
/// (formats B A1, BL A1 and BLX A2).
///
///   00000000:Imm24 ->  Imm24:00
///
LLVM_ABI int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
  return SignExtend64<26>((Value & 0x00ffffff) << 2);
}

/// Encode 16-bit immediate value for move instruction formats MOVT T1 and
/// MOVW T3.
///
///   Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ]
///
LLVM_ABI HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
  uint32_t Imm4 = (Value >> 12) & 0x0f;
  uint32_t Imm1 = (Value >> 11) & 0x01;
  uint32_t Imm3 = (Value >> 8) & 0x07;
  uint32_t Imm8 = Value & 0xff;
  return HalfWords{Imm1 << 10 | Imm4, Imm3 << 12 | Imm8};
}

/// Decode 16-bit immediate value from move instruction formats MOVT T1 and
/// MOVW T3.
///
///   [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8
///
LLVM_ABI uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
  uint32_t Imm4 = Hi & 0x0f;
  uint32_t Imm1 = (Hi >> 10) & 0x01;
  uint32_t Imm3 = (Lo >> 12) & 0x07;
  uint32_t Imm8 = Lo & 0xff;
  uint32_t Imm16 = Imm4 << 12 | Imm1 << 11 | Imm3 << 8 | Imm8;
  assert(Imm16 <= 0xffff && "Decoded value out-of-range");
  return Imm16;
}

/// Encode register ID for instruction formats MOVT T1 and MOVW T3.
///
///   Rd4 -> [0000000000000000, 0000:Rd4:00000000]
///
LLVM_ABI HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
  uint32_t Rd4 = (Value & 0x0f) << 8;
  return HalfWords{0, Rd4};
}

/// Decode register ID from instruction formats MOVT T1 and MOVW T3.
///
///   [0000000000000000, 0000:Rd4:00000000] -> Rd4
///
LLVM_ABI int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
  uint32_t Rd4 = (Lo >> 8) & 0x0f;
  return Rd4;
}

/// Encode 16-bit immediate value for move instruction formats MOVT A1 and
/// MOVW A2.
///
///   Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12
///
LLVM_ABI uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
  uint32_t Imm4 = (Value >> 12) & 0x0f;
  uint32_t Imm12 = Value & 0x0fff;
  return (Imm4 << 16) | Imm12;
}

/// Decode 16-bit immediate value for move instruction formats MOVT A1 and
/// MOVW A2.
///
///   000000000000:Imm4:0000:Imm12 -> Imm4:Imm12
///
LLVM_ABI uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
  uint32_t Imm4 = (Value >> 16) & 0x0f;
  uint32_t Imm12 = Value & 0x0fff;
  return (Imm4 << 12) | Imm12;
}

/// Encode register ID for instruction formats MOVT A1 and
/// MOVW A2.
///
///   Rd4 -> 0000000000000000:Rd4:000000000000
///
LLVM_ABI uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
  uint32_t Rd4 = (Value & 0x00000f) << 12;
  return Rd4;
}

/// Decode register ID for instruction formats MOVT A1 and
/// MOVW A2.
///
///   0000000000000000:Rd4:000000000000 -> Rd4
///
LLVM_ABI int64_t decodeRegMovtA1MovwA2(uint64_t Value) {
  uint32_t Rd4 = (Value >> 12) & 0x00000f;
  return Rd4;
}

namespace {

/// 32-bit Thumb instructions are stored as two little-endian halfwords.
/// An instruction at address A encodes bytes A+1, A in the first halfword (Hi),
/// followed by bytes A+3, A+2 in the second halfword (Lo).
struct WritableThumbRelocation {
  /// Create a writable reference to a Thumb32 fixup.
  WritableThumbRelocation(char *FixupPtr)
      : Hi{*reinterpret_cast<support::ulittle16_t *>(FixupPtr)},
        Lo{*reinterpret_cast<support::ulittle16_t *>(FixupPtr + 2)} {}

  support::ulittle16_t &Hi; // First halfword
  support::ulittle16_t &Lo; // Second halfword
};

struct ThumbRelocation {
  /// Create a read-only reference to a Thumb32 fixup.
  ThumbRelocation(const char *FixupPtr)
      : Hi{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr)},
        Lo{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr + 2)} {}

  /// Create a read-only Thumb32 fixup from a writeable one.
  ThumbRelocation(WritableThumbRelocation &Writable)
      : Hi{Writable.Hi}, Lo(Writable.Lo) {}

  const support::ulittle16_t &Hi; // First halfword
  const support::ulittle16_t &Lo; // Second halfword
};

struct WritableArmRelocation {
  WritableArmRelocation(char *FixupPtr)
      : Wd{*reinterpret_cast<support::ulittle32_t *>(FixupPtr)} {}

  support::ulittle32_t &Wd;
};

struct ArmRelocation {
  ArmRelocation(const char *FixupPtr)
      : Wd{*reinterpret_cast<const support::ulittle32_t *>(FixupPtr)} {}

  ArmRelocation(WritableArmRelocation &Writable) : Wd{Writable.Wd} {}

  const support::ulittle32_t &Wd;
};

Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R,
                                Edge::Kind Kind) {
  return make_error<JITLinkError>(
      formatv("Invalid opcode [ {0:x4}, {1:x4} ] for relocation: {2}",
              R.Hi.value(), R.Lo.value(), G.getEdgeKindName(Kind)));
}

Error makeUnexpectedOpcodeError(const LinkGraph &G, const ArmRelocation &R,
                                Edge::Kind Kind) {
  return make_error<JITLinkError>(
      formatv("Invalid opcode {0:x8} for relocation: {1}", R.Wd.value(),
              G.getEdgeKindName(Kind)));
}

template <EdgeKind_aarch32 K> constexpr bool isArm() {
  return FirstArmRelocation <= K && K <= LastArmRelocation;
}
template <EdgeKind_aarch32 K> constexpr bool isThumb() {
  return FirstThumbRelocation <= K && K <= LastThumbRelocation;
}

template <EdgeKind_aarch32 K> static bool checkOpcodeArm(uint32_t Wd) {
  return (Wd & FixupInfo<K>::OpcodeMask) == FixupInfo<K>::Opcode;
}

template <EdgeKind_aarch32 K>
static bool checkOpcodeThumb(uint16_t Hi, uint16_t Lo) {
  return (Hi & FixupInfo<K>::OpcodeMask.Hi) == FixupInfo<K>::Opcode.Hi &&
         (Lo & FixupInfo<K>::OpcodeMask.Lo) == FixupInfo<K>::Opcode.Lo;
}

class FixupInfoTable {
  static constexpr size_t Items = LastRelocation + 1;

public:
  FixupInfoTable() {
    populateEntries<FirstArmRelocation, LastArmRelocation>();
    populateEntries<FirstThumbRelocation, LastThumbRelocation>();
  }

  const FixupInfoBase *getEntry(Edge::Kind K) {
    assert(K < Data.size() && "Index out of bounds");
    return Data.at(K).get();
  }

private:
  template <EdgeKind_aarch32 K, EdgeKind_aarch32 LastK> void populateEntries() {
    assert(K < Data.size() && "Index out of range");
    assert(Data.at(K) == nullptr && "Initialized entries are immutable");
    Data[K] = initEntry<K>();
    if constexpr (K < LastK) {
      constexpr auto Next = static_cast<EdgeKind_aarch32>(K + 1);
      populateEntries<Next, LastK>();
    }
  }

  template <EdgeKind_aarch32 K>
  static std::unique_ptr<FixupInfoBase> initEntry() {
    auto Entry = std::make_unique<FixupInfo<K>>();
    static_assert(isArm<K>() != isThumb<K>(), "Classes are mutually exclusive");
    if constexpr (isArm<K>())
      Entry->checkOpcode = checkOpcodeArm<K>;
    if constexpr (isThumb<K>())
      Entry->checkOpcode = checkOpcodeThumb<K>;
    return Entry;
  }

private:
  std::array<std::unique_ptr<FixupInfoBase>, Items> Data;
};

ManagedStatic<FixupInfoTable> DynFixupInfos;

} // namespace

static Error checkOpcode(LinkGraph &G, const ArmRelocation &R,
                         Edge::Kind Kind) {
  assert(Kind >= FirstArmRelocation && Kind <= LastArmRelocation &&
         "Edge kind must be Arm relocation");
  const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind);
  const FixupInfoArm &Info = *static_cast<const FixupInfoArm *>(Entry);
  assert(Info.checkOpcode && "Opcode check is mandatory for Arm edges");
  if (!Info.checkOpcode(R.Wd))
    return makeUnexpectedOpcodeError(G, R, Kind);

  return Error::success();
}

static Error checkOpcode(LinkGraph &G, const ThumbRelocation &R,
                         Edge::Kind Kind) {
  assert(Kind >= FirstThumbRelocation && Kind <= LastThumbRelocation &&
         "Edge kind must be Thumb relocation");
  const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind);
  const FixupInfoThumb &Info = *static_cast<const FixupInfoThumb *>(Entry);
  assert(Info.checkOpcode && "Opcode check is mandatory for Thumb edges");
  if (!Info.checkOpcode(R.Hi, R.Lo))
    return makeUnexpectedOpcodeError(G, R, Kind);

  return Error::success();
}

const FixupInfoBase *FixupInfoBase::getDynFixupInfo(Edge::Kind K) {
  return DynFixupInfos->getEntry(K);
}

template <EdgeKind_aarch32 Kind>
[[maybe_unused]] bool checkRegister(const ThumbRelocation &R, HalfWords Reg) {
  uint16_t Hi = R.Hi & FixupInfo<Kind>::RegMask.Hi;
  uint16_t Lo = R.Lo & FixupInfo<Kind>::RegMask.Lo;
  return Hi == Reg.Hi && Lo == Reg.Lo;
}

template <EdgeKind_aarch32 Kind>
[[maybe_unused]] bool checkRegister(const ArmRelocation &R, uint32_t Reg) {
  uint32_t Wd = R.Wd & FixupInfo<Kind>::RegMask;
  return Wd == Reg;
}

template <EdgeKind_aarch32 Kind>
void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) {
  static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask;
  assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Lo & Imm.Lo) == Imm.Lo &&
         "Value bits exceed bit range of given mask");
  R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi;
  R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo;
}

template <EdgeKind_aarch32 Kind>
void writeImmediate(WritableArmRelocation &R, uint32_t Imm) {
  static constexpr uint32_t Mask = FixupInfo<Kind>::ImmMask;
  assert((Mask & Imm) == Imm && "Value bits exceed bit range of given mask");
  R.Wd = (R.Wd & ~Mask) | Imm;
}

Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset,
                                 Edge::Kind Kind) {
  endianness Endian = G.getEndianness();
  const char *BlockWorkingMem = B.getContent().data();
  const char *FixupPtr = BlockWorkingMem + Offset;

  switch (Kind) {
  case Data_Delta32:
  case Data_Pointer32:
  case Data_RequestGOTAndTransformToDelta32:
    return SignExtend64<32>(support::endian::read32(FixupPtr, Endian));
  case Data_PRel31:
    return SignExtend64<31>(support::endian::read32(FixupPtr, Endian));
  default:
    return make_error<JITLinkError>(
        "In graph " + G.getName() + ", section " + B.getSection().getName() +
        " can not read implicit addend for aarch32 edge kind " +
        G.getEdgeKindName(Kind));
  }
}

Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset,
                                Edge::Kind Kind) {
  ArmRelocation R(B.getContent().data() + Offset);
  if (Error Err = checkOpcode(G, R, Kind))
    return std::move(Err);

  switch (Kind) {
  case Arm_Call:
  case Arm_Jump24:
    return decodeImmBA1BlA1BlxA2(R.Wd);

  case Arm_MovtAbs:
  case Arm_MovwAbsNC:
    return decodeImmMovtA1MovwA2(R.Wd);

  default:
    return make_error<JITLinkError>(
        "In graph " + G.getName() + ", section " + B.getSection().getName() +
        " can not read implicit addend for aarch32 edge kind " +
        G.getEdgeKindName(Kind));
  }
}

Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset,
                                  Edge::Kind Kind, const ArmConfig &ArmCfg) {
  ThumbRelocation R(B.getContent().data() + Offset);
  if (Error Err = checkOpcode(G, R, Kind))
    return std::move(Err);

  switch (Kind) {
  case Thumb_Call:
  case Thumb_Jump24:
    return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)
               ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo)
               : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo);

  case Thumb_MovwAbsNC:
  case Thumb_MovwPrelNC:
    // Initial addend is interpreted as a signed value
    return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo));

  case Thumb_MovtAbs:
  case Thumb_MovtPrel:
    // Initial addend is interpreted as a signed value
    return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo));

  default:
    return make_error<JITLinkError>(
        "In graph " + G.getName() + ", section " + B.getSection().getName() +
        " can not read implicit addend for aarch32 edge kind " +
        G.getEdgeKindName(Kind));
  }
}

Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) {
  using namespace support;

  char *BlockWorkingMem = B.getAlreadyMutableContent().data();
  char *FixupPtr = BlockWorkingMem + E.getOffset();

  Edge::Kind Kind = E.getKind();
  uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
  int64_t Addend = E.getAddend();
  Symbol &TargetSymbol = E.getTarget();
  uint64_t TargetAddress = TargetSymbol.getAddress().getValue();

  // Data relocations have alignment 1, size 4 (except R_ARM_ABS8 and
  // R_ARM_ABS16) and write the full 32-bit result (except R_ARM_PREL31).
  switch (Kind) {
  case Data_Delta32: {
    int64_t Value = TargetAddress - FixupAddress + Addend;
    if (!isInt<32>(Value))
      return makeTargetOutOfRangeError(G, B, E);
    if (LLVM_LIKELY(G.getEndianness() == endianness::little))
      endian::write32le(FixupPtr, Value);
    else
      endian::write32be(FixupPtr, Value);
    return Error::success();
  }
  case Data_Pointer32: {
    int64_t Value = TargetAddress + Addend;
    if (!isUInt<32>(Value))
      return makeTargetOutOfRangeError(G, B, E);
    if (LLVM_LIKELY(G.getEndianness() == endianness::little))
      endian::write32le(FixupPtr, Value);
    else
      endian::write32be(FixupPtr, Value);
    return Error::success();
  }
  case Data_PRel31: {
    int64_t Value = TargetAddress - FixupAddress + Addend;
    if (!isInt<31>(Value))
      return makeTargetOutOfRangeError(G, B, E);
    if (LLVM_LIKELY(G.getEndianness() == endianness::little)) {
      uint32_t MSB = endian::read32le(FixupPtr) & 0x80000000;
      endian::write32le(FixupPtr, MSB | (Value & ~0x80000000));
    } else {
      uint32_t MSB = endian::read32be(FixupPtr) & 0x80000000;
      endian::write32be(FixupPtr, MSB | (Value & ~0x80000000));
    }
    return Error::success();
  }
  case Data_RequestGOTAndTransformToDelta32:
    llvm_unreachable("Should be transformed");
  default:
    return make_error<JITLinkError>(
        "In graph " + G.getName() + ", section " + B.getSection().getName() +
        " encountered unfixable aarch32 edge kind " +
        G.getEdgeKindName(E.getKind()));
  }
}

Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) {
  WritableArmRelocation R(B.getAlreadyMutableContent().data() + E.getOffset());
  Edge::Kind Kind = E.getKind();
  if (Error Err = checkOpcode(G, R, Kind))
    return Err;

  uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
  int64_t Addend = E.getAddend();
  Symbol &TargetSymbol = E.getTarget();
  uint64_t TargetAddress = TargetSymbol.getAddress().getValue();

  switch (Kind) {
  case Arm_Jump24: {
    if (hasTargetFlags(TargetSymbol, ThumbSymbol))
      return make_error<JITLinkError>("Branch relocation needs interworking "
                                      "stub when bridging to Thumb: " +
                                      StringRef(G.getEdgeKindName(Kind)));

    int64_t Value = TargetAddress - FixupAddress + Addend;

    if (!isInt<26>(Value))
      return makeTargetOutOfRangeError(G, B, E);
    writeImmediate<Arm_Jump24>(R, encodeImmBA1BlA1BlxA2(Value));

    return Error::success();
  }
  case Arm_Call: {
    if ((R.Wd & FixupInfo<Arm_Call>::CondMask) !=
        FixupInfo<Arm_Call>::Unconditional)
      return make_error<JITLinkError>("Relocation expects an unconditional "
                                      "BL/BLX branch instruction: " +
                                      StringRef(G.getEdgeKindName(Kind)));

    int64_t Value = TargetAddress - FixupAddress + Addend;

    // The call instruction itself is Arm. The call destination can either be
    // Thumb or Arm. We use BL to stay in Arm and BLX to change to Thumb.
    bool TargetIsThumb = hasTargetFlags(TargetSymbol, ThumbSymbol);
    bool InstrIsBlx = (~R.Wd & FixupInfo<Arm_Call>::BitBlx) == 0;
    if (TargetIsThumb != InstrIsBlx) {
      if (LLVM_LIKELY(TargetIsThumb)) {
        // Change opcode BL -> BLX
        R.Wd = R.Wd | FixupInfo<Arm_Call>::BitBlx;
        R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitH;
      } else {
        // Change opcode BLX -> BL
        R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitBlx;
      }
    }

    if (!isInt<26>(Value))
      return makeTargetOutOfRangeError(G, B, E);
    writeImmediate<Arm_Call>(R, encodeImmBA1BlA1BlxA2(Value));

    return Error::success();
  }
  case Arm_MovwAbsNC: {
    uint16_t Value = (TargetAddress + Addend) & 0xffff;
    writeImmediate<Arm_MovwAbsNC>(R, encodeImmMovtA1MovwA2(Value));
    return Error::success();
  }
  case Arm_MovtAbs: {
    uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
    writeImmediate<Arm_MovtAbs>(R, encodeImmMovtA1MovwA2(Value));
    return Error::success();
  }
  default:
    return make_error<JITLinkError>(
        "In graph " + G.getName() + ", section " + B.getSection().getName() +
        " encountered unfixable aarch32 edge kind " +
        G.getEdgeKindName(E.getKind()));
  }
}

Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
                      const ArmConfig &ArmCfg) {
  WritableThumbRelocation R(B.getAlreadyMutableContent().data() +
                            E.getOffset());
  Edge::Kind Kind = E.getKind();
  if (Error Err = checkOpcode(G, R, Kind))
    return Err;

  uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
  int64_t Addend = E.getAddend();
  Symbol &TargetSymbol = E.getTarget();
  uint64_t TargetAddress = TargetSymbol.getAddress().getValue();

  switch (Kind) {
  case Thumb_Jump24: {
    if (!hasTargetFlags(TargetSymbol, ThumbSymbol))
      return make_error<JITLinkError>("Branch relocation needs interworking "
                                      "stub when bridging to ARM: " +
                                      StringRef(G.getEdgeKindName(Kind)));

    int64_t Value = TargetAddress - FixupAddress + Addend;
    if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
      if (!isInt<25>(Value))
        return makeTargetOutOfRangeError(G, B, E);
      writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2_J1J2(Value));
    } else {
      if (!isInt<22>(Value))
        return makeTargetOutOfRangeError(G, B, E);
      writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2(Value));
    }

    return Error::success();
  }

  case Thumb_Call: {
    int64_t Value = TargetAddress - FixupAddress + Addend;

    // The call instruction itself is Thumb. The call destination can either be
    // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm.
    bool TargetIsArm = !hasTargetFlags(TargetSymbol, ThumbSymbol);
    bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0;
    if (TargetIsArm != InstrIsBlx) {
      if (LLVM_LIKELY(TargetIsArm)) {
        // Change opcode BL -> BLX and fix range value: account for 4-byte
        // aligned destination while instruction may only be 2-byte aligned
        R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
        R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH;
        Value = alignTo(Value, 4);
      } else {
        // Change opcode BLX -> BL
        R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
      }
    }

    if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
      if (!isInt<25>(Value))
        return makeTargetOutOfRangeError(G, B, E);
      writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2_J1J2(Value));
    } else {
      if (!isInt<22>(Value))
        return makeTargetOutOfRangeError(G, B, E);
      writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2(Value));
    }

    assert(((R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) ||
            (R.Lo & FixupInfo<Thumb_Call>::LoBitH) == 0) &&
           "Opcode BLX implies H bit is clear (avoid UB in BLX T2)");
    return Error::success();
  }

  case Thumb_MovwAbsNC: {
    uint16_t Value = (TargetAddress + Addend) & 0xffff;
    writeImmediate<Thumb_MovwAbsNC>(R, encodeImmMovtT1MovwT3(Value));
    return Error::success();
  }
  case Thumb_MovtAbs: {
    uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
    writeImmediate<Thumb_MovtAbs>(R, encodeImmMovtT1MovwT3(Value));
    return Error::success();
  }
  case Thumb_MovwPrelNC: {
    uint16_t Value = ((TargetAddress + Addend - FixupAddress) & 0xffff);
    writeImmediate<Thumb_MovwPrelNC>(R, encodeImmMovtT1MovwT3(Value));
    return Error::success();
  }
  case Thumb_MovtPrel: {
    uint16_t Value = (((TargetAddress + Addend - FixupAddress) >> 16) & 0xffff);
    writeImmediate<Thumb_MovtPrel>(R, encodeImmMovtT1MovwT3(Value));
    return Error::success();
  }

  default:
    return make_error<JITLinkError>(
        "In graph " + G.getName() + ", section " + B.getSection().getName() +
        " encountered unfixable aarch32 edge kind " +
        G.getEdgeKindName(E.getKind()));
  }
}

const uint8_t GOTEntryInit[] = {
    0x00,
    0x00,
    0x00,
    0x00,
};

/// Create a new node in the link-graph for the given pointer value.
template <size_t Size>
static Block &allocPointer(LinkGraph &G, Section &S,
                           const uint8_t (&Content)[Size]) {
  static_assert(Size == 4, "Pointers are 32-bit");
  constexpr uint64_t Alignment = 4;
  ArrayRef<char> Init(reinterpret_cast<const char *>(Content), Size);
  return G.createContentBlock(S, Init, orc::ExecutorAddr(), Alignment, 0);
}

Symbol &GOTBuilder::createEntry(LinkGraph &G, Symbol &Target) {
  if (!GOTSection)
    GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read);
  Block &B = allocPointer(G, *GOTSection, GOTEntryInit);
  constexpr int64_t GOTEntryAddend = 0;
  B.addEdge(Data_Pointer32, 0, Target, GOTEntryAddend);
  return G.addAnonymousSymbol(B, 0, B.getSize(), false, false);
}

bool GOTBuilder::visitEdge(LinkGraph &G, Block *B, Edge &E) {
  Edge::Kind KindToSet = Edge::Invalid;
  switch (E.getKind()) {
  case aarch32::Data_RequestGOTAndTransformToDelta32: {
    KindToSet = aarch32::Data_Delta32;
    break;
  }
  default:
    return false;
  }
  LLVM_DEBUG(dbgs() << "  Transforming " << G.getEdgeKindName(E.getKind())
                    << " edge at " << B->getFixupAddress(E) << " ("
                    << B->getAddress() << " + "
                    << formatv("{0:x}", E.getOffset()) << ") into "
                    << G.getEdgeKindName(KindToSet) << "\n");
  E.setKind(KindToSet);
  E.setTarget(getEntryForTarget(G, E.getTarget()));
  return true;
}

const uint8_t ArmThumbv5LdrPc[] = {
    0x78, 0x47,             // bx pc
    0xfd, 0xe7,             // b #-6 ; Arm recommended sequence to follow bx pc
    0x04, 0xf0, 0x1f, 0xe5, // ldr pc, [pc,#-4] ; L1
    0x00, 0x00, 0x00, 0x00, // L1: .word S
};

const uint8_t Armv7ABS[] = {
    0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000     ; lower 16-bit
    0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000     ; upper 16-bit
    0x1c, 0xff, 0x2f, 0xe1  // bx   r12
};

const uint8_t Thumbv7ABS[] = {
    0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000    ; lower 16-bit
    0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000    ; upper 16-bit
    0x60, 0x47              // bx   r12
};

/// Create a new node in the link-graph for the given stub template.
template <size_t Size>
static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) {
  constexpr uint64_t Alignment = 4;
  ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);
  return G.createContentBlock(S, Template, orc::ExecutorAddr(), Alignment, 0);
}

static Block &createStubPrev7(LinkGraph &G, Section &S, Symbol &Target) {
  Block &B = allocStub(G, S, ArmThumbv5LdrPc);
  B.addEdge(Data_Pointer32, 8, Target, 0);
  return B;
}

static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) {
  Block &B = allocStub(G, S, Thumbv7ABS);
  B.addEdge(Thumb_MovwAbsNC, 0, Target, 0);
  B.addEdge(Thumb_MovtAbs, 4, Target, 0);

  [[maybe_unused]] const char *StubPtr = B.getContent().data();
  [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(12);
  assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&
         checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&
         "Linker generated stubs may only corrupt register r12 (IP)");
  return B;
}

static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) {
  Block &B = allocStub(G, S, Armv7ABS);
  B.addEdge(Arm_MovwAbsNC, 0, Target, 0);
  B.addEdge(Arm_MovtAbs, 4, Target, 0);

  [[maybe_unused]] const char *StubPtr = B.getContent().data();
  [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(12);
  assert(checkRegister<Arm_MovwAbsNC>(StubPtr, Reg12) &&
         checkRegister<Arm_MovtAbs>(StubPtr + 4, Reg12) &&
         "Linker generated stubs may only corrupt register r12 (IP)");
  return B;
}

static bool needsStub(const Edge &E) {
  Symbol &Target = E.getTarget();

  // Create stubs for external branch targets.
  if (!Target.isDefined()) {
    switch (E.getKind()) {
    case Arm_Call:
    case Arm_Jump24:
    case Thumb_Call:
    case Thumb_Jump24:
      return true;
    default:
      return false;
    }
  }

  // For local targets, create interworking stubs if we switch Arm/Thumb with an
  // instruction that cannot switch the instruction set state natively.
  bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol;
  switch (E.getKind()) {
  case Arm_Jump24:
    return TargetIsThumb; // Branch to Thumb needs interworking stub
  case Thumb_Jump24:
    return !TargetIsThumb; // Branch to Arm needs interworking stub
  default:
    break;
  }

  return false;
}

// The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
// for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
// entrypoint at offset 4. Arm branches always use that one.
Symbol *StubsManager_prev7::getOrCreateSlotEntrypoint(LinkGraph &G,
                                                      StubMapEntry &Slot,
                                                      bool Thumb) {
  constexpr orc::ExecutorAddrDiff ThumbEntrypointOffset = 0;
  constexpr orc::ExecutorAddrDiff ArmEntrypointOffset = 4;
  if (Thumb && !Slot.ThumbEntry) {
    Slot.ThumbEntry =
        &G.addAnonymousSymbol(*Slot.B, ThumbEntrypointOffset, 4, true, false);
    Slot.ThumbEntry->setTargetFlags(ThumbSymbol);
  }
  if (!Thumb && !Slot.ArmEntry)
    Slot.ArmEntry =
        &G.addAnonymousSymbol(*Slot.B, ArmEntrypointOffset, 8, true, false);
  return Thumb ? Slot.ThumbEntry : Slot.ArmEntry;
}

bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
  if (!needsStub(E))
    return false;

  Symbol &Target = E.getTarget();
  assert(Target.hasName() && "Edge cannot point to anonymous target");
  auto [Slot, NewStub] = getStubMapSlot(*Target.getName());

  if (NewStub) {
    if (!StubsSection)
      StubsSection = &G.createSection(getSectionName(),
                                      orc::MemProt::Read | orc::MemProt::Exec);
    LLVM_DEBUG({
      dbgs() << "    Created stub entry for " << Target.getName() << " in "
             << StubsSection->getName() << "\n";
    });
    Slot->B = &createStubPrev7(G, *StubsSection, Target);
  }

  // The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
  // for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
  // entrypoint at offset 4. Arm branches always use that one.
  bool UseThumb = E.getKind() == Thumb_Jump24;
  Symbol *StubEntrypoint = getOrCreateSlotEntrypoint(G, *Slot, UseThumb);

  LLVM_DEBUG({
    dbgs() << "    Using " << (UseThumb ? "Thumb" : "Arm") << " entrypoint "
           << *StubEntrypoint << " in "
           << StubEntrypoint->getSection().getName() << "\n";
  });

  E.setTarget(*StubEntrypoint);
  return true;
}

bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
  if (!needsStub(E))
    return false;

  // Stub Arm/Thumb follows instruction set state at relocation site.
  // TODO: We may reduce them at relaxation time and reuse freed slots.
  bool MakeThumb = (E.getKind() > LastArmRelocation);
  LLVM_DEBUG(dbgs() << "  Preparing " << (MakeThumb ? "Thumb" : "Arm")
                    << " stub for " << G.getEdgeKindName(E.getKind())
                    << " edge at " << B->getFixupAddress(E) << " ("
                    << B->getAddress() << " + "
                    << formatv("{0:x}", E.getOffset()) << ")\n");

  Symbol &Target = E.getTarget();
  assert(Target.hasName() && "Edge cannot point to anonymous target");
  Symbol *&StubSymbol = getStubSymbolSlot(*Target.getName(), MakeThumb);

  if (!StubSymbol) {
    if (!StubsSection)
      StubsSection = &G.createSection(getSectionName(),
                                      orc::MemProt::Read | orc::MemProt::Exec);
    Block &B = MakeThumb ? createStubThumbv7(G, *StubsSection, Target)
                         : createStubArmv7(G, *StubsSection, Target);
    StubSymbol = &G.addAnonymousSymbol(B, 0, B.getSize(), true, false);
    if (MakeThumb)
      StubSymbol->setTargetFlags(ThumbSymbol);

    LLVM_DEBUG({
      dbgs() << "    Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for "
             << Target.getName() << " in " << StubsSection->getName() << ": "
             << *StubSymbol << "\n";
    });
  }

  assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) &&
         "Instruction set states of stub and relocation site should be equal");
  LLVM_DEBUG({
    dbgs() << "    Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "
           << *StubSymbol << " in " << StubSymbol->getSection().getName()
           << "\n";
  });

  E.setTarget(*StubSymbol);
  return true;
}

const char *getEdgeKindName(Edge::Kind K) {
#define KIND_NAME_CASE(K)                                                      \
  case K:                                                                      \
    return #K;

  switch (K) {
    KIND_NAME_CASE(Data_Delta32)
    KIND_NAME_CASE(Data_Pointer32)
    KIND_NAME_CASE(Data_PRel31)
    KIND_NAME_CASE(Data_RequestGOTAndTransformToDelta32)
    KIND_NAME_CASE(Arm_Call)
    KIND_NAME_CASE(Arm_Jump24)
    KIND_NAME_CASE(Arm_MovwAbsNC)
    KIND_NAME_CASE(Arm_MovtAbs)
    KIND_NAME_CASE(Thumb_Call)
    KIND_NAME_CASE(Thumb_Jump24)
    KIND_NAME_CASE(Thumb_MovwAbsNC)
    KIND_NAME_CASE(Thumb_MovtAbs)
    KIND_NAME_CASE(Thumb_MovwPrelNC)
    KIND_NAME_CASE(Thumb_MovtPrel)
    KIND_NAME_CASE(None)
  default:
    return getGenericEdgeKindName(K);
  }
#undef KIND_NAME_CASE
}

const char *getCPUArchName(ARMBuildAttrs::CPUArch K) {
#define CPUARCH_NAME_CASE(K)                                                   \
  case K:                                                                      \
    return #K;

  using namespace ARMBuildAttrs;
  switch (K) {
    CPUARCH_NAME_CASE(Pre_v4)
    CPUARCH_NAME_CASE(v4)
    CPUARCH_NAME_CASE(v4T)
    CPUARCH_NAME_CASE(v5T)
    CPUARCH_NAME_CASE(v5TE)
    CPUARCH_NAME_CASE(v5TEJ)
    CPUARCH_NAME_CASE(v6)
    CPUARCH_NAME_CASE(v6KZ)
    CPUARCH_NAME_CASE(v6T2)
    CPUARCH_NAME_CASE(v6K)
    CPUARCH_NAME_CASE(v7)
    CPUARCH_NAME_CASE(v6_M)
    CPUARCH_NAME_CASE(v6S_M)
    CPUARCH_NAME_CASE(v7E_M)
    CPUARCH_NAME_CASE(v8_A)
    CPUARCH_NAME_CASE(v8_R)
    CPUARCH_NAME_CASE(v8_M_Base)
    CPUARCH_NAME_CASE(v8_M_Main)
    CPUARCH_NAME_CASE(v8_1_M_Main)
    CPUARCH_NAME_CASE(v9_A)
  }
  llvm_unreachable("Missing CPUArch in switch?");
#undef CPUARCH_NAME_CASE
}

} // namespace aarch32
} // namespace jitlink
} // namespace llvm