//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file describes the BPF instructions in TableGen format.
//
//===----------------------------------------------------------------------===//

include "BPFInstrFormats.td"

// Instruction Operands and Patterns

// These are target-independent nodes, but have target-specific formats.
def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>,
                                          SDTCisVT<1, iPTR>]>;
def SDT_BPFCallSeqEnd   : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>;
def SDT_BPFCall         : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>;
def SDT_BPFSetFlag      : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>;
def SDT_BPFSelectCC     : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>,
                                               SDTCisSameAs<0, 4>,
                                               SDTCisSameAs<4, 5>]>;
def SDT_BPFBrCC         : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>,
                                               SDTCisVT<3, OtherVT>]>;
def SDT_BPFWrapper      : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
                                               SDTCisPtrTy<0>]>;
def SDT_BPFMEMCPY       : SDTypeProfile<0, 4, [SDTCisVT<0, i64>,
                                               SDTCisVT<1, i64>,
                                               SDTCisVT<2, i64>,
                                               SDTCisVT<3, i64>]>;

def BPFcall         : SDNode<"BPFISD::CALL", SDT_BPFCall,
                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                              SDNPVariadic]>;
def BPFretglue      : SDNode<"BPFISD::RET_GLUE", SDTNone,
                             [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart,
                             [SDNPHasChain, SDNPOutGlue]>;
def BPFcallseq_end  : SDNode<"ISD::CALLSEQ_END",   SDT_BPFCallSeqEnd,
                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
def BPFbrcc         : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC, [SDNPHasChain]>;

def SDT_BPFLoadStackArg : SDTypeProfile<1, 1, [
  SDTCisVT<0, i64>,  // result value
  SDTCisVT<1, i64>   // operand: offset
]>;
def SDT_BPFStoreStackArg : SDTypeProfile<0, 2, [
  SDTCisVT<0, i64>,  // operand 0: offset
  SDTCisVT<1, i64>   // operand 1: stored value
]>;

def BPFload_stack_arg : SDNode<"BPFISD::LOAD_STACK_ARG",
                               SDT_BPFLoadStackArg,
                               [SDNPHasChain, SDNPMayLoad]>;
def BPFstore_stack_arg : SDNode<"BPFISD::STORE_STACK_ARG",
                                SDT_BPFStoreStackArg,
                                [SDNPHasChain, SDNPMayStore]>;

def BPFselectcc     : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC>;
def BPFWrapper      : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>;
def BPFmemcpy       : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY,
                             [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">;
def BPFIsBigEndian    : Predicate<"!Subtarget->isLittleEndian()">;
def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">;
def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">;
def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">;
def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">;
def BPFHasBswap : Predicate<"Subtarget->hasBswap()">;
def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">;
def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">;
def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">;
def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">;
def BPFHasLoadAcqStoreRel : Predicate<"Subtarget->hasLoadAcqStoreRel()">;
def BPFHasGotox : Predicate<"Subtarget->hasGotox()">;

class ImmediateAsmOperand<string name> : AsmOperandClass {
  let Name = name;
  let RenderMethod = "addImmOperands";
  let DiagnosticType = !strconcat("Invalid", name);
}

def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">;

def brtarget : Operand<OtherVT> {
  let PrintMethod = "printBrTargetOperand";
  let ParserMatchClass = ImmediateAsmOperand<"BrTarget">;
}
def calltarget : Operand<i64>;

def u64imm   : Operand<i64> {
  let PrintMethod = "printImm64Operand";
}

def s16imm : Operand<i16> {
  let ParserMatchClass = SImm16AsmOperand;
}

def gpr_or_imm : Operand<i64>;

def i64immSExt32 : PatLeaf<(i64 imm),
                [{return isInt<32>(N->getSExtValue()); }]>;
def i32immSExt32 : PatLeaf<(i32 imm),
                [{return isInt<32>(N->getSExtValue()); }]>;
def i64immZExt32 : PatLeaf<(i64 imm),
                [{return isUInt<32>(N->getZExtValue()); }]>;

def imm_to_i64 : SDNodeXForm<timm, [{
  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64);
}]>;

// Addressing modes.
def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>;
def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>;

// Address operands
def MEMri : Operand<i64> {
  let PrintMethod = "printMemOperand";
  let EncoderMethod = "getMemoryOpValue";
  let DecoderMethod = "decodeMemoryOpValue";
  let MIOperandInfo = (ops GPR, s16imm);
}

// Conditional code predicates - used for pattern matching for jump instructions
def BPF_CC_EQ  : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETEQ);}]>;
def BPF_CC_NE  : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETNE);}]>;
def BPF_CC_GE  : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETGE);}]>;
def BPF_CC_GT  : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETGT);}]>;
def BPF_CC_GTU : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETUGT);}]>;
def BPF_CC_GEU : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETUGE);}]>;
def BPF_CC_LE  : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETLE);}]>;
def BPF_CC_LT  : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETLT);}]>;
def BPF_CC_LTU : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETULT);}]>;
def BPF_CC_LEU : PatLeaf<(i64 imm),
                         [{return (N->getZExtValue() == ISD::SETULE);}]>;
def BPF_CC_EQ_32  : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETEQ);}]>;
def BPF_CC_NE_32  : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETNE);}]>;
def BPF_CC_GE_32  : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETGE);}]>;
def BPF_CC_GT_32  : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETGT);}]>;
def BPF_CC_GTU_32 : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETUGT);}]>;
def BPF_CC_GEU_32 : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETUGE);}]>;
def BPF_CC_LE_32  : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETLE);}]>;
def BPF_CC_LT_32  : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETLT);}]>;
def BPF_CC_LTU_32 : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETULT);}]>;
def BPF_CC_LEU_32 : PatLeaf<(i32 imm),
                         [{return (N->getZExtValue() == ISD::SETULE);}]>;
def NoCond : PatLeaf<(vt)> {}

// For arithmetic and jump instructions the 8-bit 'code'
// field is divided into three parts:
//
//  +----------------+--------+--------------------+
//  |   4 bits       |  1 bit |   3 bits           |
//  | operation code | source | instruction class  |
//  +----------------+--------+--------------------+
//  (MSB)                                      (LSB)
class TYPE_ALU_JMP<bits<4> op, bits<1> srctype,
                   dag outs, dag ins, string asmstr, list<dag> pattern>
  : InstBPF<outs, ins, asmstr, pattern> {

  let Inst{63-60} = op;
  let Inst{59} = srctype;
}

//For load and store instructions the 8-bit 'code' field is divided as:
//
//  +--------+--------+-------------------+
//  | 3 bits | 2 bits |   3 bits          |
//  |  mode  |  size  | instruction class |
//  +--------+--------+-------------------+
//  (MSB)                             (LSB)
class TYPE_LD_ST<bits<3> mode, bits<2> size,
                 dag outs, dag ins, string asmstr, list<dag> pattern>
  : InstBPF<outs, ins, asmstr, pattern> {

  let Inst{63-61} = mode;
  let Inst{60-59} = size;
}

// jump instructions
class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
                   (outs),
                   (ins GPR:$dst, GPR:$src, brtarget:$BrDst),
                   "if $dst "#OpcodeStr#" $src goto $BrDst",
                   [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> {
  bits<4> dst;
  bits<4> src;
  bits<16> BrDst;

  let Inst{55-52} = src;
  let Inst{51-48} = dst;
  let Inst{47-32} = BrDst;
  let BPFClass = BPF_JMP;
}

class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
                   (outs),
                   (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst),
                   "if $dst "#OpcodeStr#" $imm goto $BrDst",
                   [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> {
  bits<4> dst;
  bits<16> BrDst;
  bits<32> imm;

  let Inst{51-48} = dst;
  let Inst{47-32} = BrDst;
  let Inst{31-0} = imm;
  let BPFClass = BPF_JMP;
}

class JMP_IND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
                   (outs),
                   (ins GPR:$dst),
                   !strconcat(OpcodeStr, " $dst"),
                   Pattern> {
  bits<4> dst;

  let Inst{51-48} = dst;
  let BPFClass = BPF_JMP;
}

class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
                   (outs),
                   (ins brtarget:$BrDst),
                   !strconcat(OpcodeStr, " $BrDst"),
                   Pattern> {
  bits<16> BrDst;

  let Inst{47-32} = BrDst;
  let BPFClass = BPF_JMP;
}

class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
                   (outs),
                   (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst),
                   "if $dst "#OpcodeStr#" $src goto $BrDst",
                   [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> {
  bits<4> dst;
  bits<4> src;
  bits<16> BrDst;

  let Inst{55-52} = src;
  let Inst{51-48} = dst;
  let Inst{47-32} = BrDst;
  let BPFClass = BPF_JMP32;
}

class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
                   (outs),
                   (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst),
                   "if $dst "#OpcodeStr#" $imm goto $BrDst",
                   [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> {
  bits<4> dst;
  bits<16> BrDst;
  bits<32> imm;

  let Inst{51-48} = dst;
  let Inst{47-32} = BrDst;
  let Inst{31-0} = imm;
  let BPFClass = BPF_JMP32;
}

multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> {
  def _rr : JMP_RR<Opc, OpcodeStr, Cond>;
  def _ri : JMP_RI<Opc, OpcodeStr, Cond>;
  def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>;
  def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>;
}

let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in {
// cmp+goto instructions
defm JEQ  : J<BPF_JEQ, "==",  BPF_CC_EQ, BPF_CC_EQ_32>;
defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>;
defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>;
defm JNE  : J<BPF_JNE, "!=",  BPF_CC_NE, BPF_CC_NE_32>;
defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>;
defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>;
defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>;
defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>;
defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;
def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>;

let Predicates = [BPFHasGotox] in {
  let isIndirectBranch = 1, isBarrier = 1 in {
    def JX : JMP_IND<BPF_JA, "gotox", [(brind i64:$dst)]>;
  }
}
}

let hasSideEffects = 1, mayLoad = 1 in {
  def LOAD_STACK_ARG_PSEUDO
    : Pseudo<(outs GPR:$dst), (ins s16imm:$off),
             "load_stack_arg\t$dst, $off", []>;
}

let hasSideEffects = 1, mayStore = 1 in {
  def STORE_STACK_ARG_PSEUDO
    : Pseudo<(outs), (ins s16imm:$off, GPR:$src),
             "store_stack_arg\t$off, $src", []>;

  def STORE_STACK_ARG_IMM_PSEUDO
    : Pseudo<(outs), (ins s16imm:$off, i32imm:$val),
             "store_stack_arg_imm\t$off, $val", []>;
}

// ALU instructions
class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off,
             dag outs, dag ins, string asmstr, list<dag> pattern>
    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> {
  bits<4> dst;
  bits<32> imm;

  let Inst{51-48} = dst;
  let Inst{47-32} = off;
  let Inst{31-0} = imm;
  let BPFClass = Class;
}

class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off,
             dag outs, dag ins, string asmstr, list<dag> pattern>
    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> {
  bits<4> dst;
  bits<4> src;

  let Inst{55-52} = src;
  let Inst{51-48} = dst;
  let Inst{47-32} = off;
  let BPFClass = Class;
}

multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> {
  def _rr : ALU_RR<BPF_ALU64, Opc, off,
                   (outs GPR:$dst),
                   (ins GPR:$src2, GPR:$src),
                   "$dst "#OpcodeStr#" $src",
                   [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>;
  def _ri : ALU_RI<BPF_ALU64, Opc, off,
                   (outs GPR:$dst),
                   (ins GPR:$src2, i64imm:$imm),
                   "$dst "#OpcodeStr#" $imm",
                   [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>;
  def _rr_32 : ALU_RR<BPF_ALU, Opc, off,
                   (outs GPR32:$dst),
                   (ins GPR32:$src2, GPR32:$src),
                   "$dst "#OpcodeStr#" $src",
                   [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>;
  def _ri_32 : ALU_RI<BPF_ALU, Opc, off,
                   (outs GPR32:$dst),
                   (ins GPR32:$src2, i32imm:$imm),
                   "$dst "#OpcodeStr#" $imm",
                   [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>;
}

let Constraints = "$dst = $src2" in {
let isAsCheapAsAMove = 1 in {
  defm ADD : ALU<BPF_ADD, 0, "+=", add>;
  defm SUB : ALU<BPF_SUB, 0, "-=", sub>;
  defm OR  : ALU<BPF_OR, 0, "|=", or>;
  defm AND : ALU<BPF_AND, 0, "&=", and>;
  defm SLL : ALU<BPF_LSH, 0, "<<=", shl>;
  defm SRL : ALU<BPF_RSH, 0, ">>=", srl>;
  defm XOR : ALU<BPF_XOR, 0, "^=", xor>;
  defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>;
}
  defm MUL : ALU<BPF_MUL, 0, "*=", mul>;
  defm DIV : ALU<BPF_DIV, 0, "/=", udiv>;
  defm MOD : ALU<BPF_MOD, 0, "%=", urem>;

  let Predicates = [BPFHasSdivSmod] in {
    defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>;
    defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>;
  }
}

class NEG_RR<BPFOpClass Class, BPFArithOp Opc,
             dag outs, dag ins, string asmstr, list<dag> pattern>
    : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> {
  bits<4> dst;

  let Inst{51-48} = dst;
  let BPFClass = Class;
}

let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in {
  def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src),
                     "$dst = -$src",
                     [(set GPR:$dst, (ineg i64:$src))]>;
  def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src),
                     "$dst = -$src",
                     [(set GPR32:$dst, (ineg i32:$src))]>;
}

class LD_IMM64<bits<4> Pseudo, string OpcodeStr>
    : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
                 (outs GPR:$dst),
                 (ins u64imm:$imm),
                 "$dst "#OpcodeStr#" ${imm} ll",
                 [(set GPR:$dst, (i64 imm:$imm))]> {

  bits<4> dst;
  bits<64> imm;

  let Inst{51-48} = dst;
  let Inst{55-52} = Pseudo;
  let Inst{47-32} = 0;
  let Inst{31-0} = imm{31-0};
  let BPFClass = BPF_LD;
}

let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
def LD_imm64 : LD_IMM64<0, "=">;
def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0,
                    (outs GPR:$dst),
                    (ins GPR:$src),
                    "$dst = $src",
                    []>;
def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0,
                    (outs GPR:$dst),
                    (ins i64imm:$imm),
                    "$dst = $imm",
                    [(set GPR:$dst, (i64 i64immSExt32:$imm))]>;
def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0,
                    (outs GPR32:$dst),
                    (ins GPR32:$src),
                    "$dst = $src",
                    []>;
def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0,
                    (outs GPR32:$dst),
                    (ins i32imm:$imm),
                    "$dst = $imm",
                    [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>;

let Predicates = [BPFHasMovsx] in {
  def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8,
                      (outs GPR:$dst), (ins GPR:$src),
                      "$dst = (s8)$src",
                      [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>;
  def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16,
                      (outs GPR:$dst), (ins GPR:$src),
                      "$dst = (s16)$src",
                      [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>;
  def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32,
                      (outs GPR:$dst), (ins GPR:$src),
                      "$dst = (s32)$src",
                      [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>;
  def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8,
                      (outs GPR32:$dst), (ins GPR32:$src),
                      "$dst = (s8)$src",
                      [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>;
  def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16,
                      (outs GPR32:$dst), (ins GPR32:$src),
                      "$dst = (s16)$src",
                      [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>;
}
}

def ADDR_SPACE_CAST
    : ALU_RR<BPF_ALU64, BPF_MOV, 64,
             (outs GPR:$dst),
             (ins GPR:$src, i64imm:$dst_as, i64imm:$src_as),
             "$dst = addr_space_cast($src, $dst_as, $src_as)",
             []> {
  bits<64> dst_as;
  bits<64> src_as;

  let Inst{47-32} = 1;
  let Inst{31-16} = dst_as{15-0};
  let Inst{15-0} = src_as{15-0};
}

def SrcAddrSpace : SDNodeXForm<addrspacecast, [{
  return CurDAG->getTargetConstant(
    cast<AddrSpaceCastSDNode>(N)->getSrcAddressSpace(),
    SDLoc(N), MVT::i64);
}]>;

def DstAddrSpace : SDNodeXForm<addrspacecast, [{
  return CurDAG->getTargetConstant(
    cast<AddrSpaceCastSDNode>(N)->getDestAddressSpace(),
    SDLoc(N), MVT::i64);
}]>;

def : Pat<(addrspacecast:$this GPR:$src),
          (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>;

def FI_ri
    : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
                 (outs GPR:$dst),
                 (ins MEMri:$addr),
                 "lea\t$dst, $addr",
                 [(set i64:$dst, FIri:$addr)]> {
  // This is a tentative instruction, and will be replaced
  // with MOV_rr and ADD_ri in PEI phase
  let Inst{51-48} = 0;
  let Inst{55-52} = 2;
  let Inst{47-32} = 0;
  let Inst{31-0} = 0;
  let BPFClass = BPF_LD;
  bit isPseudo = true;
}

def LD_pseudo
    : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
                 (outs GPR:$dst),
                 (ins i64imm:$pseudo, u64imm:$imm),
                 "ld_pseudo\t$dst, $pseudo, $imm",
                 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> {

  bits<4> dst;
  bits<64> imm;
  bits<4> pseudo;

  let Inst{51-48} = dst;
  let Inst{55-52} = pseudo;
  let Inst{47-32} = 0;
  let Inst{31-0} = imm{31-0};
  let BPFClass = BPF_LD;
}

// STORE instructions
class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern>
    : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
                 (outs),
                 (ins GPR:$src, MEMri:$addr),
                 "*("#OpcodeStr#" *)($addr) = $src",
                 Pattern> {
  bits<4> src;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = src;
  let Inst{47-32} = addr{15-0}; // offset
  let BPFClass = BPF_STX;
}

class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>
    : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>;

let Predicates = [BPFNoALU32] in {
  def STW : STOREi64<BPF_W, "u32", truncstorei32>;
  def STH : STOREi64<BPF_H, "u16", truncstorei16>;
  def STB : STOREi64<BPF_B, "u8", truncstorei8>;
}
def STD : STOREi64<BPF_DW, "u64", store>;

class STORE_imm<BPFWidthModifer SizeOp,
                string OpcodeStr, dag Pattern>
    : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
                 (outs),
                 (ins i64imm:$imm, MEMri:$addr),
                 "*("#OpcodeStr#" *)($addr) = $imm",
                 [Pattern]> {
  bits<20> addr;
  bits<32> imm;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{47-32} = addr{15-0};  // offset
  let Inst{31-0} = imm;
  let BPFClass = BPF_ST;
}

let Predicates = [BPFHasStoreImm] in {
  // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for
  // value stored to memory:
  // - it is fine to generate such write when immediate is -1
  // - it is incorrect to generate such write when immediate is
  //   +0xffff_ffff.
  //
  // In the latter case two instructions would be generated instead of
  // one BPF_ST:
  //   rA = 0xffffffff ll     ; LD_imm64
  //   *(u64 *)(rB + 0) = rA  ; STX
  //
  // For BPF_{B,H,W} the size of value stored matches size of the immediate.
  def STD_imm : STORE_imm<BPF_DW, "u64", (store         (i64 i64immSExt32:$imm), ADDRri:$addr)>;
  def STW_imm : STORE_imm<BPF_W,  "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>;
  def STH_imm : STORE_imm<BPF_H,  "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>;
  def STB_imm : STORE_imm<BPF_B,  "u8",  (truncstorei8  (i64 i64immZExt32:$imm), ADDRri:$addr)>;
}

let Predicates = [BPFHasALU32, BPFHasStoreImm] in {
  def : Pat<(store (i32 imm:$src), ADDRri:$dst),
            (STW_imm (imm_to_i64 $src), ADDRri:$dst)>;
  def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst),
            (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>;
  def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst),
            (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>;
}

class STORE_RELEASE<BPFWidthModifer SizeOp, string OpcodeStr, RegisterClass RegTp>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs),
                 (ins RegTp:$src, MEMri:$addr),
                 "store_release(("#OpcodeStr#" *)($addr), $src)",
                 []> {
  bits<4> src;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = src;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{8-4} = BPF_STORE_REL.Value;
  let BPFClass = BPF_STX;
}

class STORE_RELEASEi64<BPFWidthModifer Opc, string OpcodeStr>
    : STORE_RELEASE<Opc, OpcodeStr, GPR>;

class relaxed_store<PatFrag base>
  : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> {
  let IsAtomic = 1;
  let IsAtomicOrderingReleaseOrStronger = 0;
}

class releasing_store<PatFrag base>
  : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> {
  let IsAtomic = 1;
  let IsAtomicOrderingRelease = 1;
}

let Predicates = [BPFHasLoadAcqStoreRel] in {
  def STDREL : STORE_RELEASEi64<BPF_DW, "u64">;

  foreach P = [[relaxed_store<atomic_store_64>, STD],
               [releasing_store<atomic_store_64>, STDREL],
              ] in {
    def : Pat<(P[0] GPR:$val, ADDRri:$addr), (P[1] GPR:$val, ADDRri:$addr)>;
  }
}

// LOAD instructions
class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern>
    : TYPE_LD_ST<ModOp.Value, SizeOp.Value,
                 (outs GPR:$dst),
                 (ins MEMri:$addr),
                 "$dst = *("#OpcodeStr#" *)($addr)",
                 Pattern> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = dst;
  let Inst{55-52} = addr{19-16};
  let Inst{47-32} = addr{15-0};
  let BPFClass = BPF_LDX;
}

class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode>
    : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>;

let isCodeGenOnly = 1 in {
  class CORE_LD<RegisterClass RegClass, string Sz>
                : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,
                             (outs RegClass:$dst),
                             (ins u64imm:$opcode, GPR:$src, u64imm:$offset),
                             "$dst = core_ld"#Sz#"($opcode, $src, $offset)",
                             []>;
  def CORE_LD64 : CORE_LD<GPR, "64">;
  def CORE_LD32 : CORE_LD<GPR32, "32">;
  def CORE_ST   : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,
                             (outs),
                             (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset),
                             "core_st($src, $opcode, $ptr, $offset)",
                             []>;
  let Constraints = "$dst = $src" in {
    def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0,
                             (outs GPR:$dst),
                             (ins u64imm:$opcode, GPR:$src, u64imm:$offset),
                             "$dst = core_shift($opcode, $src, $offset)",
                             []>;
  }
}

let Predicates = [BPFNoALU32] in {
  def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>;
  def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>;
  def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>;
}

let Predicates = [BPFHasLdsx] in {
  def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>;
  def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>;
  def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8",  sextloadi8>;
}

def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>;

class LOAD_ACQUIRE<BPFWidthModifer SizeOp, string OpcodeStr, RegisterClass RegTp>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs RegTp:$dst),
                 (ins MEMri:$addr),
                 "$dst = load_acquire(("#OpcodeStr#" *)($addr))",
                 []> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = dst;
  let Inst{55-52} = addr{19-16}; // base reg
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{8-4} = BPF_LOAD_ACQ.Value;
  let BPFClass = BPF_STX;
}

class LOAD_ACQUIREi64<BPFWidthModifer SizeOp, string OpcodeStr>
    : LOAD_ACQUIRE<SizeOp, OpcodeStr, GPR>;

class relaxed_load<PatFrags base>
    : PatFrag<(ops node:$ptr), (base node:$ptr)> {
  let IsAtomic = 1;
  let IsAtomicOrderingAcquireOrStronger = 0;
}

class acquiring_load<PatFrags base>
    : PatFrag<(ops node:$ptr), (base node:$ptr)> {
  let IsAtomic = 1;
  let IsAtomicOrderingAcquire = 1;
}

let Predicates = [BPFHasLoadAcqStoreRel] in {
  def LDDACQ : LOAD_ACQUIREi64<BPF_DW, "u64">;

  foreach P = [[relaxed_load<atomic_load_nonext_64>, LDD],
               [acquiring_load<atomic_load_nonext_64>, LDDACQ],
              ] in {
    def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;
  }
}

class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
                   (outs),
                   (ins brtarget:$BrDst),
                   !strconcat(OpcodeStr, " $BrDst"),
                   Pattern> {
  bits<16> BrDst;

  let Inst{47-32} = BrDst;
  let BPFClass = BPF_JMP;
}

class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
                   (outs),
                   (ins brtarget:$BrDst),
                   !strconcat(OpcodeStr, " $BrDst"),
                   Pattern> {
  bits<32> BrDst;

  let Inst{31-0} = BrDst;
  let BPFClass = BPF_JMP32;
}

class CALL<string OpcodeStr>
    : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value,
                   (outs),
                   (ins calltarget:$BrDst),
                   !strconcat(OpcodeStr, " $BrDst"),
                   []> {
  bits<32> BrDst;

  let Inst{31-0} = BrDst;
  let BPFClass = BPF_JMP;
}

class CALLX<string OpcodeStr>
    : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value,
                   (outs),
                   (ins GPR:$BrDst),
                   !strconcat(OpcodeStr, " $BrDst"),
                   []> {
  bits<4> BrDst;

  let Inst{51-48} = BrDst;
  let BPFClass = BPF_JMP;
}

// Jump always
let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in {
  def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>;
  def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>;
}

// Jump and link
let isCall=1, hasDelaySlot=0, Uses = [R11] in {
  def JAL  : CALL<"call">;
  def JALX  : CALLX<"callx">;
}

class NOP_I<string OpcodeStr>
    : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value,
                   (outs),
                   (ins i32imm:$imm),
                   !strconcat(OpcodeStr, "\t$imm"),
                   []> {
  // mov r0, r0 == nop
  let Inst{55-52} = 0;
  let Inst{51-48} = 0;
  let BPFClass = BPF_ALU64;
}

let hasSideEffects = 0, isCodeGenOnly = 1 in
  def NOP : NOP_I<"nop">;

class RET<string OpcodeStr>
    : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value,
                   (outs),
                   (ins),
                   !strconcat(OpcodeStr, ""),
                   [(BPFretglue)]> {
  let Inst{31-0} = 0;
  let BPFClass = BPF_JMP;
}

let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1,
    isNotDuplicable = 1 in {
  def RET : RET<"exit">;
}

// ADJCALLSTACKDOWN/UP pseudo insns
let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in {
def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2),
                              "#ADJCALLSTACKDOWN $amt1 $amt2",
                              [(BPFcallseq_start timm:$amt1, timm:$amt2)]>;
def ADJCALLSTACKUP   : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2),
                              "#ADJCALLSTACKUP $amt1 $amt2",
                              [(BPFcallseq_end timm:$amt1, timm:$amt2)]>;
}

let usesCustomInserter = 1, isCodeGenOnly = 1 in {
  def Select : Pseudo<(outs GPR:$dst),
                      (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i64:$dst,
                       (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>;
  def Select_Ri : Pseudo<(outs GPR:$dst),
                      (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i64:$dst,
                       (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>;
  def Select_64_32 : Pseudo<(outs GPR32:$dst),
                      (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i32:$dst,
                       (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>;
  def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst),
                      (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i32:$dst,
                       (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>;
  def Select_32 : Pseudo<(outs GPR32:$dst),
                      (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i32:$dst,
                       (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>;
  def Select_Ri_32 : Pseudo<(outs GPR32:$dst),
                      (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i32:$dst,
                       (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>;
  def Select_32_64 : Pseudo<(outs GPR:$dst),
                      (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i64:$dst,
                       (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>;
  def Select_Ri_32_64 : Pseudo<(outs GPR:$dst),
                      (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2),
                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
                      [(set i64:$dst,
                       (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>;
}

// load 64-bit global addr into register
def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>;
def : Pat<(BPFWrapper tjumptable:$in), (LD_imm64 tjumptable:$in)>;

// 0xffffFFFF doesn't fit into simm32, optimize common case
def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)),
          (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>;

// Calls
def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>;
def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>;
def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>;
def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>;

// Loads
let Predicates = [BPFNoALU32] in {
  def : Pat<(i64 (extloadi8  ADDRri:$src)), (i64 (LDB ADDRri:$src))>;
  def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>;
  def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>;
}

// Atomic add, and, or, xor
class ATOMIC_NOFETCH<BPFWidthModifer SizeOp, string OpType, RegisterClass RegTp,
                     BPFArithOp Opc, string Opstr>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs RegTp:$dst),
                 (ins MEMri:$addr, RegTp:$val),
                 "lock *(" #OpType# " *)($addr) " #Opstr# "= $val",
                 []> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = dst;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = Opc.Value;
  let BPFClass = BPF_STX;
}

let Constraints = "$dst = $val" in {
  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
    def XADDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_ADD, "+">;
    def XANDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_AND, "&">;
    def XORW32  : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_OR, "|">;
    def XXORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_XOR, "^">;
  }
  def XADDW  : ATOMIC_NOFETCH<BPF_W,  "u32", GPR, BPF_ADD, "+">;
  def XADDD  : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_ADD, "+">;
  def XANDD  : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_AND, "&">;
  def XORD   : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_OR, "|">;
  def XXORD  : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_XOR, "^">;
}

let Predicates = [BPFNoALU32] in {
  def : Pat<(atomic_load_add_i32 ADDRri:$addr, GPR:$val),
            (XADDW ADDRri:$addr, GPR:$val)>;
  def : Pat<(atomic_load_add_i64 ADDRri:$addr, GPR:$val),
            (XADDD ADDRri:$addr, GPR:$val)>;
}

// Atomic Fetch-and-<add, and, or, xor> operations
class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs GPR:$dst),
                 (ins MEMri:$addr, GPR:$val),
                 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
                 []> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = dst;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = Opc.Value;
  let Inst{3-0} = BPF_FETCH.Value;
  let BPFClass = BPF_STX;
}

class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs GPR32:$dst),
                 (ins MEMri:$addr, GPR32:$val),
                 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
                 []> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = dst;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = Opc.Value;
  let Inst{3-0} = BPF_FETCH.Value;
  let BPFClass = BPF_STX;
}

let Constraints = "$dst = $val" in {
  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
    def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add">;
    def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and">;
    def XFORW32  : XFALU32<BPF_W, BPF_OR,  "u32", "or">;
    def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor">;
  }

  let Predicates = [BPFHasALU32] in {
    def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add">;
  }
  def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and">;
  def XFORD  : XFALU64<BPF_DW, BPF_OR,  "u64", "or">;
  def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor">;
}

let Predicates = [BPFHasALU32] in {
    foreach P = [// add
                 [atomic_load_add_i32_monotonic,  XADDW32],
                 [atomic_load_add_i32_acquire,   XFADDW32],
                 [atomic_load_add_i32_release,   XFADDW32],
                 [atomic_load_add_i32_acq_rel,   XFADDW32],
                 [atomic_load_add_i32_seq_cst,   XFADDW32],
                 // and
                 [atomic_load_and_i32_monotonic,  XANDW32],
                 [atomic_load_and_i32_acquire,   XFANDW32],
                 [atomic_load_and_i32_release,   XFANDW32],
                 [atomic_load_and_i32_acq_rel,   XFANDW32],
                 [atomic_load_and_i32_seq_cst,   XFANDW32],
                 // or
                 [atomic_load_or_i32_monotonic,   XORW32],
                 [atomic_load_or_i32_acquire,    XFORW32],
                 [atomic_load_or_i32_release,    XFORW32],
                 [atomic_load_or_i32_acq_rel,    XFORW32],
                 [atomic_load_or_i32_seq_cst,    XFORW32],
                 // xor
                 [atomic_load_xor_i32_monotonic,  XXORW32],
                 [atomic_load_xor_i32_acquire,   XFXORW32],
                 [atomic_load_xor_i32_release,   XFXORW32],
                 [atomic_load_xor_i32_acq_rel,   XFXORW32],
                 [atomic_load_xor_i32_seq_cst,   XFXORW32],
                ] in {
      def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1]  ADDRri:$addr, GPR32:$val)>;
    }

    // atomic_load_sub can be represented as a neg followed
    // by an atomic_load_add.
    foreach P = [[atomic_load_sub_i32_monotonic,  XADDW32],
                 [atomic_load_sub_i32_acquire,   XFADDW32],
                 [atomic_load_sub_i32_release,   XFADDW32],
                 [atomic_load_sub_i32_acq_rel,   XFADDW32],
                 [atomic_load_sub_i32_seq_cst,   XFADDW32],
                ] in {
      def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1]  ADDRri:$addr, (NEG_32 GPR32:$val))>;
    }

    foreach P = [// add
                 [atomic_load_add_i64_monotonic,  XADDD],
                 [atomic_load_add_i64_acquire,   XFADDD],
                 [atomic_load_add_i64_release,   XFADDD],
                 [atomic_load_add_i64_acq_rel,   XFADDD],
                 [atomic_load_add_i64_seq_cst,   XFADDD],
                ] in {
      def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1]  ADDRri:$addr, GPR:$val)>;
    }
}

foreach P = [[atomic_load_sub_i64_monotonic,  XADDD],
             [atomic_load_sub_i64_acquire,   XFADDD],
             [atomic_load_sub_i64_release,   XFADDD],
             [atomic_load_sub_i64_acq_rel,   XFADDD],
             [atomic_load_sub_i64_seq_cst,   XFADDD],
            ] in {
  def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1]  ADDRri:$addr, (NEG_64 GPR:$val))>;
}

// Borrow the idea from X86InstrFragments.td
class binop_no_use<SDPatternOperator operator>
      : PatFrag<(ops node:$A, node:$B),
                (operator node:$A, node:$B),
                [{ return SDValue(N, 0).use_empty(); }]>;

class binop_has_use<SDPatternOperator operator>
      : PatFrag<(ops node:$A, node:$B),
                (operator node:$A, node:$B),
                [{ return !SDValue(N, 0).use_empty(); }]>;

foreach op = [add, and, or, xor] in {
def atomic_load_ # op # _i64_monotonic_nu:
    binop_no_use <!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>;
def atomic_load_ # op # _i64_monotonic_hu:
    binop_has_use<!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>;
}

foreach P = [// and
             [atomic_load_and_i64_monotonic_nu, XANDD],
             [atomic_load_and_i64_monotonic_hu, XFANDD],
             [atomic_load_and_i64_acquire,   XFANDD],
             [atomic_load_and_i64_release,   XFANDD],
             [atomic_load_and_i64_acq_rel,   XFANDD],
             [atomic_load_and_i64_seq_cst,   XFANDD],
             // or
             [atomic_load_or_i64_monotonic_nu, XORD],
             [atomic_load_or_i64_monotonic_hu, XFORD],
             [atomic_load_or_i64_acquire,    XFORD],
             [atomic_load_or_i64_release,    XFORD],
             [atomic_load_or_i64_acq_rel,    XFORD],
             [atomic_load_or_i64_seq_cst,    XFORD],
             // xor
             [atomic_load_xor_i64_monotonic_nu, XXORD],
             [atomic_load_xor_i64_monotonic_hu, XFXORD],
             [atomic_load_xor_i64_acquire,   XFXORD],
             [atomic_load_xor_i64_release,   XFXORD],
             [atomic_load_xor_i64_acq_rel,   XFXORD],
             [atomic_load_xor_i64_seq_cst,   XFXORD],
            ] in {
  def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1]  ADDRri:$addr, GPR:$val)>;
}

// Atomic Exchange
class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs GPR:$dst),
                 (ins MEMri:$addr, GPR:$val),
                 "$dst = xchg_"#OpcodeStr#"($addr, $val)",
                 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = dst;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = BPF_XCHG.Value;
  let Inst{3-0} = BPF_FETCH.Value;
  let BPFClass = BPF_STX;
}

class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs GPR32:$dst),
                 (ins MEMri:$addr, GPR32:$val),
                 "$dst = xchg32_"#OpcodeStr#"($addr, $val)",
                 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = dst;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = BPF_XCHG.Value;
  let Inst{3-0} = BPF_FETCH.Value;
  let BPFClass = BPF_STX;
}

let Constraints = "$dst = $val" in {
  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
    def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_i32>;
  }

  def XCHGD : XCHG<BPF_DW, "64", atomic_swap_i64>;
}

// Compare-And-Exchange
class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs),
                 (ins MEMri:$addr, GPR:$new),
                 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)",
                 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> {
  bits<4> new;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = new;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = BPF_CMPXCHG.Value;
  let Inst{3-0} = BPF_FETCH.Value;
  let BPFClass = BPF_STX;
}

class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                 (outs),
                 (ins MEMri:$addr, GPR32:$new),
                 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)",
                 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> {
  bits<4> new;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = new;
  let Inst{47-32} = addr{15-0}; // offset
  let Inst{7-4} = BPF_CMPXCHG.Value;
  let Inst{3-0} = BPF_FETCH.Value;
  let BPFClass = BPF_STX;
}

let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0],
    DecoderNamespace = "BPFALU32" in {
  def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_i32>;
}

let Defs = [R0], Uses = [R0] in {
  def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_i64>;
}

// bswap16, bswap32, bswap64
class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern>
    : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value,
                   (outs GPR:$dst),
                   (ins GPR:$src),
                   "$dst = "#OpcodeStr#" $src",
                   Pattern> {
  bits<4> dst;

  let Inst{51-48} = dst;
  let Inst{31-0} = SizeOp;
  let BPFClass = Class;
}


let Constraints = "$dst = $src" in {
  let Predicates = [BPFHasBswap] in {
    def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;
    def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;
    def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>;
  }

  let Predicates = [BPFNoBswap] in {
    let Predicates = [BPFIsLittleEndian] in {
        def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;
        def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;
        def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>;
    }
    let Predicates = [BPFIsBigEndian] in {
        def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;
        def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;
        def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>;
    }
  }
}

let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1,
    hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in {
class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode>
    : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value,
                 (outs), (ins i64imm:$imm),
                 "r0 = *("#OpcodeStr#" *)skb[$imm]",
                 [(set R0, (OpNode R6, i64immSExt32:$imm))]> {
  bits<32> imm;

  let Inst{31-0} = imm;
  let BPFClass = BPF_LD;
}

class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode>
    : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value,
                 (outs), (ins GPR:$val),
                 "r0 = *("#OpcodeStr#" *)skb[$val]",
                 [(set R0, (OpNode R6, GPR:$val))]> {
  bits<4> val;

  let Inst{55-52} = val;
  let BPFClass = BPF_LD;
}
}

def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>;
def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>;
def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>;

def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>;
def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>;
def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>;

let isCodeGenOnly = 1 in {
  def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0,
                         (outs GPR:$dst), (ins GPR32:$src),
                         "$dst = $src", []>;
}

let Predicates = [BPFNoMovsx] in {
  def : Pat<(i64 (sext GPR32:$src)),
            (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>;
}

let Predicates = [BPFHasMovsx] in {
  def : Pat<(i64 (sext GPR32:$src)),
            (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>;
}

def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>;

// For i64 -> i32 truncation, use the 32-bit subregister directly.
def : Pat<(i32 (trunc GPR:$src)),
          (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>;

// For i32 -> i64 anyext, we don't care about the high bits.
def : Pat<(i64 (anyext GPR32:$src)),
          (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>;

class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern>
    : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
                 (outs),
                 (ins GPR32:$src, MEMri:$addr),
                 "*("#OpcodeStr#" *)($addr) = $src",
                 Pattern> {
  bits<4> src;
  bits<20> addr;

  let Inst{51-48} = addr{19-16}; // base reg
  let Inst{55-52} = src;
  let Inst{47-32} = addr{15-0}; // offset
  let BPFClass = BPF_STX;
}

class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>
    : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>;

class STORE_RELEASEi32<BPFWidthModifer Opc, string OpcodeStr>
    : STORE_RELEASE<Opc, OpcodeStr, GPR32>;

let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
  def STW32 : STOREi32<BPF_W, "u32", store>;
  def STH32 : STOREi32<BPF_H, "u16", truncstorei16>;
  def STB32 : STOREi32<BPF_B, "u8", truncstorei8>;

  let Predicates = [BPFHasLoadAcqStoreRel] in {
    def STWREL32 : STORE_RELEASEi32<BPF_W, "u32">;
    def STHREL32 : STORE_RELEASEi32<BPF_H, "u16">;
    def STBREL32 : STORE_RELEASEi32<BPF_B, "u8">;
  }
}

class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern>
    : TYPE_LD_ST<ModOp.Value, SizeOp.Value,
                (outs GPR32:$dst),
                (ins MEMri:$addr),
                "$dst = *("#OpcodeStr#" *)($addr)",
                Pattern> {
  bits<4> dst;
  bits<20> addr;

  let Inst{51-48} = dst;
  let Inst{55-52} = addr{19-16};
  let Inst{47-32} = addr{15-0};
  let BPFClass = BPF_LDX;
}

class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode>
    : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>;

class LOAD_ACQUIREi32<BPFWidthModifer SizeOp, string OpcodeStr>
    : LOAD_ACQUIRE<SizeOp, OpcodeStr, GPR32>;

let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
  def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>;
  def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>;
  def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>;

  let Predicates = [BPFHasLoadAcqStoreRel] in {
    def LDWACQ32 : LOAD_ACQUIREi32<BPF_W, "u32">;
    def LDHACQ32 : LOAD_ACQUIREi32<BPF_H, "u16">;
    def LDBACQ32 : LOAD_ACQUIREi32<BPF_B, "u8">;
  }
}

let Predicates = [BPFHasALU32] in {
  def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst),
            (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;
  def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst),
            (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;
  def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst),
            (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;
  def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>;
  def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>;

  let Predicates = [BPFHasLdsx] in {
    def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>;
    def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>;
  }

  def : Pat<(i64 (zextloadi8  ADDRri:$src)),
            (SUBREG_TO_REG (LDB32 ADDRri:$src), sub_32)>;
  def : Pat<(i64 (zextloadi16 ADDRri:$src)),
            (SUBREG_TO_REG (LDH32 ADDRri:$src), sub_32)>;
  def : Pat<(i64 (zextloadi32 ADDRri:$src)),
            (SUBREG_TO_REG (LDW32 ADDRri:$src), sub_32)>;
  def : Pat<(i64 (extloadi8  ADDRri:$src)),
            (SUBREG_TO_REG (LDB32 ADDRri:$src), sub_32)>;
  def : Pat<(i64 (extloadi16 ADDRri:$src)),
            (SUBREG_TO_REG (LDH32 ADDRri:$src), sub_32)>;
  def : Pat<(i64 (extloadi32 ADDRri:$src)),
            (SUBREG_TO_REG (LDW32 ADDRri:$src), sub_32)>;

  let Predicates = [BPFHasLoadAcqStoreRel] in {
    foreach P = [[relaxed_load<atomic_load_nonext_32>, LDW32],
                 [relaxed_load<atomic_load_azext_16>, LDH32],
                 [relaxed_load<atomic_load_azext_8>, LDB32],
                 [acquiring_load<atomic_load_nonext_32>, LDWACQ32],
                 [acquiring_load<atomic_load_azext_16>, LDHACQ32],
                 [acquiring_load<atomic_load_azext_8>, LDBACQ32],
                ] in {
      def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;
    }
  }

  let Predicates = [BPFHasLoadAcqStoreRel] in {
    foreach P = [[relaxed_store<atomic_store_32>, STW32],
                 [relaxed_store<atomic_store_16>, STH32],
                 [relaxed_store<atomic_store_8>, STB32],
                 [releasing_store<atomic_store_32>, STWREL32],
                 [releasing_store<atomic_store_16>, STHREL32],
                 [releasing_store<atomic_store_8>, STBREL32],
                ] in {
      def : Pat<(P[0] GPR32:$val, ADDRri:$addr), (P[1] GPR32:$val, ADDRri:$addr)>;
    }
  }
}

let usesCustomInserter = 1, isCodeGenOnly = 1 in {
    def MEMCPY : Pseudo<
      (outs),
      (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops),
      "#memcpy dst: $dst, src: $src, len: $len, align: $align",
      [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>;
}

// For GlobalValue and BlockAddress.
let usesCustomInserter = 1, isCodeGenOnly = 1 in {
  def LDIMM64 : Pseudo<(outs GPR:$dst), (ins i64imm:$addr), "", []>;
}