//===- AArch64Disassembler.cpp - Disassembler for AArch64 -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
//
//===----------------------------------------------------------------------===//

#include "AArch64Disassembler.h"
#include "AArch64ExternalSymbolizer.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "TargetInfo/AArch64TargetInfo.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm-c/Disassembler.h"
#include "llvm/MC/MCDisassembler/MCRelocationInfo.h"
#include "llvm/MC/MCFixedLenDisassembler.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/TargetRegistry.h"
#include <algorithm>
#include <memory>

using namespace llvm;

#define DEBUG_TYPE "aarch64-disassembler"

// Pull DecodeStatus and its enum values into the global namespace.
using DecodeStatus = MCDisassembler::DecodeStatus;

// Forward declare these because the autogenerated code will reference them.
// Definitions are further down.
static DecodeStatus DecodeFPR128RegisterClass(MCInst &Inst,
                                              unsigned RegNo, uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeFPR128_loRegisterClass(MCInst &Inst,
                                                 unsigned RegNo,
                                                 uint64_t Address,
                                                 const void *Decoder);
static DecodeStatus DecodeFPR64RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeFPR32RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeFPR16RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeFPR8RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
static DecodeStatus DecodeGPR64commonRegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeGPR64RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeGPR64x8ClassRegisterClass(MCInst &Inst,
                                                    unsigned RegNo,
                                                    uint64_t Address,
                                                    const void *Decoder);
static DecodeStatus DecodeGPR64spRegisterClass(MCInst &Inst,
                                               unsigned RegNo, uint64_t Address,
                                               const void *Decoder);
static DecodeStatus DecodeMatrixIndexGPR32_12_15RegisterClass(MCInst &Inst,
                                                              unsigned RegNo,
                                                              uint64_t Address,
                                                              const void *Decoder);
static DecodeStatus DecodeGPR32RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeGPR32spRegisterClass(MCInst &Inst,
                                               unsigned RegNo, uint64_t Address,
                                               const void *Decoder);
static DecodeStatus DecodeQQRegisterClass(MCInst &Inst, unsigned RegNo,
                                          uint64_t Address,
                                          const void *Decoder);
static DecodeStatus DecodeQQQRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Address,
                                           const void *Decoder);
static DecodeStatus DecodeQQQQRegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
static DecodeStatus DecodeDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                          uint64_t Address,
                                          const void *Decoder);
static DecodeStatus DecodeDDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Address,
                                           const void *Decoder);
static DecodeStatus DecodeDDDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
static DecodeStatus DecodeZPRRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Address,
                                           const void *Decoder);
static DecodeStatus DecodeZPR_4bRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeZPR_3bRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeZPR2RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
static DecodeStatus DecodeZPR3RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
static DecodeStatus DecodeZPR4RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void *Decoder);
template <unsigned NumBitsForTile>
static DecodeStatus DecodeMatrixTile(MCInst &Inst, unsigned RegNo,
                                     uint64_t Address, const void *Decoder);
static DecodeStatus DecodeMatrixTileListRegisterClass(MCInst &Inst,
                                                      unsigned RegMask,
                                                      uint64_t Address,
                                                      const void *Decoder);
static DecodeStatus DecodePPRRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Address,
                                           const void *Decoder);
static DecodeStatus DecodePPR_3bRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Address,
                                              const void *Decoder);

static DecodeStatus DecodeFixedPointScaleImm32(MCInst &Inst, unsigned Imm,
                                               uint64_t Address,
                                               const void *Decoder);
static DecodeStatus DecodeFixedPointScaleImm64(MCInst &Inst, unsigned Imm,
                                               uint64_t Address,
                                               const void *Decoder);
static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm,
                                       uint64_t Address, const void *Decoder);
static DecodeStatus DecodeMemExtend(MCInst &Inst, unsigned Imm,
                                    uint64_t Address, const void *Decoder);
static DecodeStatus DecodeMRSSystemRegister(MCInst &Inst, unsigned Imm,
                                            uint64_t Address, const void *Decoder);
static DecodeStatus DecodeMSRSystemRegister(MCInst &Inst, unsigned Imm,
                                            uint64_t Address, const void *Decoder);
static DecodeStatus DecodeThreeAddrSRegInstruction(MCInst &Inst, uint32_t insn,
                                                   uint64_t Address,
                                                   const void *Decoder);
static DecodeStatus DecodeMoveImmInstruction(MCInst &Inst, uint32_t insn,
                                             uint64_t Address,
                                             const void *Decoder);
static DecodeStatus DecodeUnsignedLdStInstruction(MCInst &Inst, uint32_t insn,
                                                  uint64_t Address,
                                                  const void *Decoder);
static DecodeStatus DecodeSignedLdStInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Address,
                                                const void *Decoder);
static DecodeStatus DecodeExclusiveLdStInstruction(MCInst &Inst, uint32_t insn,
                                                   uint64_t Address,
                                                   const void *Decoder);
static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn,
                                              uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeAuthLoadInstruction(MCInst &Inst, uint32_t insn,
                                              uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeAddSubERegInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Address,
                                                const void *Decoder);
static DecodeStatus DecodeLogicalImmInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Address,
                                                const void *Decoder);
static DecodeStatus DecodeModImmInstruction(MCInst &Inst, uint32_t insn,
                                            uint64_t Address,
                                            const void *Decoder);
static DecodeStatus DecodeModImmTiedInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Address,
                                                const void *Decoder);
static DecodeStatus DecodeAdrInstruction(MCInst &Inst, uint32_t insn,
                                         uint64_t Address, const void *Decoder);
static DecodeStatus DecodeAddSubImmShift(MCInst &Inst, uint32_t insn,
                                         uint64_t Address, const void *Decoder);
static DecodeStatus DecodeUnconditionalBranch(MCInst &Inst, uint32_t insn,
                                              uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeSystemPStateInstruction(MCInst &Inst, uint32_t insn,
                                                  uint64_t Address,
                                                  const void *Decoder);
static DecodeStatus DecodeTestAndBranch(MCInst &Inst, uint32_t insn,
                                        uint64_t Address, const void *Decoder);

static DecodeStatus DecodeFMOVLaneInstruction(MCInst &Inst, unsigned Insn,
                                              uint64_t Address,
                                              const void *Decoder);
static DecodeStatus DecodeVecShiftR64Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftR64ImmNarrow(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder);
static DecodeStatus DecodeVecShiftR32Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftR32ImmNarrow(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder);
static DecodeStatus DecodeVecShiftR16Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftR16ImmNarrow(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder);
static DecodeStatus DecodeVecShiftR8Imm(MCInst &Inst, unsigned Imm,
                                        uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftL64Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftL32Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftL16Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeVecShiftL8Imm(MCInst &Inst, unsigned Imm,
                                        uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeWSeqPairsClassRegisterClass(MCInst &Inst,
                                                      unsigned RegNo,
                                                      uint64_t Addr,
                                                      const void *Decoder);
static DecodeStatus DecodeXSeqPairsClassRegisterClass(MCInst &Inst,
                                                      unsigned RegNo,
                                                      uint64_t Addr,
                                                      const void *Decoder);
static DecodeStatus DecodeSVELogicalImmInstruction(llvm::MCInst &Inst,
                                                   uint32_t insn,
                                                   uint64_t Address,
                                                   const void *Decoder);
template<int Bits>
static DecodeStatus DecodeSImm(llvm::MCInst &Inst, uint64_t Imm,
                               uint64_t Address, const void *Decoder);
template <int ElementWidth>
static DecodeStatus DecodeImm8OptLsl(MCInst &Inst, unsigned Imm,
                                     uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeSVEIncDecImm(MCInst &Inst, unsigned Imm,
                                       uint64_t Addr, const void *Decoder);
static DecodeStatus DecodeSVCROp(MCInst &Inst, unsigned Imm, uint64_t Address,
                                 const void *Decoder);

static bool Check(DecodeStatus &Out, DecodeStatus In) {
  switch (In) {
    case MCDisassembler::Success:
      // Out stays the same.
      return true;
    case MCDisassembler::SoftFail:
      Out = In;
      return true;
    case MCDisassembler::Fail:
      Out = In;
      return false;
  }
  llvm_unreachable("Invalid DecodeStatus!");
}

#include "AArch64GenDisassemblerTables.inc"
#include "AArch64GenInstrInfo.inc"

#define Success MCDisassembler::Success
#define Fail MCDisassembler::Fail
#define SoftFail MCDisassembler::SoftFail

static MCDisassembler *createAArch64Disassembler(const Target &T,
                                               const MCSubtargetInfo &STI,
                                               MCContext &Ctx) {
  return new AArch64Disassembler(STI, Ctx);
}

DecodeStatus AArch64Disassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                                 ArrayRef<uint8_t> Bytes,
                                                 uint64_t Address,
                                                 raw_ostream &CS) const {
  CommentStream = &CS;

  Size = 0;
  // We want to read exactly 4 bytes of data.
  if (Bytes.size() < 4)
    return Fail;
  Size = 4;

  // Encoded as a small-endian 32-bit word in the stream.
  uint32_t Insn =
      (Bytes[3] << 24) | (Bytes[2] << 16) | (Bytes[1] << 8) | (Bytes[0] << 0);

  const uint8_t *Tables[] = {DecoderTable32, DecoderTableFallback32};

  for (auto Table : Tables) {
    DecodeStatus Result =
        decodeInstruction(Table, MI, Insn, Address, this, STI);

    switch (MI.getOpcode()) {
    default:
      break;
    // For Scalable Matrix Extension (SME) instructions that have an implicit
    // operand for the accumulator (ZA) which isn't encoded, manually insert
    // operand.
    case AArch64::LDR_ZA:
    case AArch64::STR_ZA: {
      MI.insert(MI.begin(), MCOperand::createReg(AArch64::ZA));
      // Spill and fill instructions have a single immediate used for both the
      // vector select offset and optional memory offset. Replicate the decoded
      // immediate.
      const MCOperand &Imm4Op = MI.getOperand(2);
      assert(Imm4Op.isImm() && "Unexpected operand type!");
      MI.addOperand(Imm4Op);
      break;
    }
    case AArch64::LD1_MXIPXX_H_B:
    case AArch64::LD1_MXIPXX_V_B:
    case AArch64::ST1_MXIPXX_H_B:
    case AArch64::ST1_MXIPXX_V_B:
    case AArch64::INSERT_MXIPZ_H_B:
    case AArch64::INSERT_MXIPZ_V_B:
      // e.g.
      // MOVA ZA0<HV>.B[<Ws>, <imm>], <Pg>/M, <Zn>.B
      //      ^ insert implicit 8-bit element tile
      MI.insert(MI.begin(), MCOperand::createReg(AArch64::ZAB0));
      break;
    case AArch64::EXTRACT_ZPMXI_H_B:
    case AArch64::EXTRACT_ZPMXI_V_B:
      // MOVA <Zd>.B, <Pg>/M, ZA0<HV>.B[<Ws>, <imm>]
      //                      ^ insert implicit 8-bit element tile
      MI.insert(MI.begin()+2, MCOperand::createReg(AArch64::ZAB0));
      break;
    }

    if (Result != MCDisassembler::Fail)
      return Result;
  }

  return MCDisassembler::Fail;
}

static MCSymbolizer *
createAArch64ExternalSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo,
                                LLVMSymbolLookupCallback SymbolLookUp,
                                void *DisInfo, MCContext *Ctx,
                                std::unique_ptr<MCRelocationInfo> &&RelInfo) {
  return new AArch64ExternalSymbolizer(*Ctx, std::move(RelInfo), GetOpInfo,
                                       SymbolLookUp, DisInfo);
}

extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Disassembler() {
  TargetRegistry::RegisterMCDisassembler(getTheAArch64leTarget(),
                                         createAArch64Disassembler);
  TargetRegistry::RegisterMCDisassembler(getTheAArch64beTarget(),
                                         createAArch64Disassembler);
  TargetRegistry::RegisterMCSymbolizer(getTheAArch64leTarget(),
                                       createAArch64ExternalSymbolizer);
  TargetRegistry::RegisterMCSymbolizer(getTheAArch64beTarget(),
                                       createAArch64ExternalSymbolizer);
  TargetRegistry::RegisterMCDisassembler(getTheAArch64_32Target(),
                                         createAArch64Disassembler);
  TargetRegistry::RegisterMCSymbolizer(getTheAArch64_32Target(),
                                       createAArch64ExternalSymbolizer);

  TargetRegistry::RegisterMCDisassembler(getTheARM64Target(),
                                         createAArch64Disassembler);
  TargetRegistry::RegisterMCSymbolizer(getTheARM64Target(),
                                       createAArch64ExternalSymbolizer);
  TargetRegistry::RegisterMCDisassembler(getTheARM64_32Target(),
                                         createAArch64Disassembler);
  TargetRegistry::RegisterMCSymbolizer(getTheARM64_32Target(),
                                       createAArch64ExternalSymbolizer);
}

static const unsigned FPR128DecoderTable[] = {
    AArch64::Q0,  AArch64::Q1,  AArch64::Q2,  AArch64::Q3,  AArch64::Q4,
    AArch64::Q5,  AArch64::Q6,  AArch64::Q7,  AArch64::Q8,  AArch64::Q9,
    AArch64::Q10, AArch64::Q11, AArch64::Q12, AArch64::Q13, AArch64::Q14,
    AArch64::Q15, AArch64::Q16, AArch64::Q17, AArch64::Q18, AArch64::Q19,
    AArch64::Q20, AArch64::Q21, AArch64::Q22, AArch64::Q23, AArch64::Q24,
    AArch64::Q25, AArch64::Q26, AArch64::Q27, AArch64::Q28, AArch64::Q29,
    AArch64::Q30, AArch64::Q31
};

static DecodeStatus DecodeFPR128RegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Addr,
                                              const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = FPR128DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeFPR128_loRegisterClass(MCInst &Inst, unsigned RegNo,
                                                 uint64_t Addr,
                                                 const void *Decoder) {
  if (RegNo > 15)
    return Fail;
  return DecodeFPR128RegisterClass(Inst, RegNo, Addr, Decoder);
}

static const unsigned FPR64DecoderTable[] = {
    AArch64::D0,  AArch64::D1,  AArch64::D2,  AArch64::D3,  AArch64::D4,
    AArch64::D5,  AArch64::D6,  AArch64::D7,  AArch64::D8,  AArch64::D9,
    AArch64::D10, AArch64::D11, AArch64::D12, AArch64::D13, AArch64::D14,
    AArch64::D15, AArch64::D16, AArch64::D17, AArch64::D18, AArch64::D19,
    AArch64::D20, AArch64::D21, AArch64::D22, AArch64::D23, AArch64::D24,
    AArch64::D25, AArch64::D26, AArch64::D27, AArch64::D28, AArch64::D29,
    AArch64::D30, AArch64::D31
};

static DecodeStatus DecodeFPR64RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Addr,
                                             const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = FPR64DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned FPR32DecoderTable[] = {
    AArch64::S0,  AArch64::S1,  AArch64::S2,  AArch64::S3,  AArch64::S4,
    AArch64::S5,  AArch64::S6,  AArch64::S7,  AArch64::S8,  AArch64::S9,
    AArch64::S10, AArch64::S11, AArch64::S12, AArch64::S13, AArch64::S14,
    AArch64::S15, AArch64::S16, AArch64::S17, AArch64::S18, AArch64::S19,
    AArch64::S20, AArch64::S21, AArch64::S22, AArch64::S23, AArch64::S24,
    AArch64::S25, AArch64::S26, AArch64::S27, AArch64::S28, AArch64::S29,
    AArch64::S30, AArch64::S31
};

static DecodeStatus DecodeFPR32RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Addr,
                                             const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = FPR32DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned FPR16DecoderTable[] = {
    AArch64::H0,  AArch64::H1,  AArch64::H2,  AArch64::H3,  AArch64::H4,
    AArch64::H5,  AArch64::H6,  AArch64::H7,  AArch64::H8,  AArch64::H9,
    AArch64::H10, AArch64::H11, AArch64::H12, AArch64::H13, AArch64::H14,
    AArch64::H15, AArch64::H16, AArch64::H17, AArch64::H18, AArch64::H19,
    AArch64::H20, AArch64::H21, AArch64::H22, AArch64::H23, AArch64::H24,
    AArch64::H25, AArch64::H26, AArch64::H27, AArch64::H28, AArch64::H29,
    AArch64::H30, AArch64::H31
};

static DecodeStatus DecodeFPR16RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Addr,
                                             const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = FPR16DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned FPR8DecoderTable[] = {
    AArch64::B0,  AArch64::B1,  AArch64::B2,  AArch64::B3,  AArch64::B4,
    AArch64::B5,  AArch64::B6,  AArch64::B7,  AArch64::B8,  AArch64::B9,
    AArch64::B10, AArch64::B11, AArch64::B12, AArch64::B13, AArch64::B14,
    AArch64::B15, AArch64::B16, AArch64::B17, AArch64::B18, AArch64::B19,
    AArch64::B20, AArch64::B21, AArch64::B22, AArch64::B23, AArch64::B24,
    AArch64::B25, AArch64::B26, AArch64::B27, AArch64::B28, AArch64::B29,
    AArch64::B30, AArch64::B31
};

static DecodeStatus DecodeFPR8RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Addr,
                                            const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = FPR8DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned GPR64DecoderTable[] = {
    AArch64::X0,  AArch64::X1,  AArch64::X2,  AArch64::X3,  AArch64::X4,
    AArch64::X5,  AArch64::X6,  AArch64::X7,  AArch64::X8,  AArch64::X9,
    AArch64::X10, AArch64::X11, AArch64::X12, AArch64::X13, AArch64::X14,
    AArch64::X15, AArch64::X16, AArch64::X17, AArch64::X18, AArch64::X19,
    AArch64::X20, AArch64::X21, AArch64::X22, AArch64::X23, AArch64::X24,
    AArch64::X25, AArch64::X26, AArch64::X27, AArch64::X28, AArch64::FP,
    AArch64::LR,  AArch64::XZR
};

static DecodeStatus DecodeGPR64commonRegisterClass(MCInst &Inst, unsigned RegNo,
                                                   uint64_t Addr,
                                                   const void *Decoder) {
  if (RegNo > 30)
    return Fail;

  unsigned Register = GPR64DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeGPR64RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Addr,
                                             const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = GPR64DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned GPR64x8DecoderTable[] = {
  AArch64::X0_X1_X2_X3_X4_X5_X6_X7,
  AArch64::X2_X3_X4_X5_X6_X7_X8_X9,
  AArch64::X4_X5_X6_X7_X8_X9_X10_X11,
  AArch64::X6_X7_X8_X9_X10_X11_X12_X13,
  AArch64::X8_X9_X10_X11_X12_X13_X14_X15,
  AArch64::X10_X11_X12_X13_X14_X15_X16_X17,
  AArch64::X12_X13_X14_X15_X16_X17_X18_X19,
  AArch64::X14_X15_X16_X17_X18_X19_X20_X21,
  AArch64::X16_X17_X18_X19_X20_X21_X22_X23,
  AArch64::X18_X19_X20_X21_X22_X23_X24_X25,
  AArch64::X20_X21_X22_X23_X24_X25_X26_X27,
  AArch64::X22_X23_X24_X25_X26_X27_X28_FP,
};

static DecodeStatus DecodeGPR64x8ClassRegisterClass(MCInst &Inst,
                                                    unsigned RegNo,
                                                    uint64_t Address,
                                                    const void *Decoder) {
  if (RegNo > 22)
    return Fail;
  if (RegNo & 1)
    return Fail;

  unsigned Register = GPR64x8DecoderTable[RegNo >> 1];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeGPR64spRegisterClass(MCInst &Inst, unsigned RegNo,
                                               uint64_t Addr,
                                               const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = GPR64DecoderTable[RegNo];
  if (Register == AArch64::XZR)
    Register = AArch64::SP;
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned MatrixIndexGPR32_12_15DecoderTable[] = {
    AArch64::W12, AArch64::W13, AArch64::W14, AArch64::W15
};

static DecodeStatus DecodeMatrixIndexGPR32_12_15RegisterClass(MCInst &Inst,
                                                              unsigned RegNo,
                                                              uint64_t Addr,
                                                              const void *Decoder) {
  if (RegNo > 3)
    return Fail;

  unsigned Register = MatrixIndexGPR32_12_15DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned GPR32DecoderTable[] = {
    AArch64::W0,  AArch64::W1,  AArch64::W2,  AArch64::W3,  AArch64::W4,
    AArch64::W5,  AArch64::W6,  AArch64::W7,  AArch64::W8,  AArch64::W9,
    AArch64::W10, AArch64::W11, AArch64::W12, AArch64::W13, AArch64::W14,
    AArch64::W15, AArch64::W16, AArch64::W17, AArch64::W18, AArch64::W19,
    AArch64::W20, AArch64::W21, AArch64::W22, AArch64::W23, AArch64::W24,
    AArch64::W25, AArch64::W26, AArch64::W27, AArch64::W28, AArch64::W29,
    AArch64::W30, AArch64::WZR
};

static DecodeStatus DecodeGPR32RegisterClass(MCInst &Inst, unsigned RegNo,
                                             uint64_t Addr,
                                             const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = GPR32DecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeGPR32spRegisterClass(MCInst &Inst, unsigned RegNo,
                                               uint64_t Addr,
                                               const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = GPR32DecoderTable[RegNo];
  if (Register == AArch64::WZR)
    Register = AArch64::WSP;
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}
static const unsigned ZPRDecoderTable[] = {
    AArch64::Z0,  AArch64::Z1,  AArch64::Z2,  AArch64::Z3,
    AArch64::Z4,  AArch64::Z5,  AArch64::Z6,  AArch64::Z7,
    AArch64::Z8,  AArch64::Z9,  AArch64::Z10, AArch64::Z11,
    AArch64::Z12, AArch64::Z13, AArch64::Z14, AArch64::Z15,
    AArch64::Z16, AArch64::Z17, AArch64::Z18, AArch64::Z19,
    AArch64::Z20, AArch64::Z21, AArch64::Z22, AArch64::Z23,
    AArch64::Z24, AArch64::Z25, AArch64::Z26, AArch64::Z27,
    AArch64::Z28, AArch64::Z29, AArch64::Z30, AArch64::Z31
};

static DecodeStatus DecodeZPRRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Address,
                                           const void* Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = ZPRDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeZPR_4bRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Address,
                                              const void *Decoder) {
  if (RegNo > 15)
    return Fail;
  return DecodeZPRRegisterClass(Inst, RegNo, Address, Decoder);
}

static DecodeStatus DecodeZPR_3bRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Address,
                                              const void *Decoder) {
  if (RegNo > 7)
    return Fail;
  return DecodeZPRRegisterClass(Inst, RegNo, Address, Decoder);
}

static const unsigned ZZDecoderTable[] = {
  AArch64::Z0_Z1,   AArch64::Z1_Z2,   AArch64::Z2_Z3,   AArch64::Z3_Z4,
  AArch64::Z4_Z5,   AArch64::Z5_Z6,   AArch64::Z6_Z7,   AArch64::Z7_Z8,
  AArch64::Z8_Z9,   AArch64::Z9_Z10,  AArch64::Z10_Z11, AArch64::Z11_Z12,
  AArch64::Z12_Z13, AArch64::Z13_Z14, AArch64::Z14_Z15, AArch64::Z15_Z16,
  AArch64::Z16_Z17, AArch64::Z17_Z18, AArch64::Z18_Z19, AArch64::Z19_Z20,
  AArch64::Z20_Z21, AArch64::Z21_Z22, AArch64::Z22_Z23, AArch64::Z23_Z24,
  AArch64::Z24_Z25, AArch64::Z25_Z26, AArch64::Z26_Z27, AArch64::Z27_Z28,
  AArch64::Z28_Z29, AArch64::Z29_Z30, AArch64::Z30_Z31, AArch64::Z31_Z0
};

static DecodeStatus DecodeZPR2RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void* Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = ZZDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned ZZZDecoderTable[] = {
  AArch64::Z0_Z1_Z2,    AArch64::Z1_Z2_Z3,    AArch64::Z2_Z3_Z4,
  AArch64::Z3_Z4_Z5,    AArch64::Z4_Z5_Z6,    AArch64::Z5_Z6_Z7,
  AArch64::Z6_Z7_Z8,    AArch64::Z7_Z8_Z9,    AArch64::Z8_Z9_Z10,
  AArch64::Z9_Z10_Z11,  AArch64::Z10_Z11_Z12, AArch64::Z11_Z12_Z13,
  AArch64::Z12_Z13_Z14, AArch64::Z13_Z14_Z15, AArch64::Z14_Z15_Z16,
  AArch64::Z15_Z16_Z17, AArch64::Z16_Z17_Z18, AArch64::Z17_Z18_Z19,
  AArch64::Z18_Z19_Z20, AArch64::Z19_Z20_Z21, AArch64::Z20_Z21_Z22,
  AArch64::Z21_Z22_Z23, AArch64::Z22_Z23_Z24, AArch64::Z23_Z24_Z25,
  AArch64::Z24_Z25_Z26, AArch64::Z25_Z26_Z27, AArch64::Z26_Z27_Z28,
  AArch64::Z27_Z28_Z29, AArch64::Z28_Z29_Z30, AArch64::Z29_Z30_Z31,
  AArch64::Z30_Z31_Z0,  AArch64::Z31_Z0_Z1
};

static DecodeStatus DecodeZPR3RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void* Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = ZZZDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned ZZZZDecoderTable[] = {
  AArch64::Z0_Z1_Z2_Z3,     AArch64::Z1_Z2_Z3_Z4,     AArch64::Z2_Z3_Z4_Z5,
  AArch64::Z3_Z4_Z5_Z6,     AArch64::Z4_Z5_Z6_Z7,     AArch64::Z5_Z6_Z7_Z8,
  AArch64::Z6_Z7_Z8_Z9,     AArch64::Z7_Z8_Z9_Z10,    AArch64::Z8_Z9_Z10_Z11,
  AArch64::Z9_Z10_Z11_Z12,  AArch64::Z10_Z11_Z12_Z13, AArch64::Z11_Z12_Z13_Z14,
  AArch64::Z12_Z13_Z14_Z15, AArch64::Z13_Z14_Z15_Z16, AArch64::Z14_Z15_Z16_Z17,
  AArch64::Z15_Z16_Z17_Z18, AArch64::Z16_Z17_Z18_Z19, AArch64::Z17_Z18_Z19_Z20,
  AArch64::Z18_Z19_Z20_Z21, AArch64::Z19_Z20_Z21_Z22, AArch64::Z20_Z21_Z22_Z23,
  AArch64::Z21_Z22_Z23_Z24, AArch64::Z22_Z23_Z24_Z25, AArch64::Z23_Z24_Z25_Z26,
  AArch64::Z24_Z25_Z26_Z27, AArch64::Z25_Z26_Z27_Z28, AArch64::Z26_Z27_Z28_Z29,
  AArch64::Z27_Z28_Z29_Z30, AArch64::Z28_Z29_Z30_Z31, AArch64::Z29_Z30_Z31_Z0,
  AArch64::Z30_Z31_Z0_Z1,   AArch64::Z31_Z0_Z1_Z2
};

static DecodeStatus DecodeZPR4RegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Address,
                                            const void* Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = ZZZZDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeMatrixTileListRegisterClass(MCInst &Inst,
                                                      unsigned RegMask,
                                                      uint64_t Address,
                                                      const void *Decoder) {
  if (RegMask > 0xFF)
    return Fail;
  Inst.addOperand(MCOperand::createImm(RegMask));
  return Success;
}

static const SmallVector<SmallVector<unsigned, 16>, 5>
    MatrixZATileDecoderTable = {
        {AArch64::ZAB0},
        {AArch64::ZAH0, AArch64::ZAH1},
        {AArch64::ZAS0, AArch64::ZAS1, AArch64::ZAS2, AArch64::ZAS3},
        {AArch64::ZAD0, AArch64::ZAD1, AArch64::ZAD2, AArch64::ZAD3,
         AArch64::ZAD4, AArch64::ZAD5, AArch64::ZAD6, AArch64::ZAD7},
        {AArch64::ZAQ0, AArch64::ZAQ1, AArch64::ZAQ2, AArch64::ZAQ3,
         AArch64::ZAQ4, AArch64::ZAQ5, AArch64::ZAQ6, AArch64::ZAQ7,
         AArch64::ZAQ8, AArch64::ZAQ9, AArch64::ZAQ10, AArch64::ZAQ11,
         AArch64::ZAQ12, AArch64::ZAQ13, AArch64::ZAQ14, AArch64::ZAQ15}};

template <unsigned NumBitsForTile>
static DecodeStatus DecodeMatrixTile(MCInst &Inst, unsigned RegNo,
                                     uint64_t Address, const void *Decoder) {
  unsigned LastReg = (1 << NumBitsForTile) - 1;
  if (RegNo > LastReg)
    return Fail;
  Inst.addOperand(
      MCOperand::createReg(MatrixZATileDecoderTable[NumBitsForTile][RegNo]));
  return Success;
}

static const unsigned PPRDecoderTable[] = {
  AArch64::P0,  AArch64::P1,  AArch64::P2,  AArch64::P3,
  AArch64::P4,  AArch64::P5,  AArch64::P6,  AArch64::P7,
  AArch64::P8,  AArch64::P9,  AArch64::P10, AArch64::P11,
  AArch64::P12, AArch64::P13, AArch64::P14, AArch64::P15
};

static DecodeStatus DecodePPRRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Addr, const void *Decoder) {
  if (RegNo > 15)
    return Fail;

  unsigned Register = PPRDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodePPR_3bRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Addr,
                                              const void* Decoder) {
  if (RegNo > 7)
    return Fail;

  // Just reuse the PPR decode table
  return DecodePPRRegisterClass(Inst, RegNo, Addr, Decoder);
}

static const unsigned VectorDecoderTable[] = {
    AArch64::Q0,  AArch64::Q1,  AArch64::Q2,  AArch64::Q3,  AArch64::Q4,
    AArch64::Q5,  AArch64::Q6,  AArch64::Q7,  AArch64::Q8,  AArch64::Q9,
    AArch64::Q10, AArch64::Q11, AArch64::Q12, AArch64::Q13, AArch64::Q14,
    AArch64::Q15, AArch64::Q16, AArch64::Q17, AArch64::Q18, AArch64::Q19,
    AArch64::Q20, AArch64::Q21, AArch64::Q22, AArch64::Q23, AArch64::Q24,
    AArch64::Q25, AArch64::Q26, AArch64::Q27, AArch64::Q28, AArch64::Q29,
    AArch64::Q30, AArch64::Q31
};

static DecodeStatus DecodeVectorRegisterClass(MCInst &Inst, unsigned RegNo,
                                              uint64_t Addr,
                                              const void *Decoder) {
  if (RegNo > 31)
    return Fail;

  unsigned Register = VectorDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned QQDecoderTable[] = {
  AArch64::Q0_Q1,   AArch64::Q1_Q2,   AArch64::Q2_Q3,   AArch64::Q3_Q4,
  AArch64::Q4_Q5,   AArch64::Q5_Q6,   AArch64::Q6_Q7,   AArch64::Q7_Q8,
  AArch64::Q8_Q9,   AArch64::Q9_Q10,  AArch64::Q10_Q11, AArch64::Q11_Q12,
  AArch64::Q12_Q13, AArch64::Q13_Q14, AArch64::Q14_Q15, AArch64::Q15_Q16,
  AArch64::Q16_Q17, AArch64::Q17_Q18, AArch64::Q18_Q19, AArch64::Q19_Q20,
  AArch64::Q20_Q21, AArch64::Q21_Q22, AArch64::Q22_Q23, AArch64::Q23_Q24,
  AArch64::Q24_Q25, AArch64::Q25_Q26, AArch64::Q26_Q27, AArch64::Q27_Q28,
  AArch64::Q28_Q29, AArch64::Q29_Q30, AArch64::Q30_Q31, AArch64::Q31_Q0
};

static DecodeStatus DecodeQQRegisterClass(MCInst &Inst, unsigned RegNo,
                                          uint64_t Addr, const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = QQDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned QQQDecoderTable[] = {
  AArch64::Q0_Q1_Q2,    AArch64::Q1_Q2_Q3,    AArch64::Q2_Q3_Q4,
  AArch64::Q3_Q4_Q5,    AArch64::Q4_Q5_Q6,    AArch64::Q5_Q6_Q7,
  AArch64::Q6_Q7_Q8,    AArch64::Q7_Q8_Q9,    AArch64::Q8_Q9_Q10,
  AArch64::Q9_Q10_Q11,  AArch64::Q10_Q11_Q12, AArch64::Q11_Q12_Q13,
  AArch64::Q12_Q13_Q14, AArch64::Q13_Q14_Q15, AArch64::Q14_Q15_Q16,
  AArch64::Q15_Q16_Q17, AArch64::Q16_Q17_Q18, AArch64::Q17_Q18_Q19,
  AArch64::Q18_Q19_Q20, AArch64::Q19_Q20_Q21, AArch64::Q20_Q21_Q22,
  AArch64::Q21_Q22_Q23, AArch64::Q22_Q23_Q24, AArch64::Q23_Q24_Q25,
  AArch64::Q24_Q25_Q26, AArch64::Q25_Q26_Q27, AArch64::Q26_Q27_Q28,
  AArch64::Q27_Q28_Q29, AArch64::Q28_Q29_Q30, AArch64::Q29_Q30_Q31,
  AArch64::Q30_Q31_Q0,  AArch64::Q31_Q0_Q1
};

static DecodeStatus DecodeQQQRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Addr, const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = QQQDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned QQQQDecoderTable[] = {
  AArch64::Q0_Q1_Q2_Q3,     AArch64::Q1_Q2_Q3_Q4,     AArch64::Q2_Q3_Q4_Q5,
  AArch64::Q3_Q4_Q5_Q6,     AArch64::Q4_Q5_Q6_Q7,     AArch64::Q5_Q6_Q7_Q8,
  AArch64::Q6_Q7_Q8_Q9,     AArch64::Q7_Q8_Q9_Q10,    AArch64::Q8_Q9_Q10_Q11,
  AArch64::Q9_Q10_Q11_Q12,  AArch64::Q10_Q11_Q12_Q13, AArch64::Q11_Q12_Q13_Q14,
  AArch64::Q12_Q13_Q14_Q15, AArch64::Q13_Q14_Q15_Q16, AArch64::Q14_Q15_Q16_Q17,
  AArch64::Q15_Q16_Q17_Q18, AArch64::Q16_Q17_Q18_Q19, AArch64::Q17_Q18_Q19_Q20,
  AArch64::Q18_Q19_Q20_Q21, AArch64::Q19_Q20_Q21_Q22, AArch64::Q20_Q21_Q22_Q23,
  AArch64::Q21_Q22_Q23_Q24, AArch64::Q22_Q23_Q24_Q25, AArch64::Q23_Q24_Q25_Q26,
  AArch64::Q24_Q25_Q26_Q27, AArch64::Q25_Q26_Q27_Q28, AArch64::Q26_Q27_Q28_Q29,
  AArch64::Q27_Q28_Q29_Q30, AArch64::Q28_Q29_Q30_Q31, AArch64::Q29_Q30_Q31_Q0,
  AArch64::Q30_Q31_Q0_Q1,   AArch64::Q31_Q0_Q1_Q2
};

static DecodeStatus DecodeQQQQRegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Addr,
                                            const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = QQQQDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned DDDecoderTable[] = {
  AArch64::D0_D1,   AArch64::D1_D2,   AArch64::D2_D3,   AArch64::D3_D4,
  AArch64::D4_D5,   AArch64::D5_D6,   AArch64::D6_D7,   AArch64::D7_D8,
  AArch64::D8_D9,   AArch64::D9_D10,  AArch64::D10_D11, AArch64::D11_D12,
  AArch64::D12_D13, AArch64::D13_D14, AArch64::D14_D15, AArch64::D15_D16,
  AArch64::D16_D17, AArch64::D17_D18, AArch64::D18_D19, AArch64::D19_D20,
  AArch64::D20_D21, AArch64::D21_D22, AArch64::D22_D23, AArch64::D23_D24,
  AArch64::D24_D25, AArch64::D25_D26, AArch64::D26_D27, AArch64::D27_D28,
  AArch64::D28_D29, AArch64::D29_D30, AArch64::D30_D31, AArch64::D31_D0
};

static DecodeStatus DecodeDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                          uint64_t Addr, const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = DDDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned DDDDecoderTable[] = {
  AArch64::D0_D1_D2,    AArch64::D1_D2_D3,    AArch64::D2_D3_D4,
  AArch64::D3_D4_D5,    AArch64::D4_D5_D6,    AArch64::D5_D6_D7,
  AArch64::D6_D7_D8,    AArch64::D7_D8_D9,    AArch64::D8_D9_D10,
  AArch64::D9_D10_D11,  AArch64::D10_D11_D12, AArch64::D11_D12_D13,
  AArch64::D12_D13_D14, AArch64::D13_D14_D15, AArch64::D14_D15_D16,
  AArch64::D15_D16_D17, AArch64::D16_D17_D18, AArch64::D17_D18_D19,
  AArch64::D18_D19_D20, AArch64::D19_D20_D21, AArch64::D20_D21_D22,
  AArch64::D21_D22_D23, AArch64::D22_D23_D24, AArch64::D23_D24_D25,
  AArch64::D24_D25_D26, AArch64::D25_D26_D27, AArch64::D26_D27_D28,
  AArch64::D27_D28_D29, AArch64::D28_D29_D30, AArch64::D29_D30_D31,
  AArch64::D30_D31_D0,  AArch64::D31_D0_D1
};

static DecodeStatus DecodeDDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                           uint64_t Addr, const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = DDDDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static const unsigned DDDDDecoderTable[] = {
  AArch64::D0_D1_D2_D3,     AArch64::D1_D2_D3_D4,     AArch64::D2_D3_D4_D5,
  AArch64::D3_D4_D5_D6,     AArch64::D4_D5_D6_D7,     AArch64::D5_D6_D7_D8,
  AArch64::D6_D7_D8_D9,     AArch64::D7_D8_D9_D10,    AArch64::D8_D9_D10_D11,
  AArch64::D9_D10_D11_D12,  AArch64::D10_D11_D12_D13, AArch64::D11_D12_D13_D14,
  AArch64::D12_D13_D14_D15, AArch64::D13_D14_D15_D16, AArch64::D14_D15_D16_D17,
  AArch64::D15_D16_D17_D18, AArch64::D16_D17_D18_D19, AArch64::D17_D18_D19_D20,
  AArch64::D18_D19_D20_D21, AArch64::D19_D20_D21_D22, AArch64::D20_D21_D22_D23,
  AArch64::D21_D22_D23_D24, AArch64::D22_D23_D24_D25, AArch64::D23_D24_D25_D26,
  AArch64::D24_D25_D26_D27, AArch64::D25_D26_D27_D28, AArch64::D26_D27_D28_D29,
  AArch64::D27_D28_D29_D30, AArch64::D28_D29_D30_D31, AArch64::D29_D30_D31_D0,
  AArch64::D30_D31_D0_D1,   AArch64::D31_D0_D1_D2
};

static DecodeStatus DecodeDDDDRegisterClass(MCInst &Inst, unsigned RegNo,
                                            uint64_t Addr,
                                            const void *Decoder) {
  if (RegNo > 31)
    return Fail;
  unsigned Register = DDDDDecoderTable[RegNo];
  Inst.addOperand(MCOperand::createReg(Register));
  return Success;
}

static DecodeStatus DecodeFixedPointScaleImm32(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder) {
  // scale{5} is asserted as 1 in tblgen.
  Imm |= 0x20;
  Inst.addOperand(MCOperand::createImm(64 - Imm));
  return Success;
}

static DecodeStatus DecodeFixedPointScaleImm64(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder) {
  Inst.addOperand(MCOperand::createImm(64 - Imm));
  return Success;
}

static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm,
                                       uint64_t Addr, const void *Decoder) {
  int64_t ImmVal = Imm;
  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);

  // Sign-extend 19-bit immediate.
  if (ImmVal & (1 << (19 - 1)))
    ImmVal |= ~((1LL << 19) - 1);

  if (!Dis->tryAddingSymbolicOperand(Inst, ImmVal *  4, Addr,
                                     Inst.getOpcode() != AArch64::LDRXl, 0, 4))
    Inst.addOperand(MCOperand::createImm(ImmVal));
  return Success;
}

static DecodeStatus DecodeMemExtend(MCInst &Inst, unsigned Imm,
                                    uint64_t Address, const void *Decoder) {
  Inst.addOperand(MCOperand::createImm((Imm  >> 1) & 1));
  Inst.addOperand(MCOperand::createImm(Imm & 1));
  return Success;
}

static DecodeStatus DecodeMRSSystemRegister(MCInst &Inst, unsigned Imm,
                                            uint64_t Address,
                                            const void *Decoder) {
  Inst.addOperand(MCOperand::createImm(Imm));

  // Every system register in the encoding space is valid with the syntax
  // S<op0>_<op1>_<Cn>_<Cm>_<op2>, so decoding system registers always succeeds.
  return Success;
}

static DecodeStatus DecodeMSRSystemRegister(MCInst &Inst, unsigned Imm,
                                            uint64_t Address,
                                            const void *Decoder) {
  Inst.addOperand(MCOperand::createImm(Imm));

  return Success;
}

static DecodeStatus DecodeFMOVLaneInstruction(MCInst &Inst, unsigned Insn,
                                              uint64_t Address,
                                              const void *Decoder) {
  // This decoder exists to add the dummy Lane operand to the MCInst, which must
  // be 1 in assembly but has no other real manifestation.
  unsigned Rd = fieldFromInstruction(Insn, 0, 5);
  unsigned Rn = fieldFromInstruction(Insn, 5, 5);
  unsigned IsToVec = fieldFromInstruction(Insn, 16, 1);

  if (IsToVec) {
    DecodeFPR128RegisterClass(Inst, Rd, Address, Decoder);
    DecodeGPR64RegisterClass(Inst, Rn, Address, Decoder);
  } else {
    DecodeGPR64RegisterClass(Inst, Rd, Address, Decoder);
    DecodeFPR128RegisterClass(Inst, Rn, Address, Decoder);
  }

  // Add the lane
  Inst.addOperand(MCOperand::createImm(1));

  return Success;
}

static DecodeStatus DecodeVecShiftRImm(MCInst &Inst, unsigned Imm,
                                       unsigned Add) {
  Inst.addOperand(MCOperand::createImm(Add - Imm));
  return Success;
}

static DecodeStatus DecodeVecShiftLImm(MCInst &Inst, unsigned Imm,
                                       unsigned Add) {
  Inst.addOperand(MCOperand::createImm((Imm + Add) & (Add - 1)));
  return Success;
}

static DecodeStatus DecodeVecShiftR64Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm, 64);
}

static DecodeStatus DecodeVecShiftR64ImmNarrow(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm | 0x20, 64);
}

static DecodeStatus DecodeVecShiftR32Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm, 32);
}

static DecodeStatus DecodeVecShiftR32ImmNarrow(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm | 0x10, 32);
}

static DecodeStatus DecodeVecShiftR16Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm, 16);
}

static DecodeStatus DecodeVecShiftR16ImmNarrow(MCInst &Inst, unsigned Imm,
                                               uint64_t Addr,
                                               const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm | 0x8, 16);
}

static DecodeStatus DecodeVecShiftR8Imm(MCInst &Inst, unsigned Imm,
                                        uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftRImm(Inst, Imm, 8);
}

static DecodeStatus DecodeVecShiftL64Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftLImm(Inst, Imm, 64);
}

static DecodeStatus DecodeVecShiftL32Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftLImm(Inst, Imm, 32);
}

static DecodeStatus DecodeVecShiftL16Imm(MCInst &Inst, unsigned Imm,
                                         uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftLImm(Inst, Imm, 16);
}

static DecodeStatus DecodeVecShiftL8Imm(MCInst &Inst, unsigned Imm,
                                        uint64_t Addr, const void *Decoder) {
  return DecodeVecShiftLImm(Inst, Imm, 8);
}

static DecodeStatus DecodeThreeAddrSRegInstruction(MCInst &Inst, uint32_t insn,
                                                   uint64_t Addr,
                                                   const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned Rm = fieldFromInstruction(insn, 16, 5);
  unsigned shiftHi = fieldFromInstruction(insn, 22, 2);
  unsigned shiftLo = fieldFromInstruction(insn, 10, 6);
  unsigned shift = (shiftHi << 6) | shiftLo;
  switch (Inst.getOpcode()) {
  default:
    return Fail;
  case AArch64::ADDWrs:
  case AArch64::ADDSWrs:
  case AArch64::SUBWrs:
  case AArch64::SUBSWrs:
    // if shift == '11' then ReservedValue()
    if (shiftHi == 0x3)
      return Fail;
    LLVM_FALLTHROUGH;
  case AArch64::ANDWrs:
  case AArch64::ANDSWrs:
  case AArch64::BICWrs:
  case AArch64::BICSWrs:
  case AArch64::ORRWrs:
  case AArch64::ORNWrs:
  case AArch64::EORWrs:
  case AArch64::EONWrs: {
    // if sf == '0' and imm6<5> == '1' then ReservedValue()
    if (shiftLo >> 5 == 1)
      return Fail;
    DecodeGPR32RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  }
  case AArch64::ADDXrs:
  case AArch64::ADDSXrs:
  case AArch64::SUBXrs:
  case AArch64::SUBSXrs:
    // if shift == '11' then ReservedValue()
    if (shiftHi == 0x3)
      return Fail;
    LLVM_FALLTHROUGH;
  case AArch64::ANDXrs:
  case AArch64::ANDSXrs:
  case AArch64::BICXrs:
  case AArch64::BICSXrs:
  case AArch64::ORRXrs:
  case AArch64::ORNXrs:
  case AArch64::EORXrs:
  case AArch64::EONXrs:
    DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  }

  Inst.addOperand(MCOperand::createImm(shift));
  return Success;
}

static DecodeStatus DecodeMoveImmInstruction(MCInst &Inst, uint32_t insn,
                                             uint64_t Addr,
                                             const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned imm = fieldFromInstruction(insn, 5, 16);
  unsigned shift = fieldFromInstruction(insn, 21, 2);
  shift <<= 4;
  switch (Inst.getOpcode()) {
  default:
    return Fail;
  case AArch64::MOVZWi:
  case AArch64::MOVNWi:
  case AArch64::MOVKWi:
    if (shift & (1U << 5))
      return Fail;
    DecodeGPR32RegisterClass(Inst, Rd, Addr, Decoder);
    break;
  case AArch64::MOVZXi:
  case AArch64::MOVNXi:
  case AArch64::MOVKXi:
    DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
    break;
  }

  if (Inst.getOpcode() == AArch64::MOVKWi ||
      Inst.getOpcode() == AArch64::MOVKXi)
    Inst.addOperand(Inst.getOperand(0));

  Inst.addOperand(MCOperand::createImm(imm));
  Inst.addOperand(MCOperand::createImm(shift));
  return Success;
}

static DecodeStatus DecodeUnsignedLdStInstruction(MCInst &Inst, uint32_t insn,
                                                  uint64_t Addr,
                                                  const void *Decoder) {
  unsigned Rt = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned offset = fieldFromInstruction(insn, 10, 12);
  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);

  switch (Inst.getOpcode()) {
  default:
    return Fail;
  case AArch64::PRFMui:
    // Rt is an immediate in prefetch.
    Inst.addOperand(MCOperand::createImm(Rt));
    break;
  case AArch64::STRBBui:
  case AArch64::LDRBBui:
  case AArch64::LDRSBWui:
  case AArch64::STRHHui:
  case AArch64::LDRHHui:
  case AArch64::LDRSHWui:
  case AArch64::STRWui:
  case AArch64::LDRWui:
    DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDRSBXui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSWui:
  case AArch64::STRXui:
  case AArch64::LDRXui:
    DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDRQui:
  case AArch64::STRQui:
    DecodeFPR128RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDRDui:
  case AArch64::STRDui:
    DecodeFPR64RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDRSui:
  case AArch64::STRSui:
    DecodeFPR32RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDRHui:
  case AArch64::STRHui:
    DecodeFPR16RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDRBui:
  case AArch64::STRBui:
    DecodeFPR8RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  }

  DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
  if (!Dis->tryAddingSymbolicOperand(Inst, offset, Addr, Fail, 0, 4))
    Inst.addOperand(MCOperand::createImm(offset));
  return Success;
}

static DecodeStatus DecodeSignedLdStInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Addr,
                                                const void *Decoder) {
  unsigned Rt = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  int64_t offset = fieldFromInstruction(insn, 12, 9);

  // offset is a 9-bit signed immediate, so sign extend it to
  // fill the unsigned.
  if (offset & (1 << (9 - 1)))
    offset |= ~((1LL << 9) - 1);

  // First operand is always the writeback to the address register, if needed.
  switch (Inst.getOpcode()) {
  default:
    break;
  case AArch64::LDRSBWpre:
  case AArch64::LDRSHWpre:
  case AArch64::STRBBpre:
  case AArch64::LDRBBpre:
  case AArch64::STRHHpre:
  case AArch64::LDRHHpre:
  case AArch64::STRWpre:
  case AArch64::LDRWpre:
  case AArch64::LDRSBWpost:
  case AArch64::LDRSHWpost:
  case AArch64::STRBBpost:
  case AArch64::LDRBBpost:
  case AArch64::STRHHpost:
  case AArch64::LDRHHpost:
  case AArch64::STRWpost:
  case AArch64::LDRWpost:
  case AArch64::LDRSBXpre:
  case AArch64::LDRSHXpre:
  case AArch64::STRXpre:
  case AArch64::LDRSWpre:
  case AArch64::LDRXpre:
  case AArch64::LDRSBXpost:
  case AArch64::LDRSHXpost:
  case AArch64::STRXpost:
  case AArch64::LDRSWpost:
  case AArch64::LDRXpost:
  case AArch64::LDRQpre:
  case AArch64::STRQpre:
  case AArch64::LDRQpost:
  case AArch64::STRQpost:
  case AArch64::LDRDpre:
  case AArch64::STRDpre:
  case AArch64::LDRDpost:
  case AArch64::STRDpost:
  case AArch64::LDRSpre:
  case AArch64::STRSpre:
  case AArch64::LDRSpost:
  case AArch64::STRSpost:
  case AArch64::LDRHpre:
  case AArch64::STRHpre:
  case AArch64::LDRHpost:
  case AArch64::STRHpost:
  case AArch64::LDRBpre:
  case AArch64::STRBpre:
  case AArch64::LDRBpost:
  case AArch64::STRBpost:
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
    break;
  }

  switch (Inst.getOpcode()) {
  default:
    return Fail;
  case AArch64::PRFUMi:
    // Rt is an immediate in prefetch.
    Inst.addOperand(MCOperand::createImm(Rt));
    break;
  case AArch64::STURBBi:
  case AArch64::LDURBBi:
  case AArch64::LDURSBWi:
  case AArch64::STURHHi:
  case AArch64::LDURHHi:
  case AArch64::LDURSHWi:
  case AArch64::STURWi:
  case AArch64::LDURWi:
  case AArch64::LDTRSBWi:
  case AArch64::LDTRSHWi:
  case AArch64::STTRWi:
  case AArch64::LDTRWi:
  case AArch64::STTRHi:
  case AArch64::LDTRHi:
  case AArch64::LDTRBi:
  case AArch64::STTRBi:
  case AArch64::LDRSBWpre:
  case AArch64::LDRSHWpre:
  case AArch64::STRBBpre:
  case AArch64::LDRBBpre:
  case AArch64::STRHHpre:
  case AArch64::LDRHHpre:
  case AArch64::STRWpre:
  case AArch64::LDRWpre:
  case AArch64::LDRSBWpost:
  case AArch64::LDRSHWpost:
  case AArch64::STRBBpost:
  case AArch64::LDRBBpost:
  case AArch64::STRHHpost:
  case AArch64::LDRHHpost:
  case AArch64::STRWpost:
  case AArch64::LDRWpost:
  case AArch64::STLURBi:
  case AArch64::STLURHi:
  case AArch64::STLURWi:
  case AArch64::LDAPURBi:
  case AArch64::LDAPURSBWi:
  case AArch64::LDAPURHi:
  case AArch64::LDAPURSHWi:
  case AArch64::LDAPURi:
    DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDURSBXi:
  case AArch64::LDURSHXi:
  case AArch64::LDURSWi:
  case AArch64::STURXi:
  case AArch64::LDURXi:
  case AArch64::LDTRSBXi:
  case AArch64::LDTRSHXi:
  case AArch64::LDTRSWi:
  case AArch64::STTRXi:
  case AArch64::LDTRXi:
  case AArch64::LDRSBXpre:
  case AArch64::LDRSHXpre:
  case AArch64::STRXpre:
  case AArch64::LDRSWpre:
  case AArch64::LDRXpre:
  case AArch64::LDRSBXpost:
  case AArch64::LDRSHXpost:
  case AArch64::STRXpost:
  case AArch64::LDRSWpost:
  case AArch64::LDRXpost:
  case AArch64::LDAPURSWi:
  case AArch64::LDAPURSHXi:
  case AArch64::LDAPURSBXi:
  case AArch64::STLURXi:
  case AArch64::LDAPURXi:
    DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDURQi:
  case AArch64::STURQi:
  case AArch64::LDRQpre:
  case AArch64::STRQpre:
  case AArch64::LDRQpost:
  case AArch64::STRQpost:
    DecodeFPR128RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDURDi:
  case AArch64::STURDi:
  case AArch64::LDRDpre:
  case AArch64::STRDpre:
  case AArch64::LDRDpost:
  case AArch64::STRDpost:
    DecodeFPR64RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDURSi:
  case AArch64::STURSi:
  case AArch64::LDRSpre:
  case AArch64::STRSpre:
  case AArch64::LDRSpost:
  case AArch64::STRSpost:
    DecodeFPR32RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDURHi:
  case AArch64::STURHi:
  case AArch64::LDRHpre:
  case AArch64::STRHpre:
  case AArch64::LDRHpost:
  case AArch64::STRHpost:
    DecodeFPR16RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::LDURBi:
  case AArch64::STURBi:
  case AArch64::LDRBpre:
  case AArch64::STRBpre:
  case AArch64::LDRBpost:
  case AArch64::STRBpost:
    DecodeFPR8RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  }

  DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
  Inst.addOperand(MCOperand::createImm(offset));

  bool IsLoad = fieldFromInstruction(insn, 22, 1);
  bool IsIndexed = fieldFromInstruction(insn, 10, 2) != 0;
  bool IsFP = fieldFromInstruction(insn, 26, 1);

  // Cannot write back to a transfer register (but xzr != sp).
  if (IsLoad && IsIndexed && !IsFP && Rn != 31 && Rt == Rn)
    return SoftFail;

  return Success;
}

static DecodeStatus DecodeExclusiveLdStInstruction(MCInst &Inst, uint32_t insn,
                                                   uint64_t Addr,
                                                   const void *Decoder) {
  unsigned Rt = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned Rt2 = fieldFromInstruction(insn, 10, 5);
  unsigned Rs = fieldFromInstruction(insn, 16, 5);

  unsigned Opcode = Inst.getOpcode();
  switch (Opcode) {
  default:
    return Fail;
  case AArch64::STLXRW:
  case AArch64::STLXRB:
  case AArch64::STLXRH:
  case AArch64::STXRW:
  case AArch64::STXRB:
  case AArch64::STXRH:
    DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder);
    LLVM_FALLTHROUGH;
  case AArch64::LDARW:
  case AArch64::LDARB:
  case AArch64::LDARH:
  case AArch64::LDAXRW:
  case AArch64::LDAXRB:
  case AArch64::LDAXRH:
  case AArch64::LDXRW:
  case AArch64::LDXRB:
  case AArch64::LDXRH:
  case AArch64::STLRW:
  case AArch64::STLRB:
  case AArch64::STLRH:
  case AArch64::STLLRW:
  case AArch64::STLLRB:
  case AArch64::STLLRH:
  case AArch64::LDLARW:
  case AArch64::LDLARB:
  case AArch64::LDLARH:
    DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::STLXRX:
  case AArch64::STXRX:
    DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder);
    LLVM_FALLTHROUGH;
  case AArch64::LDARX:
  case AArch64::LDAXRX:
  case AArch64::LDXRX:
  case AArch64::STLRX:
  case AArch64::LDLARX:
  case AArch64::STLLRX:
    DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
    break;
  case AArch64::STLXPW:
  case AArch64::STXPW:
    DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder);
    LLVM_FALLTHROUGH;
  case AArch64::LDAXPW:
  case AArch64::LDXPW:
    DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  case AArch64::STLXPX:
  case AArch64::STXPX:
    DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder);
    LLVM_FALLTHROUGH;
  case AArch64::LDAXPX:
  case AArch64::LDXPX:
    DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  }

  DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);

  // You shouldn't load to the same register twice in an instruction...
  if ((Opcode == AArch64::LDAXPW || Opcode == AArch64::LDXPW ||
       Opcode == AArch64::LDAXPX || Opcode == AArch64::LDXPX) &&
      Rt == Rt2)
    return SoftFail;

  return Success;
}

static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn,
                                              uint64_t Addr,
                                              const void *Decoder) {
  unsigned Rt = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned Rt2 = fieldFromInstruction(insn, 10, 5);
  int64_t offset = fieldFromInstruction(insn, 15, 7);
  bool IsLoad = fieldFromInstruction(insn, 22, 1);

  // offset is a 7-bit signed immediate, so sign extend it to
  // fill the unsigned.
  if (offset & (1 << (7 - 1)))
    offset |= ~((1LL << 7) - 1);

  unsigned Opcode = Inst.getOpcode();
  bool NeedsDisjointWritebackTransfer = false;

  // First operand is always writeback of base register.
  switch (Opcode) {
  default:
    break;
  case AArch64::LDPXpost:
  case AArch64::STPXpost:
  case AArch64::LDPSWpost:
  case AArch64::LDPXpre:
  case AArch64::STPXpre:
  case AArch64::LDPSWpre:
  case AArch64::LDPWpost:
  case AArch64::STPWpost:
  case AArch64::LDPWpre:
  case AArch64::STPWpre:
  case AArch64::LDPQpost:
  case AArch64::STPQpost:
  case AArch64::LDPQpre:
  case AArch64::STPQpre:
  case AArch64::LDPDpost:
  case AArch64::STPDpost:
  case AArch64::LDPDpre:
  case AArch64::STPDpre:
  case AArch64::LDPSpost:
  case AArch64::STPSpost:
  case AArch64::LDPSpre:
  case AArch64::STPSpre:
  case AArch64::STGPpre:
  case AArch64::STGPpost:
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
    break;
  }

  switch (Opcode) {
  default:
    return Fail;
  case AArch64::LDPXpost:
  case AArch64::STPXpost:
  case AArch64::LDPSWpost:
  case AArch64::LDPXpre:
  case AArch64::STPXpre:
  case AArch64::LDPSWpre:
  case AArch64::STGPpre:
  case AArch64::STGPpost:
    NeedsDisjointWritebackTransfer = true;
    LLVM_FALLTHROUGH;
  case AArch64::LDNPXi:
  case AArch64::STNPXi:
  case AArch64::LDPXi:
  case AArch64::STPXi:
  case AArch64::LDPSWi:
  case AArch64::STGPi:
    DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  case AArch64::LDPWpost:
  case AArch64::STPWpost:
  case AArch64::LDPWpre:
  case AArch64::STPWpre:
    NeedsDisjointWritebackTransfer = true;
    LLVM_FALLTHROUGH;
  case AArch64::LDNPWi:
  case AArch64::STNPWi:
  case AArch64::LDPWi:
  case AArch64::STPWi:
    DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  case AArch64::LDNPQi:
  case AArch64::STNPQi:
  case AArch64::LDPQpost:
  case AArch64::STPQpost:
  case AArch64::LDPQi:
  case AArch64::STPQi:
  case AArch64::LDPQpre:
  case AArch64::STPQpre:
    DecodeFPR128RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeFPR128RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  case AArch64::LDNPDi:
  case AArch64::STNPDi:
  case AArch64::LDPDpost:
  case AArch64::STPDpost:
  case AArch64::LDPDi:
  case AArch64::STPDi:
  case AArch64::LDPDpre:
  case AArch64::STPDpre:
    DecodeFPR64RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeFPR64RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  case AArch64::LDNPSi:
  case AArch64::STNPSi:
  case AArch64::LDPSpost:
  case AArch64::STPSpost:
  case AArch64::LDPSi:
  case AArch64::STPSi:
  case AArch64::LDPSpre:
  case AArch64::STPSpre:
    DecodeFPR32RegisterClass(Inst, Rt, Addr, Decoder);
    DecodeFPR32RegisterClass(Inst, Rt2, Addr, Decoder);
    break;
  }

  DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
  Inst.addOperand(MCOperand::createImm(offset));

  // You shouldn't load to the same register twice in an instruction...
  if (IsLoad && Rt == Rt2)
    return SoftFail;

  // ... or do any operation that writes-back to a transfer register. But note
  // that "stp xzr, xzr, [sp], #4" is fine because xzr and sp are different.
  if (NeedsDisjointWritebackTransfer && Rn != 31 && (Rt == Rn || Rt2 == Rn))
    return SoftFail;

  return Success;
}

static DecodeStatus DecodeAuthLoadInstruction(MCInst &Inst, uint32_t insn,
                                              uint64_t Addr,
                                              const void *Decoder) {
  unsigned Rt = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  uint64_t offset = fieldFromInstruction(insn, 22, 1) << 9 |
                    fieldFromInstruction(insn, 12, 9);
  unsigned writeback = fieldFromInstruction(insn, 11, 1);

  switch (Inst.getOpcode()) {
  default:
    return Fail;
  case AArch64::LDRAAwriteback:
  case AArch64::LDRABwriteback:
    DecodeGPR64spRegisterClass(Inst, Rn /* writeback register */, Addr,
                               Decoder);
    break;
  case AArch64::LDRAAindexed:
  case AArch64::LDRABindexed:
    break;
  }

  DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
  DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
  DecodeSImm<10>(Inst, offset, Addr, Decoder);

  if (writeback && Rt == Rn && Rn != 31) {
    return SoftFail;
  }

  return Success;
}

static DecodeStatus DecodeAddSubERegInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Addr,
                                                const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned Rm = fieldFromInstruction(insn, 16, 5);
  unsigned extend = fieldFromInstruction(insn, 10, 6);

  unsigned shift = extend & 0x7;
  if (shift > 4)
    return Fail;

  switch (Inst.getOpcode()) {
  default:
    return Fail;
  case AArch64::ADDWrx:
  case AArch64::SUBWrx:
    DecodeGPR32spRegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR32spRegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  case AArch64::ADDSWrx:
  case AArch64::SUBSWrx:
    DecodeGPR32RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR32spRegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  case AArch64::ADDXrx:
  case AArch64::SUBXrx:
    DecodeGPR64spRegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  case AArch64::ADDSXrx:
  case AArch64::SUBSXrx:
    DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  case AArch64::ADDXrx64:
  case AArch64::SUBXrx64:
    DecodeGPR64spRegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  case AArch64::SUBSXrx64:
  case AArch64::ADDSXrx64:
    DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rm, Addr, Decoder);
    break;
  }

  Inst.addOperand(MCOperand::createImm(extend));
  return Success;
}

static DecodeStatus DecodeLogicalImmInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Addr,
                                                const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned Datasize = fieldFromInstruction(insn, 31, 1);
  unsigned imm;

  if (Datasize) {
    if (Inst.getOpcode() == AArch64::ANDSXri)
      DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
    else
      DecodeGPR64spRegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64RegisterClass(Inst, Rn, Addr, Decoder);
    imm = fieldFromInstruction(insn, 10, 13);
    if (!AArch64_AM::isValidDecodeLogicalImmediate(imm, 64))
      return Fail;
  } else {
    if (Inst.getOpcode() == AArch64::ANDSWri)
      DecodeGPR32RegisterClass(Inst, Rd, Addr, Decoder);
    else
      DecodeGPR32spRegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR32RegisterClass(Inst, Rn, Addr, Decoder);
    imm = fieldFromInstruction(insn, 10, 12);
    if (!AArch64_AM::isValidDecodeLogicalImmediate(imm, 32))
      return Fail;
  }
  Inst.addOperand(MCOperand::createImm(imm));
  return Success;
}

static DecodeStatus DecodeModImmInstruction(MCInst &Inst, uint32_t insn,
                                            uint64_t Addr,
                                            const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned cmode = fieldFromInstruction(insn, 12, 4);
  unsigned imm = fieldFromInstruction(insn, 16, 3) << 5;
  imm |= fieldFromInstruction(insn, 5, 5);

  if (Inst.getOpcode() == AArch64::MOVID)
    DecodeFPR64RegisterClass(Inst, Rd, Addr, Decoder);
  else
    DecodeVectorRegisterClass(Inst, Rd, Addr, Decoder);

  Inst.addOperand(MCOperand::createImm(imm));

  switch (Inst.getOpcode()) {
  default:
    break;
  case AArch64::MOVIv4i16:
  case AArch64::MOVIv8i16:
  case AArch64::MVNIv4i16:
  case AArch64::MVNIv8i16:
  case AArch64::MOVIv2i32:
  case AArch64::MOVIv4i32:
  case AArch64::MVNIv2i32:
  case AArch64::MVNIv4i32:
    Inst.addOperand(MCOperand::createImm((cmode & 6) << 2));
    break;
  case AArch64::MOVIv2s_msl:
  case AArch64::MOVIv4s_msl:
  case AArch64::MVNIv2s_msl:
  case AArch64::MVNIv4s_msl:
    Inst.addOperand(MCOperand::createImm((cmode & 1) ? 0x110 : 0x108));
    break;
  }

  return Success;
}

static DecodeStatus DecodeModImmTiedInstruction(MCInst &Inst, uint32_t insn,
                                                uint64_t Addr,
                                                const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned cmode = fieldFromInstruction(insn, 12, 4);
  unsigned imm = fieldFromInstruction(insn, 16, 3) << 5;
  imm |= fieldFromInstruction(insn, 5, 5);

  // Tied operands added twice.
  DecodeVectorRegisterClass(Inst, Rd, Addr, Decoder);
  DecodeVectorRegisterClass(Inst, Rd, Addr, Decoder);

  Inst.addOperand(MCOperand::createImm(imm));
  Inst.addOperand(MCOperand::createImm((cmode & 6) << 2));

  return Success;
}

static DecodeStatus DecodeAdrInstruction(MCInst &Inst, uint32_t insn,
                                         uint64_t Addr, const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  int64_t imm = fieldFromInstruction(insn, 5, 19) << 2;
  imm |= fieldFromInstruction(insn, 29, 2);
  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);

  // Sign-extend the 21-bit immediate.
  if (imm & (1 << (21 - 1)))
    imm |= ~((1LL << 21) - 1);

  DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
  if (!Dis->tryAddingSymbolicOperand(Inst, imm, Addr, Fail, 0, 4))
    Inst.addOperand(MCOperand::createImm(imm));

  return Success;
}

static DecodeStatus DecodeAddSubImmShift(MCInst &Inst, uint32_t insn,
                                         uint64_t Addr, const void *Decoder) {
  unsigned Rd = fieldFromInstruction(insn, 0, 5);
  unsigned Rn = fieldFromInstruction(insn, 5, 5);
  unsigned Imm = fieldFromInstruction(insn, 10, 14);
  unsigned S = fieldFromInstruction(insn, 29, 1);
  unsigned Datasize = fieldFromInstruction(insn, 31, 1);

  unsigned ShifterVal = (Imm >> 12) & 3;
  unsigned ImmVal = Imm & 0xFFF;
  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);

  if (ShifterVal != 0 && ShifterVal != 1)
    return Fail;

  if (Datasize) {
    if (Rd == 31 && !S)
      DecodeGPR64spRegisterClass(Inst, Rd, Addr, Decoder);
    else
      DecodeGPR64RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR64spRegisterClass(Inst, Rn, Addr, Decoder);
  } else {
    if (Rd == 31 && !S)
      DecodeGPR32spRegisterClass(Inst, Rd, Addr, Decoder);
    else
      DecodeGPR32RegisterClass(Inst, Rd, Addr, Decoder);
    DecodeGPR32spRegisterClass(Inst, Rn, Addr, Decoder);
  }

  if (!Dis->tryAddingSymbolicOperand(Inst, Imm, Addr, Fail, 0, 4))
    Inst.addOperand(MCOperand::createImm(ImmVal));
  Inst.addOperand(MCOperand::createImm(12 * ShifterVal));
  return Success;
}

static DecodeStatus DecodeUnconditionalBranch(MCInst &Inst, uint32_t insn,
                                              uint64_t Addr,
                                              const void *Decoder) {
  int64_t imm = fieldFromInstruction(insn, 0, 26);
  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);

  // Sign-extend the 26-bit immediate.
  if (imm & (1 << (26 - 1)))
    imm |= ~((1LL << 26) - 1);

  if (!Dis->tryAddingSymbolicOperand(Inst, imm * 4, Addr, true, 0, 4))
    Inst.addOperand(MCOperand::createImm(imm));

  return Success;
}

static DecodeStatus DecodeSystemPStateInstruction(MCInst &Inst, uint32_t insn,
                                                  uint64_t Addr,
                                                  const void *Decoder) {
  uint64_t op1 = fieldFromInstruction(insn, 16, 3);
  uint64_t op2 = fieldFromInstruction(insn, 5, 3);
  uint64_t crm = fieldFromInstruction(insn, 8, 4);
  uint64_t pstate_field = (op1 << 3) | op2;

  switch (pstate_field) {
  case 0x01: // XAFlag
  case 0x02: // AXFlag
    return Fail;
  }

  if ((pstate_field == AArch64PState::PAN  ||
       pstate_field == AArch64PState::UAO  ||
       pstate_field == AArch64PState::SSBS) && crm > 1)
    return Fail;

  Inst.addOperand(MCOperand::createImm(pstate_field));
  Inst.addOperand(MCOperand::createImm(crm));

  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);
  auto PState = AArch64PState::lookupPStateByEncoding(pstate_field);
  if (PState && PState->haveFeatures(Dis->getSubtargetInfo().getFeatureBits()))
    return Success;
  return Fail;
}

static DecodeStatus DecodeTestAndBranch(MCInst &Inst, uint32_t insn,
                                        uint64_t Addr, const void *Decoder) {
  uint64_t Rt = fieldFromInstruction(insn, 0, 5);
  uint64_t bit = fieldFromInstruction(insn, 31, 1) << 5;
  bit |= fieldFromInstruction(insn, 19, 5);
  int64_t dst = fieldFromInstruction(insn, 5, 14);
  const AArch64Disassembler *Dis =
      static_cast<const AArch64Disassembler *>(Decoder);

  // Sign-extend 14-bit immediate.
  if (dst & (1 << (14 - 1)))
    dst |= ~((1LL << 14) - 1);

  if (fieldFromInstruction(insn, 31, 1) == 0)
    DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder);
  else
    DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder);
  Inst.addOperand(MCOperand::createImm(bit));
  if (!Dis->tryAddingSymbolicOperand(Inst, dst * 4, Addr, true, 0, 4))
    Inst.addOperand(MCOperand::createImm(dst));

  return Success;
}

static DecodeStatus DecodeGPRSeqPairsClassRegisterClass(MCInst &Inst,
                                                        unsigned RegClassID,
                                                        unsigned RegNo,
                                                        uint64_t Addr,
                                                        const void *Decoder) {
  // Register number must be even (see CASP instruction)
  if (RegNo & 0x1)
    return Fail;

  unsigned Reg = AArch64MCRegisterClasses[RegClassID].getRegister(RegNo / 2);
  Inst.addOperand(MCOperand::createReg(Reg));
  return Success;
}

static DecodeStatus DecodeWSeqPairsClassRegisterClass(MCInst &Inst,
                                                      unsigned RegNo,
                                                      uint64_t Addr,
                                                      const void *Decoder) {
  return DecodeGPRSeqPairsClassRegisterClass(Inst,
                                             AArch64::WSeqPairsClassRegClassID,
                                             RegNo, Addr, Decoder);
}

static DecodeStatus DecodeXSeqPairsClassRegisterClass(MCInst &Inst,
                                                      unsigned RegNo,
                                                      uint64_t Addr,
                                                      const void *Decoder) {
  return DecodeGPRSeqPairsClassRegisterClass(Inst,
                                             AArch64::XSeqPairsClassRegClassID,
                                             RegNo, Addr, Decoder);
}

static DecodeStatus DecodeSVELogicalImmInstruction(llvm::MCInst &Inst,
                                                   uint32_t insn,
                                                   uint64_t Addr,
                                                   const void *Decoder) {
  unsigned Zdn = fieldFromInstruction(insn, 0, 5);
  unsigned imm = fieldFromInstruction(insn, 5, 13);
  if (!AArch64_AM::isValidDecodeLogicalImmediate(imm, 64))
    return Fail;

  // The same (tied) operand is added twice to the instruction.
  DecodeZPRRegisterClass(Inst, Zdn, Addr, Decoder);
  if (Inst.getOpcode() != AArch64::DUPM_ZI)
    DecodeZPRRegisterClass(Inst, Zdn, Addr, Decoder);
  Inst.addOperand(MCOperand::createImm(imm));
  return Success;
}

template<int Bits>
static DecodeStatus DecodeSImm(llvm::MCInst &Inst, uint64_t Imm,
                               uint64_t Address, const void *Decoder) {
  if (Imm & ~((1LL << Bits) - 1))
      return Fail;

  // Imm is a signed immediate, so sign extend it.
  if (Imm & (1 << (Bits - 1)))
    Imm |= ~((1LL << Bits) - 1);

  Inst.addOperand(MCOperand::createImm(Imm));
  return Success;
}

// Decode 8-bit signed/unsigned immediate for a given element width.
template <int ElementWidth>
static DecodeStatus DecodeImm8OptLsl(MCInst &Inst, unsigned Imm,
                                      uint64_t Addr, const void *Decoder) {
  unsigned Val = (uint8_t)Imm;
  unsigned Shift = (Imm & 0x100) ? 8 : 0;
  if (ElementWidth == 8 && Shift)
    return Fail;
  Inst.addOperand(MCOperand::createImm(Val));
  Inst.addOperand(MCOperand::createImm(Shift));
  return Success;
}

// Decode uimm4 ranged from 1-16.
static DecodeStatus DecodeSVEIncDecImm(MCInst &Inst, unsigned Imm,
                                       uint64_t Addr, const void *Decoder) {
  Inst.addOperand(MCOperand::createImm(Imm + 1));
  return Success;
}

static DecodeStatus DecodeSVCROp(MCInst &Inst, unsigned Imm, uint64_t Address,
                                 const void *Decoder) {
  if (AArch64SVCR::lookupSVCRByEncoding(Imm)) {
    Inst.addOperand(MCOperand::createImm(Imm));
    return Success;
  }
  return Fail;
}