1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a printer that converts from our internal representation 10 // of machine-dependent LLVM code to the AArch64 assembly language. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64.h" 15 #include "AArch64MCInstLower.h" 16 #include "AArch64MachineFunctionInfo.h" 17 #include "AArch64RegisterInfo.h" 18 #include "AArch64Subtarget.h" 19 #include "AArch64TargetObjectFile.h" 20 #include "MCTargetDesc/AArch64AddressingModes.h" 21 #include "MCTargetDesc/AArch64InstPrinter.h" 22 #include "MCTargetDesc/AArch64MCExpr.h" 23 #include "MCTargetDesc/AArch64MCTargetDesc.h" 24 #include "MCTargetDesc/AArch64TargetStreamer.h" 25 #include "TargetInfo/AArch64TargetInfo.h" 26 #include "Utils/AArch64BaseInfo.h" 27 #include "llvm/ADT/SmallString.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/ADT/Triple.h" 31 #include "llvm/ADT/Twine.h" 32 #include "llvm/BinaryFormat/COFF.h" 33 #include "llvm/BinaryFormat/ELF.h" 34 #include "llvm/CodeGen/AsmPrinter.h" 35 #include "llvm/CodeGen/FaultMaps.h" 36 #include "llvm/CodeGen/MachineBasicBlock.h" 37 #include "llvm/CodeGen/MachineFunction.h" 38 #include "llvm/CodeGen/MachineInstr.h" 39 #include "llvm/CodeGen/MachineJumpTableInfo.h" 40 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 41 #include "llvm/CodeGen/MachineOperand.h" 42 #include "llvm/CodeGen/StackMaps.h" 43 #include "llvm/CodeGen/TargetRegisterInfo.h" 44 #include "llvm/IR/DataLayout.h" 45 #include "llvm/IR/DebugInfoMetadata.h" 46 #include "llvm/MC/MCAsmInfo.h" 47 #include "llvm/MC/MCContext.h" 48 #include "llvm/MC/MCInst.h" 49 #include "llvm/MC/MCInstBuilder.h" 50 #include "llvm/MC/MCSectionELF.h" 51 #include "llvm/MC/MCStreamer.h" 52 #include "llvm/MC/MCSymbol.h" 53 #include "llvm/Support/Casting.h" 54 #include "llvm/Support/ErrorHandling.h" 55 #include "llvm/Support/TargetRegistry.h" 56 #include "llvm/Support/raw_ostream.h" 57 #include "llvm/Target/TargetMachine.h" 58 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 59 #include <algorithm> 60 #include <cassert> 61 #include <cstdint> 62 #include <map> 63 #include <memory> 64 65 using namespace llvm; 66 67 #define DEBUG_TYPE "asm-printer" 68 69 namespace { 70 71 class AArch64AsmPrinter : public AsmPrinter { 72 AArch64MCInstLower MCInstLowering; 73 StackMaps SM; 74 FaultMaps FM; 75 const AArch64Subtarget *STI; 76 77 public: 78 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) 79 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this), 80 SM(*this), FM(*this) {} 81 82 StringRef getPassName() const override { return "AArch64 Assembly Printer"; } 83 84 /// Wrapper for MCInstLowering.lowerOperand() for the 85 /// tblgen'erated pseudo lowering. 86 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const { 87 return MCInstLowering.lowerOperand(MO, MCOp); 88 } 89 90 void emitStartOfAsmFile(Module &M) override; 91 void emitJumpTableInfo() override; 92 93 void emitFunctionEntryLabel() override; 94 95 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI); 96 97 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 98 const MachineInstr &MI); 99 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 100 const MachineInstr &MI); 101 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 102 const MachineInstr &MI); 103 void LowerFAULTING_OP(const MachineInstr &MI); 104 105 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); 106 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); 107 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); 108 109 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple; 110 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; 111 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); 112 void emitHwasanMemaccessSymbols(Module &M); 113 114 void emitSled(const MachineInstr &MI, SledKind Kind); 115 116 /// tblgen'erated driver function for lowering simple MI->MC 117 /// pseudo instructions. 118 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, 119 const MachineInstr *MI); 120 121 void emitInstruction(const MachineInstr *MI) override; 122 123 void emitFunctionHeaderComment() override; 124 125 void getAnalysisUsage(AnalysisUsage &AU) const override { 126 AsmPrinter::getAnalysisUsage(AU); 127 AU.setPreservesAll(); 128 } 129 130 bool runOnMachineFunction(MachineFunction &MF) override { 131 AArch64FI = MF.getInfo<AArch64FunctionInfo>(); 132 STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget()); 133 134 SetupMachineFunction(MF); 135 136 if (STI->isTargetCOFF()) { 137 bool Internal = MF.getFunction().hasInternalLinkage(); 138 COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC 139 : COFF::IMAGE_SYM_CLASS_EXTERNAL; 140 int Type = 141 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT; 142 143 OutStreamer->BeginCOFFSymbolDef(CurrentFnSym); 144 OutStreamer->EmitCOFFSymbolStorageClass(Scl); 145 OutStreamer->EmitCOFFSymbolType(Type); 146 OutStreamer->EndCOFFSymbolDef(); 147 } 148 149 // Emit the rest of the function body. 150 emitFunctionBody(); 151 152 // Emit the XRay table for this function. 153 emitXRayTable(); 154 155 // We didn't modify anything. 156 return false; 157 } 158 159 private: 160 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); 161 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O); 162 bool printAsmRegInClass(const MachineOperand &MO, 163 const TargetRegisterClass *RC, unsigned AltName, 164 raw_ostream &O); 165 166 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, 167 const char *ExtraCode, raw_ostream &O) override; 168 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, 169 const char *ExtraCode, raw_ostream &O) override; 170 171 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS); 172 173 void emitFunctionBodyEnd() override; 174 175 MCSymbol *GetCPISymbol(unsigned CPID) const override; 176 void emitEndOfAsmFile(Module &M) override; 177 178 AArch64FunctionInfo *AArch64FI = nullptr; 179 180 /// Emit the LOHs contained in AArch64FI. 181 void emitLOHs(); 182 183 /// Emit instruction to set float register to zero. 184 void emitFMov0(const MachineInstr &MI); 185 186 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; 187 188 MInstToMCSymbol LOHInstToLabel; 189 }; 190 191 } // end anonymous namespace 192 193 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) { 194 const Triple &TT = TM.getTargetTriple(); 195 196 if (TT.isOSBinFormatCOFF()) { 197 // Emit an absolute @feat.00 symbol. This appears to be some kind of 198 // compiler features bitfield read by link.exe. 199 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00")); 200 OutStreamer->BeginCOFFSymbolDef(S); 201 OutStreamer->EmitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC); 202 OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL); 203 OutStreamer->EndCOFFSymbolDef(); 204 int64_t Feat00Flags = 0; 205 206 if (M.getModuleFlag("cfguard")) { 207 Feat00Flags |= 0x800; // Object is CFG-aware. 208 } 209 210 if (M.getModuleFlag("ehcontguard")) { 211 Feat00Flags |= 0x4000; // Object also has EHCont. 212 } 213 214 OutStreamer->emitSymbolAttribute(S, MCSA_Global); 215 OutStreamer->emitAssignment( 216 S, MCConstantExpr::create(Feat00Flags, MMI->getContext())); 217 } 218 219 if (!TT.isOSBinFormatELF()) 220 return; 221 222 // Assemble feature flags that may require creation of a note section. 223 unsigned Flags = 0; 224 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>( 225 M.getModuleFlag("branch-target-enforcement"))) 226 if (BTE->getZExtValue()) 227 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI; 228 229 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>( 230 M.getModuleFlag("sign-return-address"))) 231 if (Sign->getZExtValue()) 232 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC; 233 234 if (Flags == 0) 235 return; 236 237 // Emit a .note.gnu.property section with the flags. 238 if (auto *TS = static_cast<AArch64TargetStreamer *>( 239 OutStreamer->getTargetStreamer())) 240 TS->emitNoteSection(Flags); 241 } 242 243 void AArch64AsmPrinter::emitFunctionHeaderComment() { 244 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>(); 245 Optional<std::string> OutlinerString = FI->getOutliningStyle(); 246 if (OutlinerString != None) 247 OutStreamer->GetCommentOS() << ' ' << OutlinerString; 248 } 249 250 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) 251 { 252 const Function &F = MF->getFunction(); 253 if (F.hasFnAttribute("patchable-function-entry")) { 254 unsigned Num; 255 if (F.getFnAttribute("patchable-function-entry") 256 .getValueAsString() 257 .getAsInteger(10, Num)) 258 return; 259 emitNops(Num); 260 return; 261 } 262 263 emitSled(MI, SledKind::FUNCTION_ENTER); 264 } 265 266 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) { 267 emitSled(MI, SledKind::FUNCTION_EXIT); 268 } 269 270 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) { 271 emitSled(MI, SledKind::TAIL_CALL); 272 } 273 274 void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) { 275 static const int8_t NoopsInSledCount = 7; 276 // We want to emit the following pattern: 277 // 278 // .Lxray_sled_N: 279 // ALIGN 280 // B #32 281 // ; 7 NOP instructions (28 bytes) 282 // .tmpN 283 // 284 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching 285 // over the full 32 bytes (8 instructions) with the following pattern: 286 // 287 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack 288 // LDR W0, #12 ; W0 := function ID 289 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit 290 // BLR X16 ; call the tracing trampoline 291 // ;DATA: 32 bits of function ID 292 // ;DATA: lower 32 bits of the address of the trampoline 293 // ;DATA: higher 32 bits of the address of the trampoline 294 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack 295 // 296 OutStreamer->emitCodeAlignment(4); 297 auto CurSled = OutContext.createTempSymbol("xray_sled_", true); 298 OutStreamer->emitLabel(CurSled); 299 auto Target = OutContext.createTempSymbol(); 300 301 // Emit "B #32" instruction, which jumps over the next 28 bytes. 302 // The operand has to be the number of 4-byte instructions to jump over, 303 // including the current instruction. 304 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8)); 305 306 for (int8_t I = 0; I < NoopsInSledCount; I++) 307 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 308 309 OutStreamer->emitLabel(Target); 310 recordSled(CurSled, MI, Kind, 2); 311 } 312 313 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 314 Register Reg = MI.getOperand(0).getReg(); 315 bool IsShort = 316 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES; 317 uint32_t AccessInfo = MI.getOperand(1).getImm(); 318 MCSymbol *&Sym = 319 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)]; 320 if (!Sym) { 321 // FIXME: Make this work on non-ELF. 322 if (!TM.getTargetTriple().isOSBinFormatELF()) 323 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); 324 325 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" + 326 utostr(AccessInfo); 327 if (IsShort) 328 SymName += "_short_v2"; 329 Sym = OutContext.getOrCreateSymbol(SymName); 330 } 331 332 EmitToStreamer(*OutStreamer, 333 MCInstBuilder(AArch64::BL) 334 .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); 335 } 336 337 void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) { 338 if (HwasanMemaccessSymbols.empty()) 339 return; 340 341 const Triple &TT = TM.getTargetTriple(); 342 assert(TT.isOSBinFormatELF()); 343 std::unique_ptr<MCSubtargetInfo> STI( 344 TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); 345 assert(STI && "Unable to create subtarget info"); 346 347 MCSymbol *HwasanTagMismatchV1Sym = 348 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch"); 349 MCSymbol *HwasanTagMismatchV2Sym = 350 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2"); 351 352 const MCSymbolRefExpr *HwasanTagMismatchV1Ref = 353 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext); 354 const MCSymbolRefExpr *HwasanTagMismatchV2Ref = 355 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext); 356 357 for (auto &P : HwasanMemaccessSymbols) { 358 unsigned Reg = std::get<0>(P.first); 359 bool IsShort = std::get<1>(P.first); 360 uint32_t AccessInfo = std::get<2>(P.first); 361 const MCSymbolRefExpr *HwasanTagMismatchRef = 362 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref; 363 MCSymbol *Sym = P.second; 364 365 bool HasMatchAllTag = 366 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1; 367 uint8_t MatchAllTag = 368 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff; 369 unsigned Size = 370 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); 371 bool CompileKernel = 372 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1; 373 374 OutStreamer->SwitchSection(OutContext.getELFSection( 375 ".text.hot", ELF::SHT_PROGBITS, 376 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, 377 Sym->getName(), /*IsComdat=*/true)); 378 379 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); 380 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); 381 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); 382 OutStreamer->emitLabel(Sym); 383 384 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri) 385 .addReg(AArch64::X16) 386 .addReg(Reg) 387 .addImm(4) 388 .addImm(55), 389 *STI); 390 OutStreamer->emitInstruction( 391 MCInstBuilder(AArch64::LDRBBroX) 392 .addReg(AArch64::W16) 393 .addReg(IsShort ? AArch64::X20 : AArch64::X9) 394 .addReg(AArch64::X16) 395 .addImm(0) 396 .addImm(0), 397 *STI); 398 OutStreamer->emitInstruction( 399 MCInstBuilder(AArch64::SUBSXrs) 400 .addReg(AArch64::XZR) 401 .addReg(AArch64::X16) 402 .addReg(Reg) 403 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), 404 *STI); 405 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); 406 OutStreamer->emitInstruction( 407 MCInstBuilder(AArch64::Bcc) 408 .addImm(AArch64CC::NE) 409 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, 410 OutContext)), 411 *STI); 412 MCSymbol *ReturnSym = OutContext.createTempSymbol(); 413 OutStreamer->emitLabel(ReturnSym); 414 OutStreamer->emitInstruction( 415 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); 416 OutStreamer->emitLabel(HandleMismatchOrPartialSym); 417 418 if (HasMatchAllTag) { 419 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri) 420 .addReg(AArch64::X16) 421 .addReg(Reg) 422 .addImm(56) 423 .addImm(63), 424 *STI); 425 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri) 426 .addReg(AArch64::XZR) 427 .addReg(AArch64::X16) 428 .addImm(MatchAllTag) 429 .addImm(0), 430 *STI); 431 OutStreamer->emitInstruction( 432 MCInstBuilder(AArch64::Bcc) 433 .addImm(AArch64CC::EQ) 434 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 435 *STI); 436 } 437 438 if (IsShort) { 439 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri) 440 .addReg(AArch64::WZR) 441 .addReg(AArch64::W16) 442 .addImm(15) 443 .addImm(0), 444 *STI); 445 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); 446 OutStreamer->emitInstruction( 447 MCInstBuilder(AArch64::Bcc) 448 .addImm(AArch64CC::HI) 449 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 450 *STI); 451 452 OutStreamer->emitInstruction( 453 MCInstBuilder(AArch64::ANDXri) 454 .addReg(AArch64::X17) 455 .addReg(Reg) 456 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), 457 *STI); 458 if (Size != 1) 459 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 460 .addReg(AArch64::X17) 461 .addReg(AArch64::X17) 462 .addImm(Size - 1) 463 .addImm(0), 464 *STI); 465 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs) 466 .addReg(AArch64::WZR) 467 .addReg(AArch64::W16) 468 .addReg(AArch64::W17) 469 .addImm(0), 470 *STI); 471 OutStreamer->emitInstruction( 472 MCInstBuilder(AArch64::Bcc) 473 .addImm(AArch64CC::LS) 474 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 475 *STI); 476 477 OutStreamer->emitInstruction( 478 MCInstBuilder(AArch64::ORRXri) 479 .addReg(AArch64::X16) 480 .addReg(Reg) 481 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), 482 *STI); 483 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui) 484 .addReg(AArch64::W16) 485 .addReg(AArch64::X16) 486 .addImm(0), 487 *STI); 488 OutStreamer->emitInstruction( 489 MCInstBuilder(AArch64::SUBSXrs) 490 .addReg(AArch64::XZR) 491 .addReg(AArch64::X16) 492 .addReg(Reg) 493 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), 494 *STI); 495 OutStreamer->emitInstruction( 496 MCInstBuilder(AArch64::Bcc) 497 .addImm(AArch64CC::EQ) 498 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 499 *STI); 500 501 OutStreamer->emitLabel(HandleMismatchSym); 502 } 503 504 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 505 .addReg(AArch64::SP) 506 .addReg(AArch64::X0) 507 .addReg(AArch64::X1) 508 .addReg(AArch64::SP) 509 .addImm(-32), 510 *STI); 511 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) 512 .addReg(AArch64::FP) 513 .addReg(AArch64::LR) 514 .addReg(AArch64::SP) 515 .addImm(29), 516 *STI); 517 518 if (Reg != AArch64::X0) 519 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs) 520 .addReg(AArch64::X0) 521 .addReg(AArch64::XZR) 522 .addReg(Reg) 523 .addImm(0), 524 *STI); 525 OutStreamer->emitInstruction( 526 MCInstBuilder(AArch64::MOVZXi) 527 .addReg(AArch64::X1) 528 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask) 529 .addImm(0), 530 *STI); 531 532 if (CompileKernel) { 533 // The Linux kernel's dynamic loader doesn't support GOT relative 534 // relocations, but it doesn't support late binding either, so just call 535 // the function directly. 536 OutStreamer->emitInstruction( 537 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI); 538 } else { 539 // Intentionally load the GOT entry and branch to it, rather than possibly 540 // late binding the function, which may clobber the registers before we 541 // have a chance to save them. 542 OutStreamer->emitInstruction( 543 MCInstBuilder(AArch64::ADRP) 544 .addReg(AArch64::X16) 545 .addExpr(AArch64MCExpr::create( 546 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE, 547 OutContext)), 548 *STI); 549 OutStreamer->emitInstruction( 550 MCInstBuilder(AArch64::LDRXui) 551 .addReg(AArch64::X16) 552 .addReg(AArch64::X16) 553 .addExpr(AArch64MCExpr::create( 554 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12, 555 OutContext)), 556 *STI); 557 OutStreamer->emitInstruction( 558 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); 559 } 560 } 561 } 562 563 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) { 564 emitHwasanMemaccessSymbols(M); 565 566 const Triple &TT = TM.getTargetTriple(); 567 if (TT.isOSBinFormatMachO()) { 568 // Funny Darwin hack: This flag tells the linker that no global symbols 569 // contain code that falls through to other global symbols (e.g. the obvious 570 // implementation of multiple entry points). If this doesn't occur, the 571 // linker can safely perform dead code stripping. Since LLVM never 572 // generates code that does this, it is always safe to set. 573 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols); 574 } 575 576 // Emit stack and fault map information. 577 emitStackMaps(SM); 578 FM.serializeToFaultMapSection(); 579 580 } 581 582 void AArch64AsmPrinter::emitLOHs() { 583 SmallVector<MCSymbol *, 3> MCArgs; 584 585 for (const auto &D : AArch64FI->getLOHContainer()) { 586 for (const MachineInstr *MI : D.getArgs()) { 587 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI); 588 assert(LabelIt != LOHInstToLabel.end() && 589 "Label hasn't been inserted for LOH related instruction"); 590 MCArgs.push_back(LabelIt->second); 591 } 592 OutStreamer->emitLOHDirective(D.getKind(), MCArgs); 593 MCArgs.clear(); 594 } 595 } 596 597 void AArch64AsmPrinter::emitFunctionBodyEnd() { 598 if (!AArch64FI->getLOHRelated().empty()) 599 emitLOHs(); 600 } 601 602 /// GetCPISymbol - Return the symbol for the specified constant pool entry. 603 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const { 604 // Darwin uses a linker-private symbol name for constant-pools (to 605 // avoid addends on the relocation?), ELF has no such concept and 606 // uses a normal private symbol. 607 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty()) 608 return OutContext.getOrCreateSymbol( 609 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" + 610 Twine(getFunctionNumber()) + "_" + Twine(CPID)); 611 612 return AsmPrinter::GetCPISymbol(CPID); 613 } 614 615 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, 616 raw_ostream &O) { 617 const MachineOperand &MO = MI->getOperand(OpNum); 618 switch (MO.getType()) { 619 default: 620 llvm_unreachable("<unknown operand type>"); 621 case MachineOperand::MO_Register: { 622 Register Reg = MO.getReg(); 623 assert(Register::isPhysicalRegister(Reg)); 624 assert(!MO.getSubReg() && "Subregs should be eliminated!"); 625 O << AArch64InstPrinter::getRegisterName(Reg); 626 break; 627 } 628 case MachineOperand::MO_Immediate: { 629 O << MO.getImm(); 630 break; 631 } 632 case MachineOperand::MO_GlobalAddress: { 633 PrintSymbolOperand(MO, O); 634 break; 635 } 636 case MachineOperand::MO_BlockAddress: { 637 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); 638 Sym->print(O, MAI); 639 break; 640 } 641 } 642 } 643 644 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, 645 raw_ostream &O) { 646 Register Reg = MO.getReg(); 647 switch (Mode) { 648 default: 649 return true; // Unknown mode. 650 case 'w': 651 Reg = getWRegFromXReg(Reg); 652 break; 653 case 'x': 654 Reg = getXRegFromWReg(Reg); 655 break; 656 case 't': 657 Reg = getXRegFromXRegTuple(Reg); 658 break; 659 } 660 661 O << AArch64InstPrinter::getRegisterName(Reg); 662 return false; 663 } 664 665 // Prints the register in MO using class RC using the offset in the 666 // new register class. This should not be used for cross class 667 // printing. 668 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO, 669 const TargetRegisterClass *RC, 670 unsigned AltName, raw_ostream &O) { 671 assert(MO.isReg() && "Should only get here with a register!"); 672 const TargetRegisterInfo *RI = STI->getRegisterInfo(); 673 Register Reg = MO.getReg(); 674 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg)); 675 if (!RI->regsOverlap(RegToPrint, Reg)) 676 return true; 677 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName); 678 return false; 679 } 680 681 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, 682 const char *ExtraCode, raw_ostream &O) { 683 const MachineOperand &MO = MI->getOperand(OpNum); 684 685 // First try the generic code, which knows about modifiers like 'c' and 'n'. 686 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O)) 687 return false; 688 689 // Does this asm operand have a single letter operand modifier? 690 if (ExtraCode && ExtraCode[0]) { 691 if (ExtraCode[1] != 0) 692 return true; // Unknown modifier. 693 694 switch (ExtraCode[0]) { 695 default: 696 return true; // Unknown modifier. 697 case 'w': // Print W register 698 case 'x': // Print X register 699 if (MO.isReg()) 700 return printAsmMRegister(MO, ExtraCode[0], O); 701 if (MO.isImm() && MO.getImm() == 0) { 702 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR; 703 O << AArch64InstPrinter::getRegisterName(Reg); 704 return false; 705 } 706 printOperand(MI, OpNum, O); 707 return false; 708 case 'b': // Print B register. 709 case 'h': // Print H register. 710 case 's': // Print S register. 711 case 'd': // Print D register. 712 case 'q': // Print Q register. 713 case 'z': // Print Z register. 714 if (MO.isReg()) { 715 const TargetRegisterClass *RC; 716 switch (ExtraCode[0]) { 717 case 'b': 718 RC = &AArch64::FPR8RegClass; 719 break; 720 case 'h': 721 RC = &AArch64::FPR16RegClass; 722 break; 723 case 's': 724 RC = &AArch64::FPR32RegClass; 725 break; 726 case 'd': 727 RC = &AArch64::FPR64RegClass; 728 break; 729 case 'q': 730 RC = &AArch64::FPR128RegClass; 731 break; 732 case 'z': 733 RC = &AArch64::ZPRRegClass; 734 break; 735 default: 736 return true; 737 } 738 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O); 739 } 740 printOperand(MI, OpNum, O); 741 return false; 742 } 743 } 744 745 // According to ARM, we should emit x and v registers unless we have a 746 // modifier. 747 if (MO.isReg()) { 748 Register Reg = MO.getReg(); 749 750 // If this is a w or x register, print an x register. 751 if (AArch64::GPR32allRegClass.contains(Reg) || 752 AArch64::GPR64allRegClass.contains(Reg)) 753 return printAsmMRegister(MO, 'x', O); 754 755 // If this is an x register tuple, print an x register. 756 if (AArch64::GPR64x8ClassRegClass.contains(Reg)) 757 return printAsmMRegister(MO, 't', O); 758 759 unsigned AltName = AArch64::NoRegAltName; 760 const TargetRegisterClass *RegClass; 761 if (AArch64::ZPRRegClass.contains(Reg)) { 762 RegClass = &AArch64::ZPRRegClass; 763 } else if (AArch64::PPRRegClass.contains(Reg)) { 764 RegClass = &AArch64::PPRRegClass; 765 } else { 766 RegClass = &AArch64::FPR128RegClass; 767 AltName = AArch64::vreg; 768 } 769 770 // If this is a b, h, s, d, or q register, print it as a v register. 771 return printAsmRegInClass(MO, RegClass, AltName, O); 772 } 773 774 printOperand(MI, OpNum, O); 775 return false; 776 } 777 778 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, 779 unsigned OpNum, 780 const char *ExtraCode, 781 raw_ostream &O) { 782 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a') 783 return true; // Unknown modifier. 784 785 const MachineOperand &MO = MI->getOperand(OpNum); 786 assert(MO.isReg() && "unexpected inline asm memory operand"); 787 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]"; 788 return false; 789 } 790 791 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI, 792 raw_ostream &OS) { 793 unsigned NOps = MI->getNumOperands(); 794 assert(NOps == 4); 795 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: "; 796 // cast away const; DIetc do not take const operands for some reason. 797 OS << MI->getDebugVariable()->getName(); 798 OS << " <- "; 799 // Frame address. Currently handles register +- offset only. 800 assert(MI->isIndirectDebugValue()); 801 OS << '['; 802 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(), 803 MI->debug_operands().end()); 804 I < E; ++I) { 805 if (I != 0) 806 OS << ", "; 807 printOperand(MI, I, OS); 808 } 809 OS << ']'; 810 OS << "+"; 811 printOperand(MI, NOps - 2, OS); 812 } 813 814 void AArch64AsmPrinter::emitJumpTableInfo() { 815 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); 816 if (!MJTI) return; 817 818 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 819 if (JT.empty()) return; 820 821 const Function &F = MF->getFunction(); 822 const TargetLoweringObjectFile &TLOF = getObjFileLowering(); 823 bool JTInDiffSection = 824 !STI->isTargetCOFF() || 825 !TLOF.shouldPutJumpTableInFunctionSection( 826 MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32, 827 F); 828 if (JTInDiffSection) { 829 // Drop it in the readonly section. 830 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM); 831 OutStreamer->SwitchSection(ReadOnlySec); 832 } 833 834 auto AFI = MF->getInfo<AArch64FunctionInfo>(); 835 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) { 836 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs; 837 838 // If this jump table was deleted, ignore it. 839 if (JTBBs.empty()) continue; 840 841 unsigned Size = AFI->getJumpTableEntrySize(JTI); 842 emitAlignment(Align(Size)); 843 OutStreamer->emitLabel(GetJTISymbol(JTI)); 844 845 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI); 846 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext); 847 848 for (auto *JTBB : JTBBs) { 849 const MCExpr *Value = 850 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext); 851 852 // Each entry is: 853 // .byte/.hword (LBB - Lbase)>>2 854 // or plain: 855 // .word LBB - Lbase 856 Value = MCBinaryExpr::createSub(Value, Base, OutContext); 857 if (Size != 4) 858 Value = MCBinaryExpr::createLShr( 859 Value, MCConstantExpr::create(2, OutContext), OutContext); 860 861 OutStreamer->emitValue(Value, Size); 862 } 863 } 864 } 865 866 void AArch64AsmPrinter::emitFunctionEntryLabel() { 867 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall || 868 MF->getFunction().getCallingConv() == 869 CallingConv::AArch64_SVE_VectorCall || 870 STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) { 871 auto *TS = 872 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 873 TS->emitDirectiveVariantPCS(CurrentFnSym); 874 } 875 876 return AsmPrinter::emitFunctionEntryLabel(); 877 } 878 879 /// Small jump tables contain an unsigned byte or half, representing the offset 880 /// from the lowest-addressed possible destination to the desired basic 881 /// block. Since all instructions are 4-byte aligned, this is further compressed 882 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to 883 /// materialize the correct destination we need: 884 /// 885 /// adr xDest, .LBB0_0 886 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh). 887 /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries) 888 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer, 889 const llvm::MachineInstr &MI) { 890 Register DestReg = MI.getOperand(0).getReg(); 891 Register ScratchReg = MI.getOperand(1).getReg(); 892 Register ScratchRegW = 893 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32); 894 Register TableReg = MI.getOperand(2).getReg(); 895 Register EntryReg = MI.getOperand(3).getReg(); 896 int JTIdx = MI.getOperand(4).getIndex(); 897 int Size = AArch64FI->getJumpTableEntrySize(JTIdx); 898 899 // This has to be first because the compression pass based its reachability 900 // calculations on the start of the JumpTableDest instruction. 901 auto Label = 902 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx); 903 904 // If we don't already have a symbol to use as the base, use the ADR 905 // instruction itself. 906 if (!Label) { 907 Label = MF->getContext().createTempSymbol(); 908 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label); 909 OutStreamer.emitLabel(Label); 910 } 911 912 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext()); 913 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR) 914 .addReg(DestReg) 915 .addExpr(LabelExpr)); 916 917 // Load the number of instruction-steps to offset from the label. 918 unsigned LdrOpcode; 919 switch (Size) { 920 case 1: LdrOpcode = AArch64::LDRBBroX; break; 921 case 2: LdrOpcode = AArch64::LDRHHroX; break; 922 case 4: LdrOpcode = AArch64::LDRSWroX; break; 923 default: 924 llvm_unreachable("Unknown jump table size"); 925 } 926 927 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode) 928 .addReg(Size == 4 ? ScratchReg : ScratchRegW) 929 .addReg(TableReg) 930 .addReg(EntryReg) 931 .addImm(0) 932 .addImm(Size == 1 ? 0 : 1)); 933 934 // Add to the already materialized base label address, multiplying by 4 if 935 // compressed. 936 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs) 937 .addReg(DestReg) 938 .addReg(DestReg) 939 .addReg(ScratchReg) 940 .addImm(Size == 4 ? 0 : 2)); 941 } 942 943 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 944 const MachineInstr &MI) { 945 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); 946 947 auto &Ctx = OutStreamer.getContext(); 948 MCSymbol *MILabel = Ctx.createTempSymbol(); 949 OutStreamer.emitLabel(MILabel); 950 951 SM.recordStackMap(*MILabel, MI); 952 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); 953 954 // Scan ahead to trim the shadow. 955 const MachineBasicBlock &MBB = *MI.getParent(); 956 MachineBasicBlock::const_iterator MII(MI); 957 ++MII; 958 while (NumNOPBytes > 0) { 959 if (MII == MBB.end() || MII->isCall() || 960 MII->getOpcode() == AArch64::DBG_VALUE || 961 MII->getOpcode() == TargetOpcode::PATCHPOINT || 962 MII->getOpcode() == TargetOpcode::STACKMAP) 963 break; 964 ++MII; 965 NumNOPBytes -= 4; 966 } 967 968 // Emit nops. 969 for (unsigned i = 0; i < NumNOPBytes; i += 4) 970 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 971 } 972 973 // Lower a patchpoint of the form: 974 // [<def>], <id>, <numBytes>, <target>, <numArgs> 975 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 976 const MachineInstr &MI) { 977 auto &Ctx = OutStreamer.getContext(); 978 MCSymbol *MILabel = Ctx.createTempSymbol(); 979 OutStreamer.emitLabel(MILabel); 980 SM.recordPatchPoint(*MILabel, MI); 981 982 PatchPointOpers Opers(&MI); 983 984 int64_t CallTarget = Opers.getCallTarget().getImm(); 985 unsigned EncodedBytes = 0; 986 if (CallTarget) { 987 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && 988 "High 16 bits of call target should be zero."); 989 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg(); 990 EncodedBytes = 16; 991 // Materialize the jump address: 992 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi) 993 .addReg(ScratchReg) 994 .addImm((CallTarget >> 32) & 0xFFFF) 995 .addImm(32)); 996 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) 997 .addReg(ScratchReg) 998 .addReg(ScratchReg) 999 .addImm((CallTarget >> 16) & 0xFFFF) 1000 .addImm(16)); 1001 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) 1002 .addReg(ScratchReg) 1003 .addReg(ScratchReg) 1004 .addImm(CallTarget & 0xFFFF) 1005 .addImm(0)); 1006 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg)); 1007 } 1008 // Emit padding. 1009 unsigned NumBytes = Opers.getNumPatchBytes(); 1010 assert(NumBytes >= EncodedBytes && 1011 "Patchpoint can't request size less than the length of a call."); 1012 assert((NumBytes - EncodedBytes) % 4 == 0 && 1013 "Invalid number of NOP bytes requested!"); 1014 for (unsigned i = EncodedBytes; i < NumBytes; i += 4) 1015 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1016 } 1017 1018 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 1019 const MachineInstr &MI) { 1020 StatepointOpers SOpers(&MI); 1021 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 1022 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); 1023 for (unsigned i = 0; i < PatchBytes; i += 4) 1024 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1025 } else { 1026 // Lower call target and choose correct opcode 1027 const MachineOperand &CallTarget = SOpers.getCallTarget(); 1028 MCOperand CallTargetMCOp; 1029 unsigned CallOpcode; 1030 switch (CallTarget.getType()) { 1031 case MachineOperand::MO_GlobalAddress: 1032 case MachineOperand::MO_ExternalSymbol: 1033 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp); 1034 CallOpcode = AArch64::BL; 1035 break; 1036 case MachineOperand::MO_Immediate: 1037 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm()); 1038 CallOpcode = AArch64::BL; 1039 break; 1040 case MachineOperand::MO_Register: 1041 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg()); 1042 CallOpcode = AArch64::BLR; 1043 break; 1044 default: 1045 llvm_unreachable("Unsupported operand type in statepoint call target"); 1046 break; 1047 } 1048 1049 EmitToStreamer(OutStreamer, 1050 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp)); 1051 } 1052 1053 auto &Ctx = OutStreamer.getContext(); 1054 MCSymbol *MILabel = Ctx.createTempSymbol(); 1055 OutStreamer.emitLabel(MILabel); 1056 SM.recordStatepoint(*MILabel, MI); 1057 } 1058 1059 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) { 1060 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>, 1061 // <opcode>, <operands> 1062 1063 Register DefRegister = FaultingMI.getOperand(0).getReg(); 1064 FaultMaps::FaultKind FK = 1065 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm()); 1066 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol(); 1067 unsigned Opcode = FaultingMI.getOperand(3).getImm(); 1068 unsigned OperandsBeginIdx = 4; 1069 1070 auto &Ctx = OutStreamer->getContext(); 1071 MCSymbol *FaultingLabel = Ctx.createTempSymbol(); 1072 OutStreamer->emitLabel(FaultingLabel); 1073 1074 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!"); 1075 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel); 1076 1077 MCInst MI; 1078 MI.setOpcode(Opcode); 1079 1080 if (DefRegister != (Register)0) 1081 MI.addOperand(MCOperand::createReg(DefRegister)); 1082 1083 for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx, 1084 E = FaultingMI.operands_end(); 1085 I != E; ++I) { 1086 MCOperand Dest; 1087 lowerOperand(*I, Dest); 1088 MI.addOperand(Dest); 1089 } 1090 1091 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName()); 1092 OutStreamer->emitInstruction(MI, getSubtargetInfo()); 1093 } 1094 1095 void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) { 1096 Register DestReg = MI.getOperand(0).getReg(); 1097 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) { 1098 // Convert H/S register to corresponding D register 1099 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31) 1100 DestReg = AArch64::D0 + (DestReg - AArch64::H0); 1101 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31) 1102 DestReg = AArch64::D0 + (DestReg - AArch64::S0); 1103 else 1104 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); 1105 1106 MCInst MOVI; 1107 MOVI.setOpcode(AArch64::MOVID); 1108 MOVI.addOperand(MCOperand::createReg(DestReg)); 1109 MOVI.addOperand(MCOperand::createImm(0)); 1110 EmitToStreamer(*OutStreamer, MOVI); 1111 } else { 1112 MCInst FMov; 1113 switch (MI.getOpcode()) { 1114 default: llvm_unreachable("Unexpected opcode"); 1115 case AArch64::FMOVH0: 1116 FMov.setOpcode(AArch64::FMOVWHr); 1117 FMov.addOperand(MCOperand::createReg(DestReg)); 1118 FMov.addOperand(MCOperand::createReg(AArch64::WZR)); 1119 break; 1120 case AArch64::FMOVS0: 1121 FMov.setOpcode(AArch64::FMOVWSr); 1122 FMov.addOperand(MCOperand::createReg(DestReg)); 1123 FMov.addOperand(MCOperand::createReg(AArch64::WZR)); 1124 break; 1125 case AArch64::FMOVD0: 1126 FMov.setOpcode(AArch64::FMOVXDr); 1127 FMov.addOperand(MCOperand::createReg(DestReg)); 1128 FMov.addOperand(MCOperand::createReg(AArch64::XZR)); 1129 break; 1130 } 1131 EmitToStreamer(*OutStreamer, FMov); 1132 } 1133 } 1134 1135 // Simple pseudo-instructions have their lowering (with expansion to real 1136 // instructions) auto-generated. 1137 #include "AArch64GenMCPseudoLowering.inc" 1138 1139 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { 1140 // Do any auto-generated pseudo lowerings. 1141 if (emitPseudoExpansionLowering(*OutStreamer, MI)) 1142 return; 1143 1144 if (AArch64FI->getLOHRelated().count(MI)) { 1145 // Generate a label for LOH related instruction 1146 MCSymbol *LOHLabel = createTempSymbol("loh"); 1147 // Associate the instruction with the label 1148 LOHInstToLabel[MI] = LOHLabel; 1149 OutStreamer->emitLabel(LOHLabel); 1150 } 1151 1152 AArch64TargetStreamer *TS = 1153 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 1154 // Do any manual lowerings. 1155 switch (MI->getOpcode()) { 1156 default: 1157 break; 1158 case AArch64::HINT: { 1159 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for 1160 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be 1161 // non-empty. If MI is the initial BTI, place the 1162 // __patchable_function_entries label after BTI. 1163 if (CurrentPatchableFunctionEntrySym && 1164 CurrentPatchableFunctionEntrySym == CurrentFnBegin && 1165 MI == &MF->front().front()) { 1166 int64_t Imm = MI->getOperand(0).getImm(); 1167 if ((Imm & 32) && (Imm & 6)) { 1168 MCInst Inst; 1169 MCInstLowering.Lower(MI, Inst); 1170 EmitToStreamer(*OutStreamer, Inst); 1171 CurrentPatchableFunctionEntrySym = createTempSymbol("patch"); 1172 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym); 1173 return; 1174 } 1175 } 1176 break; 1177 } 1178 case AArch64::MOVMCSym: { 1179 Register DestReg = MI->getOperand(0).getReg(); 1180 const MachineOperand &MO_Sym = MI->getOperand(1); 1181 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym); 1182 MCOperand Hi_MCSym, Lo_MCSym; 1183 1184 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S); 1185 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC); 1186 1187 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym); 1188 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym); 1189 1190 MCInst MovZ; 1191 MovZ.setOpcode(AArch64::MOVZXi); 1192 MovZ.addOperand(MCOperand::createReg(DestReg)); 1193 MovZ.addOperand(Hi_MCSym); 1194 MovZ.addOperand(MCOperand::createImm(16)); 1195 EmitToStreamer(*OutStreamer, MovZ); 1196 1197 MCInst MovK; 1198 MovK.setOpcode(AArch64::MOVKXi); 1199 MovK.addOperand(MCOperand::createReg(DestReg)); 1200 MovK.addOperand(MCOperand::createReg(DestReg)); 1201 MovK.addOperand(Lo_MCSym); 1202 MovK.addOperand(MCOperand::createImm(0)); 1203 EmitToStreamer(*OutStreamer, MovK); 1204 return; 1205 } 1206 case AArch64::MOVIv2d_ns: 1207 // If the target has <rdar://problem/16473581>, lower this 1208 // instruction to movi.16b instead. 1209 if (STI->hasZeroCycleZeroingFPWorkaround() && 1210 MI->getOperand(1).getImm() == 0) { 1211 MCInst TmpInst; 1212 TmpInst.setOpcode(AArch64::MOVIv16b_ns); 1213 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); 1214 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm())); 1215 EmitToStreamer(*OutStreamer, TmpInst); 1216 return; 1217 } 1218 break; 1219 1220 case AArch64::DBG_VALUE: 1221 case AArch64::DBG_VALUE_LIST: { 1222 if (isVerbose() && OutStreamer->hasRawTextSupport()) { 1223 SmallString<128> TmpStr; 1224 raw_svector_ostream OS(TmpStr); 1225 PrintDebugValueComment(MI, OS); 1226 OutStreamer->emitRawText(StringRef(OS.str())); 1227 } 1228 return; 1229 1230 case AArch64::EMITBKEY: { 1231 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); 1232 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && 1233 ExceptionHandlingType != ExceptionHandling::ARM) 1234 return; 1235 1236 if (getFunctionCFISectionType(*MF) == CFISection::None) 1237 return; 1238 1239 OutStreamer->emitCFIBKeyFrame(); 1240 return; 1241 } 1242 } 1243 1244 // Tail calls use pseudo instructions so they have the proper code-gen 1245 // attributes (isCall, isReturn, etc.). We lower them to the real 1246 // instruction here. 1247 case AArch64::TCRETURNri: 1248 case AArch64::TCRETURNriBTI: 1249 case AArch64::TCRETURNriALL: { 1250 MCInst TmpInst; 1251 TmpInst.setOpcode(AArch64::BR); 1252 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); 1253 EmitToStreamer(*OutStreamer, TmpInst); 1254 return; 1255 } 1256 case AArch64::TCRETURNdi: { 1257 MCOperand Dest; 1258 MCInstLowering.lowerOperand(MI->getOperand(0), Dest); 1259 MCInst TmpInst; 1260 TmpInst.setOpcode(AArch64::B); 1261 TmpInst.addOperand(Dest); 1262 EmitToStreamer(*OutStreamer, TmpInst); 1263 return; 1264 } 1265 case AArch64::SpeculationBarrierISBDSBEndBB: { 1266 // Print DSB SYS + ISB 1267 MCInst TmpInstDSB; 1268 TmpInstDSB.setOpcode(AArch64::DSB); 1269 TmpInstDSB.addOperand(MCOperand::createImm(0xf)); 1270 EmitToStreamer(*OutStreamer, TmpInstDSB); 1271 MCInst TmpInstISB; 1272 TmpInstISB.setOpcode(AArch64::ISB); 1273 TmpInstISB.addOperand(MCOperand::createImm(0xf)); 1274 EmitToStreamer(*OutStreamer, TmpInstISB); 1275 return; 1276 } 1277 case AArch64::SpeculationBarrierSBEndBB: { 1278 // Print SB 1279 MCInst TmpInstSB; 1280 TmpInstSB.setOpcode(AArch64::SB); 1281 EmitToStreamer(*OutStreamer, TmpInstSB); 1282 return; 1283 } 1284 case AArch64::TLSDESC_CALLSEQ: { 1285 /// lower this to: 1286 /// adrp x0, :tlsdesc:var 1287 /// ldr x1, [x0, #:tlsdesc_lo12:var] 1288 /// add x0, x0, #:tlsdesc_lo12:var 1289 /// .tlsdesccall var 1290 /// blr x1 1291 /// (TPIDR_EL0 offset now in x0) 1292 const MachineOperand &MO_Sym = MI->getOperand(0); 1293 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym); 1294 MCOperand Sym, SymTLSDescLo12, SymTLSDesc; 1295 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF); 1296 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE); 1297 MCInstLowering.lowerOperand(MO_Sym, Sym); 1298 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12); 1299 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc); 1300 1301 MCInst Adrp; 1302 Adrp.setOpcode(AArch64::ADRP); 1303 Adrp.addOperand(MCOperand::createReg(AArch64::X0)); 1304 Adrp.addOperand(SymTLSDesc); 1305 EmitToStreamer(*OutStreamer, Adrp); 1306 1307 MCInst Ldr; 1308 if (STI->isTargetILP32()) { 1309 Ldr.setOpcode(AArch64::LDRWui); 1310 Ldr.addOperand(MCOperand::createReg(AArch64::W1)); 1311 } else { 1312 Ldr.setOpcode(AArch64::LDRXui); 1313 Ldr.addOperand(MCOperand::createReg(AArch64::X1)); 1314 } 1315 Ldr.addOperand(MCOperand::createReg(AArch64::X0)); 1316 Ldr.addOperand(SymTLSDescLo12); 1317 Ldr.addOperand(MCOperand::createImm(0)); 1318 EmitToStreamer(*OutStreamer, Ldr); 1319 1320 MCInst Add; 1321 if (STI->isTargetILP32()) { 1322 Add.setOpcode(AArch64::ADDWri); 1323 Add.addOperand(MCOperand::createReg(AArch64::W0)); 1324 Add.addOperand(MCOperand::createReg(AArch64::W0)); 1325 } else { 1326 Add.setOpcode(AArch64::ADDXri); 1327 Add.addOperand(MCOperand::createReg(AArch64::X0)); 1328 Add.addOperand(MCOperand::createReg(AArch64::X0)); 1329 } 1330 Add.addOperand(SymTLSDescLo12); 1331 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0))); 1332 EmitToStreamer(*OutStreamer, Add); 1333 1334 // Emit a relocation-annotation. This expands to no code, but requests 1335 // the following instruction gets an R_AARCH64_TLSDESC_CALL. 1336 MCInst TLSDescCall; 1337 TLSDescCall.setOpcode(AArch64::TLSDESCCALL); 1338 TLSDescCall.addOperand(Sym); 1339 EmitToStreamer(*OutStreamer, TLSDescCall); 1340 1341 MCInst Blr; 1342 Blr.setOpcode(AArch64::BLR); 1343 Blr.addOperand(MCOperand::createReg(AArch64::X1)); 1344 EmitToStreamer(*OutStreamer, Blr); 1345 1346 return; 1347 } 1348 1349 case AArch64::JumpTableDest32: 1350 case AArch64::JumpTableDest16: 1351 case AArch64::JumpTableDest8: 1352 LowerJumpTableDest(*OutStreamer, *MI); 1353 return; 1354 1355 case AArch64::FMOVH0: 1356 case AArch64::FMOVS0: 1357 case AArch64::FMOVD0: 1358 emitFMov0(*MI); 1359 return; 1360 1361 case TargetOpcode::STACKMAP: 1362 return LowerSTACKMAP(*OutStreamer, SM, *MI); 1363 1364 case TargetOpcode::PATCHPOINT: 1365 return LowerPATCHPOINT(*OutStreamer, SM, *MI); 1366 1367 case TargetOpcode::STATEPOINT: 1368 return LowerSTATEPOINT(*OutStreamer, SM, *MI); 1369 1370 case TargetOpcode::FAULTING_OP: 1371 return LowerFAULTING_OP(*MI); 1372 1373 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: 1374 LowerPATCHABLE_FUNCTION_ENTER(*MI); 1375 return; 1376 1377 case TargetOpcode::PATCHABLE_FUNCTION_EXIT: 1378 LowerPATCHABLE_FUNCTION_EXIT(*MI); 1379 return; 1380 1381 case TargetOpcode::PATCHABLE_TAIL_CALL: 1382 LowerPATCHABLE_TAIL_CALL(*MI); 1383 return; 1384 1385 case AArch64::HWASAN_CHECK_MEMACCESS: 1386 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: 1387 LowerHWASAN_CHECK_MEMACCESS(*MI); 1388 return; 1389 1390 case AArch64::SEH_StackAlloc: 1391 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm()); 1392 return; 1393 1394 case AArch64::SEH_SaveFPLR: 1395 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm()); 1396 return; 1397 1398 case AArch64::SEH_SaveFPLR_X: 1399 assert(MI->getOperand(0).getImm() < 0 && 1400 "Pre increment SEH opcode must have a negative offset"); 1401 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm()); 1402 return; 1403 1404 case AArch64::SEH_SaveReg: 1405 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(), 1406 MI->getOperand(1).getImm()); 1407 return; 1408 1409 case AArch64::SEH_SaveReg_X: 1410 assert(MI->getOperand(1).getImm() < 0 && 1411 "Pre increment SEH opcode must have a negative offset"); 1412 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(), 1413 -MI->getOperand(1).getImm()); 1414 return; 1415 1416 case AArch64::SEH_SaveRegP: 1417 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 && 1418 MI->getOperand(0).getImm() <= 28) { 1419 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 && 1420 "Register paired with LR must be odd"); 1421 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(), 1422 MI->getOperand(2).getImm()); 1423 return; 1424 } 1425 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1426 "Non-consecutive registers not allowed for save_regp"); 1427 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(), 1428 MI->getOperand(2).getImm()); 1429 return; 1430 1431 case AArch64::SEH_SaveRegP_X: 1432 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1433 "Non-consecutive registers not allowed for save_regp_x"); 1434 assert(MI->getOperand(2).getImm() < 0 && 1435 "Pre increment SEH opcode must have a negative offset"); 1436 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(), 1437 -MI->getOperand(2).getImm()); 1438 return; 1439 1440 case AArch64::SEH_SaveFReg: 1441 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(), 1442 MI->getOperand(1).getImm()); 1443 return; 1444 1445 case AArch64::SEH_SaveFReg_X: 1446 assert(MI->getOperand(1).getImm() < 0 && 1447 "Pre increment SEH opcode must have a negative offset"); 1448 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(), 1449 -MI->getOperand(1).getImm()); 1450 return; 1451 1452 case AArch64::SEH_SaveFRegP: 1453 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1454 "Non-consecutive registers not allowed for save_regp"); 1455 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(), 1456 MI->getOperand(2).getImm()); 1457 return; 1458 1459 case AArch64::SEH_SaveFRegP_X: 1460 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1461 "Non-consecutive registers not allowed for save_regp_x"); 1462 assert(MI->getOperand(2).getImm() < 0 && 1463 "Pre increment SEH opcode must have a negative offset"); 1464 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(), 1465 -MI->getOperand(2).getImm()); 1466 return; 1467 1468 case AArch64::SEH_SetFP: 1469 TS->emitARM64WinCFISetFP(); 1470 return; 1471 1472 case AArch64::SEH_AddFP: 1473 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm()); 1474 return; 1475 1476 case AArch64::SEH_Nop: 1477 TS->emitARM64WinCFINop(); 1478 return; 1479 1480 case AArch64::SEH_PrologEnd: 1481 TS->emitARM64WinCFIPrologEnd(); 1482 return; 1483 1484 case AArch64::SEH_EpilogStart: 1485 TS->emitARM64WinCFIEpilogStart(); 1486 return; 1487 1488 case AArch64::SEH_EpilogEnd: 1489 TS->emitARM64WinCFIEpilogEnd(); 1490 return; 1491 } 1492 1493 // Finally, do the automated lowerings for everything else. 1494 MCInst TmpInst; 1495 MCInstLowering.Lower(MI, TmpInst); 1496 EmitToStreamer(*OutStreamer, TmpInst); 1497 } 1498 1499 // Force static initialization. 1500 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { 1501 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); 1502 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget()); 1503 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target()); 1504 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target()); 1505 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target()); 1506 } 1507