1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a printer that converts from our internal representation 10 // of machine-dependent LLVM code to the AArch64 assembly language. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64.h" 15 #include "AArch64MCInstLower.h" 16 #include "AArch64MachineFunctionInfo.h" 17 #include "AArch64RegisterInfo.h" 18 #include "AArch64Subtarget.h" 19 #include "AArch64TargetObjectFile.h" 20 #include "MCTargetDesc/AArch64AddressingModes.h" 21 #include "MCTargetDesc/AArch64InstPrinter.h" 22 #include "MCTargetDesc/AArch64MCExpr.h" 23 #include "MCTargetDesc/AArch64MCTargetDesc.h" 24 #include "MCTargetDesc/AArch64TargetStreamer.h" 25 #include "TargetInfo/AArch64TargetInfo.h" 26 #include "Utils/AArch64BaseInfo.h" 27 #include "llvm/ADT/SmallString.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/ADT/Twine.h" 31 #include "llvm/BinaryFormat/COFF.h" 32 #include "llvm/BinaryFormat/ELF.h" 33 #include "llvm/BinaryFormat/MachO.h" 34 #include "llvm/CodeGen/AsmPrinter.h" 35 #include "llvm/CodeGen/FaultMaps.h" 36 #include "llvm/CodeGen/MachineBasicBlock.h" 37 #include "llvm/CodeGen/MachineFunction.h" 38 #include "llvm/CodeGen/MachineInstr.h" 39 #include "llvm/CodeGen/MachineJumpTableInfo.h" 40 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 41 #include "llvm/CodeGen/MachineOperand.h" 42 #include "llvm/CodeGen/StackMaps.h" 43 #include "llvm/CodeGen/TargetRegisterInfo.h" 44 #include "llvm/IR/DataLayout.h" 45 #include "llvm/IR/DebugInfoMetadata.h" 46 #include "llvm/MC/MCAsmInfo.h" 47 #include "llvm/MC/MCContext.h" 48 #include "llvm/MC/MCInst.h" 49 #include "llvm/MC/MCInstBuilder.h" 50 #include "llvm/MC/MCSectionELF.h" 51 #include "llvm/MC/MCSectionMachO.h" 52 #include "llvm/MC/MCStreamer.h" 53 #include "llvm/MC/MCSymbol.h" 54 #include "llvm/MC/TargetRegistry.h" 55 #include "llvm/Support/Casting.h" 56 #include "llvm/Support/CommandLine.h" 57 #include "llvm/Support/ErrorHandling.h" 58 #include "llvm/Support/raw_ostream.h" 59 #include "llvm/Target/TargetMachine.h" 60 #include "llvm/TargetParser/Triple.h" 61 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 62 #include <algorithm> 63 #include <cassert> 64 #include <cstdint> 65 #include <map> 66 #include <memory> 67 68 using namespace llvm; 69 70 #define DEBUG_TYPE "asm-printer" 71 72 namespace { 73 74 class AArch64AsmPrinter : public AsmPrinter { 75 AArch64MCInstLower MCInstLowering; 76 FaultMaps FM; 77 const AArch64Subtarget *STI; 78 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false; 79 80 public: 81 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) 82 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this), 83 FM(*this) {} 84 85 StringRef getPassName() const override { return "AArch64 Assembly Printer"; } 86 87 /// Wrapper for MCInstLowering.lowerOperand() for the 88 /// tblgen'erated pseudo lowering. 89 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const { 90 return MCInstLowering.lowerOperand(MO, MCOp); 91 } 92 93 void emitStartOfAsmFile(Module &M) override; 94 void emitJumpTableInfo() override; 95 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *, 96 codeview::JumpTableEntrySize> 97 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, 98 const MCSymbol *BranchLabel) const override; 99 100 void emitFunctionEntryLabel() override; 101 102 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI); 103 104 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI); 105 106 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 107 const MachineInstr &MI); 108 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 109 const MachineInstr &MI); 110 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 111 const MachineInstr &MI); 112 void LowerFAULTING_OP(const MachineInstr &MI); 113 114 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); 115 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); 116 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); 117 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed); 118 119 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple; 120 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; 121 void LowerKCFI_CHECK(const MachineInstr &MI); 122 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); 123 void emitHwasanMemaccessSymbols(Module &M); 124 125 void emitSled(const MachineInstr &MI, SledKind Kind); 126 127 /// tblgen'erated driver function for lowering simple MI->MC 128 /// pseudo instructions. 129 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, 130 const MachineInstr *MI); 131 132 void emitInstruction(const MachineInstr *MI) override; 133 134 void emitFunctionHeaderComment() override; 135 136 void getAnalysisUsage(AnalysisUsage &AU) const override { 137 AsmPrinter::getAnalysisUsage(AU); 138 AU.setPreservesAll(); 139 } 140 141 bool runOnMachineFunction(MachineFunction &MF) override { 142 AArch64FI = MF.getInfo<AArch64FunctionInfo>(); 143 STI = &MF.getSubtarget<AArch64Subtarget>(); 144 145 SetupMachineFunction(MF); 146 147 if (STI->isTargetCOFF()) { 148 bool Local = MF.getFunction().hasLocalLinkage(); 149 COFF::SymbolStorageClass Scl = 150 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL; 151 int Type = 152 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT; 153 154 OutStreamer->beginCOFFSymbolDef(CurrentFnSym); 155 OutStreamer->emitCOFFSymbolStorageClass(Scl); 156 OutStreamer->emitCOFFSymbolType(Type); 157 OutStreamer->endCOFFSymbolDef(); 158 } 159 160 // Emit the rest of the function body. 161 emitFunctionBody(); 162 163 // Emit the XRay table for this function. 164 emitXRayTable(); 165 166 // We didn't modify anything. 167 return false; 168 } 169 170 private: 171 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); 172 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O); 173 bool printAsmRegInClass(const MachineOperand &MO, 174 const TargetRegisterClass *RC, unsigned AltName, 175 raw_ostream &O); 176 177 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, 178 const char *ExtraCode, raw_ostream &O) override; 179 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, 180 const char *ExtraCode, raw_ostream &O) override; 181 182 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS); 183 184 void emitFunctionBodyEnd() override; 185 186 MCSymbol *GetCPISymbol(unsigned CPID) const override; 187 void emitEndOfAsmFile(Module &M) override; 188 189 AArch64FunctionInfo *AArch64FI = nullptr; 190 191 /// Emit the LOHs contained in AArch64FI. 192 void emitLOHs(); 193 194 /// Emit instruction to set float register to zero. 195 void emitFMov0(const MachineInstr &MI); 196 197 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; 198 199 MInstToMCSymbol LOHInstToLabel; 200 201 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { 202 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; 203 } 204 205 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override { 206 assert(STI); 207 return STI; 208 } 209 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, 210 MCSymbol *LazyPointer) override; 211 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, 212 MCSymbol *LazyPointer) override; 213 }; 214 215 } // end anonymous namespace 216 217 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) { 218 const Triple &TT = TM.getTargetTriple(); 219 220 if (TT.isOSBinFormatCOFF()) { 221 // Emit an absolute @feat.00 symbol 222 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00")); 223 OutStreamer->beginCOFFSymbolDef(S); 224 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC); 225 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL); 226 OutStreamer->endCOFFSymbolDef(); 227 int64_t Feat00Value = 0; 228 229 if (M.getModuleFlag("cfguard")) { 230 // Object is CFG-aware. 231 Feat00Value |= COFF::Feat00Flags::GuardCF; 232 } 233 234 if (M.getModuleFlag("ehcontguard")) { 235 // Object also has EHCont. 236 Feat00Value |= COFF::Feat00Flags::GuardEHCont; 237 } 238 239 if (M.getModuleFlag("ms-kernel")) { 240 // Object is compiled with /kernel. 241 Feat00Value |= COFF::Feat00Flags::Kernel; 242 } 243 244 OutStreamer->emitSymbolAttribute(S, MCSA_Global); 245 OutStreamer->emitAssignment( 246 S, MCConstantExpr::create(Feat00Value, MMI->getContext())); 247 } 248 249 if (!TT.isOSBinFormatELF()) 250 return; 251 252 // Assemble feature flags that may require creation of a note section. 253 unsigned Flags = 0; 254 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>( 255 M.getModuleFlag("branch-target-enforcement"))) 256 if (BTE->getZExtValue()) 257 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI; 258 259 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>( 260 M.getModuleFlag("sign-return-address"))) 261 if (Sign->getZExtValue()) 262 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC; 263 264 if (Flags == 0) 265 return; 266 267 // Emit a .note.gnu.property section with the flags. 268 auto *TS = 269 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 270 TS->emitNoteSection(Flags); 271 } 272 273 void AArch64AsmPrinter::emitFunctionHeaderComment() { 274 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>(); 275 std::optional<std::string> OutlinerString = FI->getOutliningStyle(); 276 if (OutlinerString != std::nullopt) 277 OutStreamer->getCommentOS() << ' ' << OutlinerString; 278 } 279 280 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) 281 { 282 const Function &F = MF->getFunction(); 283 if (F.hasFnAttribute("patchable-function-entry")) { 284 unsigned Num; 285 if (F.getFnAttribute("patchable-function-entry") 286 .getValueAsString() 287 .getAsInteger(10, Num)) 288 return; 289 emitNops(Num); 290 return; 291 } 292 293 emitSled(MI, SledKind::FUNCTION_ENTER); 294 } 295 296 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) { 297 emitSled(MI, SledKind::FUNCTION_EXIT); 298 } 299 300 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) { 301 emitSled(MI, SledKind::TAIL_CALL); 302 } 303 304 void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) { 305 static const int8_t NoopsInSledCount = 7; 306 // We want to emit the following pattern: 307 // 308 // .Lxray_sled_N: 309 // ALIGN 310 // B #32 311 // ; 7 NOP instructions (28 bytes) 312 // .tmpN 313 // 314 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching 315 // over the full 32 bytes (8 instructions) with the following pattern: 316 // 317 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack 318 // LDR W17, #12 ; W17 := function ID 319 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit 320 // BLR X16 ; call the tracing trampoline 321 // ;DATA: 32 bits of function ID 322 // ;DATA: lower 32 bits of the address of the trampoline 323 // ;DATA: higher 32 bits of the address of the trampoline 324 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack 325 // 326 OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo()); 327 auto CurSled = OutContext.createTempSymbol("xray_sled_", true); 328 OutStreamer->emitLabel(CurSled); 329 auto Target = OutContext.createTempSymbol(); 330 331 // Emit "B #32" instruction, which jumps over the next 28 bytes. 332 // The operand has to be the number of 4-byte instructions to jump over, 333 // including the current instruction. 334 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8)); 335 336 for (int8_t I = 0; I < NoopsInSledCount; I++) 337 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 338 339 OutStreamer->emitLabel(Target); 340 recordSled(CurSled, MI, Kind, 2); 341 } 342 343 // Emit the following code for Intrinsic::{xray_customevent,xray_typedevent} 344 // (built-in functions __xray_customevent/__xray_typedevent). 345 // 346 // .Lxray_event_sled_N: 347 // b 1f 348 // save x0 and x1 (and also x2 for TYPED_EVENT_CALL) 349 // set up x0 and x1 (and also x2 for TYPED_EVENT_CALL) 350 // bl __xray_CustomEvent or __xray_TypedEvent 351 // restore x0 and x1 (and also x2 for TYPED_EVENT_CALL) 352 // 1: 353 // 354 // There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL. 355 // 356 // Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT. 357 // After patching, b .+N will become a nop. 358 void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, 359 bool Typed) { 360 auto &O = *OutStreamer; 361 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true); 362 O.emitLabel(CurSled); 363 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs) 364 .addReg(AArch64::X0) 365 .addReg(AArch64::XZR) 366 .addReg(MI.getOperand(0).getReg()) 367 .addImm(0); 368 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs) 369 .addReg(AArch64::X1) 370 .addReg(AArch64::XZR) 371 .addReg(MI.getOperand(1).getReg()) 372 .addImm(0); 373 bool MachO = TM.getTargetTriple().isOSBinFormatMachO(); 374 auto *Sym = MCSymbolRefExpr::create( 375 OutContext.getOrCreateSymbol( 376 Twine(MachO ? "_" : "") + 377 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")), 378 OutContext); 379 if (Typed) { 380 O.AddComment("Begin XRay typed event"); 381 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9)); 382 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre) 383 .addReg(AArch64::SP) 384 .addReg(AArch64::X0) 385 .addReg(AArch64::X1) 386 .addReg(AArch64::SP) 387 .addImm(-4)); 388 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui) 389 .addReg(AArch64::X2) 390 .addReg(AArch64::SP) 391 .addImm(2)); 392 EmitToStreamer(O, MovX0Op0); 393 EmitToStreamer(O, MovX1Op1); 394 EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs) 395 .addReg(AArch64::X2) 396 .addReg(AArch64::XZR) 397 .addReg(MI.getOperand(2).getReg()) 398 .addImm(0)); 399 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym)); 400 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui) 401 .addReg(AArch64::X2) 402 .addReg(AArch64::SP) 403 .addImm(2)); 404 O.AddComment("End XRay typed event"); 405 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost) 406 .addReg(AArch64::SP) 407 .addReg(AArch64::X0) 408 .addReg(AArch64::X1) 409 .addReg(AArch64::SP) 410 .addImm(4)); 411 412 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2); 413 } else { 414 O.AddComment("Begin XRay custom event"); 415 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6)); 416 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre) 417 .addReg(AArch64::SP) 418 .addReg(AArch64::X0) 419 .addReg(AArch64::X1) 420 .addReg(AArch64::SP) 421 .addImm(-2)); 422 EmitToStreamer(O, MovX0Op0); 423 EmitToStreamer(O, MovX1Op1); 424 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym)); 425 O.AddComment("End XRay custom event"); 426 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost) 427 .addReg(AArch64::SP) 428 .addReg(AArch64::X0) 429 .addReg(AArch64::X1) 430 .addReg(AArch64::SP) 431 .addImm(2)); 432 433 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2); 434 } 435 } 436 437 void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { 438 Register AddrReg = MI.getOperand(0).getReg(); 439 assert(std::next(MI.getIterator())->isCall() && 440 "KCFI_CHECK not followed by a call instruction"); 441 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg && 442 "KCFI_CHECK call target doesn't match call operand"); 443 444 // Default to using the intra-procedure-call temporary registers for 445 // comparing the hashes. 446 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17}; 447 if (AddrReg == AArch64::XZR) { 448 // Checking XZR makes no sense. Instead of emitting a load, zero 449 // ScratchRegs[0] and use it for the ESR AddrIndex below. 450 AddrReg = getXRegFromWReg(ScratchRegs[0]); 451 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs) 452 .addReg(AddrReg) 453 .addReg(AArch64::XZR) 454 .addReg(AArch64::XZR) 455 .addImm(0)); 456 } else { 457 // If one of the scratch registers is used for the call target (e.g. 458 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved 459 // temporary register instead (in this case, AArch64::W9) as the check 460 // is immediately followed by the call instruction. 461 for (auto &Reg : ScratchRegs) { 462 if (Reg == getWRegFromXReg(AddrReg)) { 463 Reg = AArch64::W9; 464 break; 465 } 466 } 467 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg && 468 "Invalid scratch registers for KCFI_CHECK"); 469 470 // Adjust the offset for patchable-function-prefix. This assumes that 471 // patchable-function-prefix is the same for all functions. 472 int64_t PrefixNops = 0; 473 (void)MI.getMF() 474 ->getFunction() 475 .getFnAttribute("patchable-function-prefix") 476 .getValueAsString() 477 .getAsInteger(10, PrefixNops); 478 479 // Load the target function type hash. 480 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi) 481 .addReg(ScratchRegs[0]) 482 .addReg(AddrReg) 483 .addImm(-(PrefixNops * 4 + 4))); 484 } 485 486 // Load the expected type hash. 487 const int64_t Type = MI.getOperand(1).getImm(); 488 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) 489 .addReg(ScratchRegs[1]) 490 .addReg(ScratchRegs[1]) 491 .addImm(Type & 0xFFFF) 492 .addImm(0)); 493 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) 494 .addReg(ScratchRegs[1]) 495 .addReg(ScratchRegs[1]) 496 .addImm((Type >> 16) & 0xFFFF) 497 .addImm(16)); 498 499 // Compare the hashes and trap if there's a mismatch. 500 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs) 501 .addReg(AArch64::WZR) 502 .addReg(ScratchRegs[0]) 503 .addReg(ScratchRegs[1]) 504 .addImm(0)); 505 506 MCSymbol *Pass = OutContext.createTempSymbol(); 507 EmitToStreamer(*OutStreamer, 508 MCInstBuilder(AArch64::Bcc) 509 .addImm(AArch64CC::EQ) 510 .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); 511 512 // The base ESR is 0x8000 and the register information is encoded in bits 513 // 0-9 as follows: 514 // - 0-4: n, where the register Xn contains the target address 515 // - 5-9: m, where the register Wm contains the expected type hash 516 // Where n, m are in [0, 30]. 517 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0; 518 unsigned AddrIndex; 519 switch (AddrReg) { 520 default: 521 AddrIndex = AddrReg - AArch64::X0; 522 break; 523 case AArch64::FP: 524 AddrIndex = 29; 525 break; 526 case AArch64::LR: 527 AddrIndex = 30; 528 break; 529 } 530 531 assert(AddrIndex < 31 && TypeIndex < 31); 532 533 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31); 534 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR)); 535 OutStreamer->emitLabel(Pass); 536 } 537 538 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 539 Register Reg = MI.getOperand(0).getReg(); 540 bool IsShort = 541 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES; 542 uint32_t AccessInfo = MI.getOperand(1).getImm(); 543 MCSymbol *&Sym = 544 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)]; 545 if (!Sym) { 546 // FIXME: Make this work on non-ELF. 547 if (!TM.getTargetTriple().isOSBinFormatELF()) 548 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); 549 550 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" + 551 utostr(AccessInfo); 552 if (IsShort) 553 SymName += "_short_v2"; 554 Sym = OutContext.getOrCreateSymbol(SymName); 555 } 556 557 EmitToStreamer(*OutStreamer, 558 MCInstBuilder(AArch64::BL) 559 .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); 560 } 561 562 void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) { 563 if (HwasanMemaccessSymbols.empty()) 564 return; 565 566 const Triple &TT = TM.getTargetTriple(); 567 assert(TT.isOSBinFormatELF()); 568 std::unique_ptr<MCSubtargetInfo> STI( 569 TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); 570 assert(STI && "Unable to create subtarget info"); 571 572 MCSymbol *HwasanTagMismatchV1Sym = 573 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch"); 574 MCSymbol *HwasanTagMismatchV2Sym = 575 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2"); 576 577 const MCSymbolRefExpr *HwasanTagMismatchV1Ref = 578 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext); 579 const MCSymbolRefExpr *HwasanTagMismatchV2Ref = 580 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext); 581 582 for (auto &P : HwasanMemaccessSymbols) { 583 unsigned Reg = std::get<0>(P.first); 584 bool IsShort = std::get<1>(P.first); 585 uint32_t AccessInfo = std::get<2>(P.first); 586 const MCSymbolRefExpr *HwasanTagMismatchRef = 587 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref; 588 MCSymbol *Sym = P.second; 589 590 bool HasMatchAllTag = 591 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1; 592 uint8_t MatchAllTag = 593 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff; 594 unsigned Size = 595 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); 596 bool CompileKernel = 597 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1; 598 599 OutStreamer->switchSection(OutContext.getELFSection( 600 ".text.hot", ELF::SHT_PROGBITS, 601 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(), 602 /*IsComdat=*/true)); 603 604 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); 605 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); 606 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); 607 OutStreamer->emitLabel(Sym); 608 609 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri) 610 .addReg(AArch64::X16) 611 .addReg(Reg) 612 .addImm(4) 613 .addImm(55), 614 *STI); 615 OutStreamer->emitInstruction( 616 MCInstBuilder(AArch64::LDRBBroX) 617 .addReg(AArch64::W16) 618 .addReg(IsShort ? AArch64::X20 : AArch64::X9) 619 .addReg(AArch64::X16) 620 .addImm(0) 621 .addImm(0), 622 *STI); 623 OutStreamer->emitInstruction( 624 MCInstBuilder(AArch64::SUBSXrs) 625 .addReg(AArch64::XZR) 626 .addReg(AArch64::X16) 627 .addReg(Reg) 628 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), 629 *STI); 630 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); 631 OutStreamer->emitInstruction( 632 MCInstBuilder(AArch64::Bcc) 633 .addImm(AArch64CC::NE) 634 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, 635 OutContext)), 636 *STI); 637 MCSymbol *ReturnSym = OutContext.createTempSymbol(); 638 OutStreamer->emitLabel(ReturnSym); 639 OutStreamer->emitInstruction( 640 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); 641 OutStreamer->emitLabel(HandleMismatchOrPartialSym); 642 643 if (HasMatchAllTag) { 644 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri) 645 .addReg(AArch64::X17) 646 .addReg(Reg) 647 .addImm(56) 648 .addImm(63), 649 *STI); 650 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri) 651 .addReg(AArch64::XZR) 652 .addReg(AArch64::X17) 653 .addImm(MatchAllTag) 654 .addImm(0), 655 *STI); 656 OutStreamer->emitInstruction( 657 MCInstBuilder(AArch64::Bcc) 658 .addImm(AArch64CC::EQ) 659 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 660 *STI); 661 } 662 663 if (IsShort) { 664 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri) 665 .addReg(AArch64::WZR) 666 .addReg(AArch64::W16) 667 .addImm(15) 668 .addImm(0), 669 *STI); 670 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); 671 OutStreamer->emitInstruction( 672 MCInstBuilder(AArch64::Bcc) 673 .addImm(AArch64CC::HI) 674 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 675 *STI); 676 677 OutStreamer->emitInstruction( 678 MCInstBuilder(AArch64::ANDXri) 679 .addReg(AArch64::X17) 680 .addReg(Reg) 681 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), 682 *STI); 683 if (Size != 1) 684 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 685 .addReg(AArch64::X17) 686 .addReg(AArch64::X17) 687 .addImm(Size - 1) 688 .addImm(0), 689 *STI); 690 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs) 691 .addReg(AArch64::WZR) 692 .addReg(AArch64::W16) 693 .addReg(AArch64::W17) 694 .addImm(0), 695 *STI); 696 OutStreamer->emitInstruction( 697 MCInstBuilder(AArch64::Bcc) 698 .addImm(AArch64CC::LS) 699 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 700 *STI); 701 702 OutStreamer->emitInstruction( 703 MCInstBuilder(AArch64::ORRXri) 704 .addReg(AArch64::X16) 705 .addReg(Reg) 706 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), 707 *STI); 708 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui) 709 .addReg(AArch64::W16) 710 .addReg(AArch64::X16) 711 .addImm(0), 712 *STI); 713 OutStreamer->emitInstruction( 714 MCInstBuilder(AArch64::SUBSXrs) 715 .addReg(AArch64::XZR) 716 .addReg(AArch64::X16) 717 .addReg(Reg) 718 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), 719 *STI); 720 OutStreamer->emitInstruction( 721 MCInstBuilder(AArch64::Bcc) 722 .addImm(AArch64CC::EQ) 723 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 724 *STI); 725 726 OutStreamer->emitLabel(HandleMismatchSym); 727 } 728 729 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 730 .addReg(AArch64::SP) 731 .addReg(AArch64::X0) 732 .addReg(AArch64::X1) 733 .addReg(AArch64::SP) 734 .addImm(-32), 735 *STI); 736 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) 737 .addReg(AArch64::FP) 738 .addReg(AArch64::LR) 739 .addReg(AArch64::SP) 740 .addImm(29), 741 *STI); 742 743 if (Reg != AArch64::X0) 744 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs) 745 .addReg(AArch64::X0) 746 .addReg(AArch64::XZR) 747 .addReg(Reg) 748 .addImm(0), 749 *STI); 750 OutStreamer->emitInstruction( 751 MCInstBuilder(AArch64::MOVZXi) 752 .addReg(AArch64::X1) 753 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask) 754 .addImm(0), 755 *STI); 756 757 if (CompileKernel) { 758 // The Linux kernel's dynamic loader doesn't support GOT relative 759 // relocations, but it doesn't support late binding either, so just call 760 // the function directly. 761 OutStreamer->emitInstruction( 762 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI); 763 } else { 764 // Intentionally load the GOT entry and branch to it, rather than possibly 765 // late binding the function, which may clobber the registers before we 766 // have a chance to save them. 767 OutStreamer->emitInstruction( 768 MCInstBuilder(AArch64::ADRP) 769 .addReg(AArch64::X16) 770 .addExpr(AArch64MCExpr::create( 771 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE, 772 OutContext)), 773 *STI); 774 OutStreamer->emitInstruction( 775 MCInstBuilder(AArch64::LDRXui) 776 .addReg(AArch64::X16) 777 .addReg(AArch64::X16) 778 .addExpr(AArch64MCExpr::create( 779 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12, 780 OutContext)), 781 *STI); 782 OutStreamer->emitInstruction( 783 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); 784 } 785 } 786 } 787 788 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) { 789 emitHwasanMemaccessSymbols(M); 790 791 const Triple &TT = TM.getTargetTriple(); 792 if (TT.isOSBinFormatMachO()) { 793 // Funny Darwin hack: This flag tells the linker that no global symbols 794 // contain code that falls through to other global symbols (e.g. the obvious 795 // implementation of multiple entry points). If this doesn't occur, the 796 // linker can safely perform dead code stripping. Since LLVM never 797 // generates code that does this, it is always safe to set. 798 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols); 799 } 800 801 // Emit stack and fault map information. 802 FM.serializeToFaultMapSection(); 803 804 } 805 806 void AArch64AsmPrinter::emitLOHs() { 807 SmallVector<MCSymbol *, 3> MCArgs; 808 809 for (const auto &D : AArch64FI->getLOHContainer()) { 810 for (const MachineInstr *MI : D.getArgs()) { 811 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI); 812 assert(LabelIt != LOHInstToLabel.end() && 813 "Label hasn't been inserted for LOH related instruction"); 814 MCArgs.push_back(LabelIt->second); 815 } 816 OutStreamer->emitLOHDirective(D.getKind(), MCArgs); 817 MCArgs.clear(); 818 } 819 } 820 821 void AArch64AsmPrinter::emitFunctionBodyEnd() { 822 if (!AArch64FI->getLOHRelated().empty()) 823 emitLOHs(); 824 } 825 826 /// GetCPISymbol - Return the symbol for the specified constant pool entry. 827 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const { 828 // Darwin uses a linker-private symbol name for constant-pools (to 829 // avoid addends on the relocation?), ELF has no such concept and 830 // uses a normal private symbol. 831 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty()) 832 return OutContext.getOrCreateSymbol( 833 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" + 834 Twine(getFunctionNumber()) + "_" + Twine(CPID)); 835 836 return AsmPrinter::GetCPISymbol(CPID); 837 } 838 839 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, 840 raw_ostream &O) { 841 const MachineOperand &MO = MI->getOperand(OpNum); 842 switch (MO.getType()) { 843 default: 844 llvm_unreachable("<unknown operand type>"); 845 case MachineOperand::MO_Register: { 846 Register Reg = MO.getReg(); 847 assert(Reg.isPhysical()); 848 assert(!MO.getSubReg() && "Subregs should be eliminated!"); 849 O << AArch64InstPrinter::getRegisterName(Reg); 850 break; 851 } 852 case MachineOperand::MO_Immediate: { 853 O << MO.getImm(); 854 break; 855 } 856 case MachineOperand::MO_GlobalAddress: { 857 PrintSymbolOperand(MO, O); 858 break; 859 } 860 case MachineOperand::MO_BlockAddress: { 861 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); 862 Sym->print(O, MAI); 863 break; 864 } 865 } 866 } 867 868 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, 869 raw_ostream &O) { 870 Register Reg = MO.getReg(); 871 switch (Mode) { 872 default: 873 return true; // Unknown mode. 874 case 'w': 875 Reg = getWRegFromXReg(Reg); 876 break; 877 case 'x': 878 Reg = getXRegFromWReg(Reg); 879 break; 880 case 't': 881 Reg = getXRegFromXRegTuple(Reg); 882 break; 883 } 884 885 O << AArch64InstPrinter::getRegisterName(Reg); 886 return false; 887 } 888 889 // Prints the register in MO using class RC using the offset in the 890 // new register class. This should not be used for cross class 891 // printing. 892 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO, 893 const TargetRegisterClass *RC, 894 unsigned AltName, raw_ostream &O) { 895 assert(MO.isReg() && "Should only get here with a register!"); 896 const TargetRegisterInfo *RI = STI->getRegisterInfo(); 897 Register Reg = MO.getReg(); 898 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg)); 899 if (!RI->regsOverlap(RegToPrint, Reg)) 900 return true; 901 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName); 902 return false; 903 } 904 905 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, 906 const char *ExtraCode, raw_ostream &O) { 907 const MachineOperand &MO = MI->getOperand(OpNum); 908 909 // First try the generic code, which knows about modifiers like 'c' and 'n'. 910 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O)) 911 return false; 912 913 // Does this asm operand have a single letter operand modifier? 914 if (ExtraCode && ExtraCode[0]) { 915 if (ExtraCode[1] != 0) 916 return true; // Unknown modifier. 917 918 switch (ExtraCode[0]) { 919 default: 920 return true; // Unknown modifier. 921 case 'w': // Print W register 922 case 'x': // Print X register 923 if (MO.isReg()) 924 return printAsmMRegister(MO, ExtraCode[0], O); 925 if (MO.isImm() && MO.getImm() == 0) { 926 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR; 927 O << AArch64InstPrinter::getRegisterName(Reg); 928 return false; 929 } 930 printOperand(MI, OpNum, O); 931 return false; 932 case 'b': // Print B register. 933 case 'h': // Print H register. 934 case 's': // Print S register. 935 case 'd': // Print D register. 936 case 'q': // Print Q register. 937 case 'z': // Print Z register. 938 if (MO.isReg()) { 939 const TargetRegisterClass *RC; 940 switch (ExtraCode[0]) { 941 case 'b': 942 RC = &AArch64::FPR8RegClass; 943 break; 944 case 'h': 945 RC = &AArch64::FPR16RegClass; 946 break; 947 case 's': 948 RC = &AArch64::FPR32RegClass; 949 break; 950 case 'd': 951 RC = &AArch64::FPR64RegClass; 952 break; 953 case 'q': 954 RC = &AArch64::FPR128RegClass; 955 break; 956 case 'z': 957 RC = &AArch64::ZPRRegClass; 958 break; 959 default: 960 return true; 961 } 962 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O); 963 } 964 printOperand(MI, OpNum, O); 965 return false; 966 } 967 } 968 969 // According to ARM, we should emit x and v registers unless we have a 970 // modifier. 971 if (MO.isReg()) { 972 Register Reg = MO.getReg(); 973 974 // If this is a w or x register, print an x register. 975 if (AArch64::GPR32allRegClass.contains(Reg) || 976 AArch64::GPR64allRegClass.contains(Reg)) 977 return printAsmMRegister(MO, 'x', O); 978 979 // If this is an x register tuple, print an x register. 980 if (AArch64::GPR64x8ClassRegClass.contains(Reg)) 981 return printAsmMRegister(MO, 't', O); 982 983 unsigned AltName = AArch64::NoRegAltName; 984 const TargetRegisterClass *RegClass; 985 if (AArch64::ZPRRegClass.contains(Reg)) { 986 RegClass = &AArch64::ZPRRegClass; 987 } else if (AArch64::PPRRegClass.contains(Reg)) { 988 RegClass = &AArch64::PPRRegClass; 989 } else if (AArch64::PNRRegClass.contains(Reg)) { 990 RegClass = &AArch64::PNRRegClass; 991 } else { 992 RegClass = &AArch64::FPR128RegClass; 993 AltName = AArch64::vreg; 994 } 995 996 // If this is a b, h, s, d, or q register, print it as a v register. 997 return printAsmRegInClass(MO, RegClass, AltName, O); 998 } 999 1000 printOperand(MI, OpNum, O); 1001 return false; 1002 } 1003 1004 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, 1005 unsigned OpNum, 1006 const char *ExtraCode, 1007 raw_ostream &O) { 1008 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a') 1009 return true; // Unknown modifier. 1010 1011 const MachineOperand &MO = MI->getOperand(OpNum); 1012 assert(MO.isReg() && "unexpected inline asm memory operand"); 1013 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]"; 1014 return false; 1015 } 1016 1017 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI, 1018 raw_ostream &OS) { 1019 unsigned NOps = MI->getNumOperands(); 1020 assert(NOps == 4); 1021 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: "; 1022 // cast away const; DIetc do not take const operands for some reason. 1023 OS << MI->getDebugVariable()->getName(); 1024 OS << " <- "; 1025 // Frame address. Currently handles register +- offset only. 1026 assert(MI->isIndirectDebugValue()); 1027 OS << '['; 1028 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(), 1029 MI->debug_operands().end()); 1030 I < E; ++I) { 1031 if (I != 0) 1032 OS << ", "; 1033 printOperand(MI, I, OS); 1034 } 1035 OS << ']'; 1036 OS << "+"; 1037 printOperand(MI, NOps - 2, OS); 1038 } 1039 1040 void AArch64AsmPrinter::emitJumpTableInfo() { 1041 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); 1042 if (!MJTI) return; 1043 1044 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1045 if (JT.empty()) return; 1046 1047 const TargetLoweringObjectFile &TLOF = getObjFileLowering(); 1048 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM); 1049 OutStreamer->switchSection(ReadOnlySec); 1050 1051 auto AFI = MF->getInfo<AArch64FunctionInfo>(); 1052 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) { 1053 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs; 1054 1055 // If this jump table was deleted, ignore it. 1056 if (JTBBs.empty()) continue; 1057 1058 unsigned Size = AFI->getJumpTableEntrySize(JTI); 1059 emitAlignment(Align(Size)); 1060 OutStreamer->emitLabel(GetJTISymbol(JTI)); 1061 1062 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI); 1063 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext); 1064 1065 for (auto *JTBB : JTBBs) { 1066 const MCExpr *Value = 1067 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext); 1068 1069 // Each entry is: 1070 // .byte/.hword (LBB - Lbase)>>2 1071 // or plain: 1072 // .word LBB - Lbase 1073 Value = MCBinaryExpr::createSub(Value, Base, OutContext); 1074 if (Size != 4) 1075 Value = MCBinaryExpr::createLShr( 1076 Value, MCConstantExpr::create(2, OutContext), OutContext); 1077 1078 OutStreamer->emitValue(Value, Size); 1079 } 1080 } 1081 } 1082 1083 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *, 1084 codeview::JumpTableEntrySize> 1085 AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI, 1086 const MachineInstr *BranchInstr, 1087 const MCSymbol *BranchLabel) const { 1088 const auto AFI = MF->getInfo<AArch64FunctionInfo>(); 1089 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI); 1090 codeview::JumpTableEntrySize EntrySize; 1091 switch (AFI->getJumpTableEntrySize(JTI)) { 1092 case 1: 1093 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft; 1094 break; 1095 case 2: 1096 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft; 1097 break; 1098 case 4: 1099 EntrySize = codeview::JumpTableEntrySize::Int32; 1100 break; 1101 default: 1102 llvm_unreachable("Unexpected jump table entry size"); 1103 } 1104 return std::make_tuple(Base, 0, BranchLabel, EntrySize); 1105 } 1106 1107 void AArch64AsmPrinter::emitFunctionEntryLabel() { 1108 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall || 1109 MF->getFunction().getCallingConv() == 1110 CallingConv::AArch64_SVE_VectorCall || 1111 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) { 1112 auto *TS = 1113 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 1114 TS->emitDirectiveVariantPCS(CurrentFnSym); 1115 } 1116 1117 return AsmPrinter::emitFunctionEntryLabel(); 1118 } 1119 1120 /// Small jump tables contain an unsigned byte or half, representing the offset 1121 /// from the lowest-addressed possible destination to the desired basic 1122 /// block. Since all instructions are 4-byte aligned, this is further compressed 1123 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to 1124 /// materialize the correct destination we need: 1125 /// 1126 /// adr xDest, .LBB0_0 1127 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh). 1128 /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries) 1129 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer, 1130 const llvm::MachineInstr &MI) { 1131 Register DestReg = MI.getOperand(0).getReg(); 1132 Register ScratchReg = MI.getOperand(1).getReg(); 1133 Register ScratchRegW = 1134 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32); 1135 Register TableReg = MI.getOperand(2).getReg(); 1136 Register EntryReg = MI.getOperand(3).getReg(); 1137 int JTIdx = MI.getOperand(4).getIndex(); 1138 int Size = AArch64FI->getJumpTableEntrySize(JTIdx); 1139 1140 // This has to be first because the compression pass based its reachability 1141 // calculations on the start of the JumpTableDest instruction. 1142 auto Label = 1143 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx); 1144 1145 // If we don't already have a symbol to use as the base, use the ADR 1146 // instruction itself. 1147 if (!Label) { 1148 Label = MF->getContext().createTempSymbol(); 1149 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label); 1150 OutStreamer.emitLabel(Label); 1151 } 1152 1153 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext()); 1154 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR) 1155 .addReg(DestReg) 1156 .addExpr(LabelExpr)); 1157 1158 // Load the number of instruction-steps to offset from the label. 1159 unsigned LdrOpcode; 1160 switch (Size) { 1161 case 1: LdrOpcode = AArch64::LDRBBroX; break; 1162 case 2: LdrOpcode = AArch64::LDRHHroX; break; 1163 case 4: LdrOpcode = AArch64::LDRSWroX; break; 1164 default: 1165 llvm_unreachable("Unknown jump table size"); 1166 } 1167 1168 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode) 1169 .addReg(Size == 4 ? ScratchReg : ScratchRegW) 1170 .addReg(TableReg) 1171 .addReg(EntryReg) 1172 .addImm(0) 1173 .addImm(Size == 1 ? 0 : 1)); 1174 1175 // Add to the already materialized base label address, multiplying by 4 if 1176 // compressed. 1177 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs) 1178 .addReg(DestReg) 1179 .addReg(DestReg) 1180 .addReg(ScratchReg) 1181 .addImm(Size == 4 ? 0 : 2)); 1182 } 1183 1184 void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer, 1185 const llvm::MachineInstr &MI) { 1186 unsigned Opcode = MI.getOpcode(); 1187 assert(STI->hasMOPS()); 1188 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo); 1189 1190 const auto Ops = [Opcode]() -> std::array<unsigned, 3> { 1191 if (Opcode == AArch64::MOPSMemoryCopyPseudo) 1192 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE}; 1193 if (Opcode == AArch64::MOPSMemoryMovePseudo) 1194 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE}; 1195 if (Opcode == AArch64::MOPSMemorySetPseudo) 1196 return {AArch64::SETP, AArch64::SETM, AArch64::SETE}; 1197 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo) 1198 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE}; 1199 llvm_unreachable("Unhandled memory operation pseudo"); 1200 }(); 1201 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo || 1202 Opcode == AArch64::MOPSMemorySetTaggingPseudo; 1203 1204 for (auto Op : Ops) { 1205 int i = 0; 1206 auto MCIB = MCInstBuilder(Op); 1207 // Destination registers 1208 MCIB.addReg(MI.getOperand(i++).getReg()); 1209 MCIB.addReg(MI.getOperand(i++).getReg()); 1210 if (!IsSet) 1211 MCIB.addReg(MI.getOperand(i++).getReg()); 1212 // Input registers 1213 MCIB.addReg(MI.getOperand(i++).getReg()); 1214 MCIB.addReg(MI.getOperand(i++).getReg()); 1215 MCIB.addReg(MI.getOperand(i++).getReg()); 1216 1217 EmitToStreamer(OutStreamer, MCIB); 1218 } 1219 } 1220 1221 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 1222 const MachineInstr &MI) { 1223 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); 1224 1225 auto &Ctx = OutStreamer.getContext(); 1226 MCSymbol *MILabel = Ctx.createTempSymbol(); 1227 OutStreamer.emitLabel(MILabel); 1228 1229 SM.recordStackMap(*MILabel, MI); 1230 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); 1231 1232 // Scan ahead to trim the shadow. 1233 const MachineBasicBlock &MBB = *MI.getParent(); 1234 MachineBasicBlock::const_iterator MII(MI); 1235 ++MII; 1236 while (NumNOPBytes > 0) { 1237 if (MII == MBB.end() || MII->isCall() || 1238 MII->getOpcode() == AArch64::DBG_VALUE || 1239 MII->getOpcode() == TargetOpcode::PATCHPOINT || 1240 MII->getOpcode() == TargetOpcode::STACKMAP) 1241 break; 1242 ++MII; 1243 NumNOPBytes -= 4; 1244 } 1245 1246 // Emit nops. 1247 for (unsigned i = 0; i < NumNOPBytes; i += 4) 1248 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1249 } 1250 1251 // Lower a patchpoint of the form: 1252 // [<def>], <id>, <numBytes>, <target>, <numArgs> 1253 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 1254 const MachineInstr &MI) { 1255 auto &Ctx = OutStreamer.getContext(); 1256 MCSymbol *MILabel = Ctx.createTempSymbol(); 1257 OutStreamer.emitLabel(MILabel); 1258 SM.recordPatchPoint(*MILabel, MI); 1259 1260 PatchPointOpers Opers(&MI); 1261 1262 int64_t CallTarget = Opers.getCallTarget().getImm(); 1263 unsigned EncodedBytes = 0; 1264 if (CallTarget) { 1265 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && 1266 "High 16 bits of call target should be zero."); 1267 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg(); 1268 EncodedBytes = 16; 1269 // Materialize the jump address: 1270 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi) 1271 .addReg(ScratchReg) 1272 .addImm((CallTarget >> 32) & 0xFFFF) 1273 .addImm(32)); 1274 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) 1275 .addReg(ScratchReg) 1276 .addReg(ScratchReg) 1277 .addImm((CallTarget >> 16) & 0xFFFF) 1278 .addImm(16)); 1279 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) 1280 .addReg(ScratchReg) 1281 .addReg(ScratchReg) 1282 .addImm(CallTarget & 0xFFFF) 1283 .addImm(0)); 1284 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg)); 1285 } 1286 // Emit padding. 1287 unsigned NumBytes = Opers.getNumPatchBytes(); 1288 assert(NumBytes >= EncodedBytes && 1289 "Patchpoint can't request size less than the length of a call."); 1290 assert((NumBytes - EncodedBytes) % 4 == 0 && 1291 "Invalid number of NOP bytes requested!"); 1292 for (unsigned i = EncodedBytes; i < NumBytes; i += 4) 1293 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1294 } 1295 1296 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 1297 const MachineInstr &MI) { 1298 StatepointOpers SOpers(&MI); 1299 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 1300 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); 1301 for (unsigned i = 0; i < PatchBytes; i += 4) 1302 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1303 } else { 1304 // Lower call target and choose correct opcode 1305 const MachineOperand &CallTarget = SOpers.getCallTarget(); 1306 MCOperand CallTargetMCOp; 1307 unsigned CallOpcode; 1308 switch (CallTarget.getType()) { 1309 case MachineOperand::MO_GlobalAddress: 1310 case MachineOperand::MO_ExternalSymbol: 1311 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp); 1312 CallOpcode = AArch64::BL; 1313 break; 1314 case MachineOperand::MO_Immediate: 1315 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm()); 1316 CallOpcode = AArch64::BL; 1317 break; 1318 case MachineOperand::MO_Register: 1319 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg()); 1320 CallOpcode = AArch64::BLR; 1321 break; 1322 default: 1323 llvm_unreachable("Unsupported operand type in statepoint call target"); 1324 break; 1325 } 1326 1327 EmitToStreamer(OutStreamer, 1328 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp)); 1329 } 1330 1331 auto &Ctx = OutStreamer.getContext(); 1332 MCSymbol *MILabel = Ctx.createTempSymbol(); 1333 OutStreamer.emitLabel(MILabel); 1334 SM.recordStatepoint(*MILabel, MI); 1335 } 1336 1337 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) { 1338 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>, 1339 // <opcode>, <operands> 1340 1341 Register DefRegister = FaultingMI.getOperand(0).getReg(); 1342 FaultMaps::FaultKind FK = 1343 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm()); 1344 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol(); 1345 unsigned Opcode = FaultingMI.getOperand(3).getImm(); 1346 unsigned OperandsBeginIdx = 4; 1347 1348 auto &Ctx = OutStreamer->getContext(); 1349 MCSymbol *FaultingLabel = Ctx.createTempSymbol(); 1350 OutStreamer->emitLabel(FaultingLabel); 1351 1352 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!"); 1353 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel); 1354 1355 MCInst MI; 1356 MI.setOpcode(Opcode); 1357 1358 if (DefRegister != (Register)0) 1359 MI.addOperand(MCOperand::createReg(DefRegister)); 1360 1361 for (const MachineOperand &MO : 1362 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) { 1363 MCOperand Dest; 1364 lowerOperand(MO, Dest); 1365 MI.addOperand(Dest); 1366 } 1367 1368 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName()); 1369 OutStreamer->emitInstruction(MI, getSubtargetInfo()); 1370 } 1371 1372 void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) { 1373 Register DestReg = MI.getOperand(0).getReg(); 1374 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() && 1375 STI->isNeonAvailable()) { 1376 // Convert H/S register to corresponding D register 1377 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31) 1378 DestReg = AArch64::D0 + (DestReg - AArch64::H0); 1379 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31) 1380 DestReg = AArch64::D0 + (DestReg - AArch64::S0); 1381 else 1382 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); 1383 1384 MCInst MOVI; 1385 MOVI.setOpcode(AArch64::MOVID); 1386 MOVI.addOperand(MCOperand::createReg(DestReg)); 1387 MOVI.addOperand(MCOperand::createImm(0)); 1388 EmitToStreamer(*OutStreamer, MOVI); 1389 } else { 1390 MCInst FMov; 1391 switch (MI.getOpcode()) { 1392 default: llvm_unreachable("Unexpected opcode"); 1393 case AArch64::FMOVH0: 1394 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); 1395 if (!STI->hasFullFP16()) 1396 DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); 1397 FMov.addOperand(MCOperand::createReg(DestReg)); 1398 FMov.addOperand(MCOperand::createReg(AArch64::WZR)); 1399 break; 1400 case AArch64::FMOVS0: 1401 FMov.setOpcode(AArch64::FMOVWSr); 1402 FMov.addOperand(MCOperand::createReg(DestReg)); 1403 FMov.addOperand(MCOperand::createReg(AArch64::WZR)); 1404 break; 1405 case AArch64::FMOVD0: 1406 FMov.setOpcode(AArch64::FMOVXDr); 1407 FMov.addOperand(MCOperand::createReg(DestReg)); 1408 FMov.addOperand(MCOperand::createReg(AArch64::XZR)); 1409 break; 1410 } 1411 EmitToStreamer(*OutStreamer, FMov); 1412 } 1413 } 1414 1415 // Simple pseudo-instructions have their lowering (with expansion to real 1416 // instructions) auto-generated. 1417 #include "AArch64GenMCPseudoLowering.inc" 1418 1419 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { 1420 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits()); 1421 1422 // Do any auto-generated pseudo lowerings. 1423 if (emitPseudoExpansionLowering(*OutStreamer, MI)) 1424 return; 1425 1426 if (MI->getOpcode() == AArch64::ADRP) { 1427 for (auto &Opd : MI->operands()) { 1428 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) == 1429 "swift_async_extendedFramePointerFlags") { 1430 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true; 1431 } 1432 } 1433 } 1434 1435 if (AArch64FI->getLOHRelated().count(MI)) { 1436 // Generate a label for LOH related instruction 1437 MCSymbol *LOHLabel = createTempSymbol("loh"); 1438 // Associate the instruction with the label 1439 LOHInstToLabel[MI] = LOHLabel; 1440 OutStreamer->emitLabel(LOHLabel); 1441 } 1442 1443 AArch64TargetStreamer *TS = 1444 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 1445 // Do any manual lowerings. 1446 switch (MI->getOpcode()) { 1447 default: 1448 break; 1449 case AArch64::HINT: { 1450 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for 1451 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be 1452 // non-empty. If MI is the initial BTI, place the 1453 // __patchable_function_entries label after BTI. 1454 if (CurrentPatchableFunctionEntrySym && 1455 CurrentPatchableFunctionEntrySym == CurrentFnBegin && 1456 MI == &MF->front().front()) { 1457 int64_t Imm = MI->getOperand(0).getImm(); 1458 if ((Imm & 32) && (Imm & 6)) { 1459 MCInst Inst; 1460 MCInstLowering.Lower(MI, Inst); 1461 EmitToStreamer(*OutStreamer, Inst); 1462 CurrentPatchableFunctionEntrySym = createTempSymbol("patch"); 1463 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym); 1464 return; 1465 } 1466 } 1467 break; 1468 } 1469 case AArch64::MOVMCSym: { 1470 Register DestReg = MI->getOperand(0).getReg(); 1471 const MachineOperand &MO_Sym = MI->getOperand(1); 1472 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym); 1473 MCOperand Hi_MCSym, Lo_MCSym; 1474 1475 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S); 1476 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC); 1477 1478 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym); 1479 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym); 1480 1481 MCInst MovZ; 1482 MovZ.setOpcode(AArch64::MOVZXi); 1483 MovZ.addOperand(MCOperand::createReg(DestReg)); 1484 MovZ.addOperand(Hi_MCSym); 1485 MovZ.addOperand(MCOperand::createImm(16)); 1486 EmitToStreamer(*OutStreamer, MovZ); 1487 1488 MCInst MovK; 1489 MovK.setOpcode(AArch64::MOVKXi); 1490 MovK.addOperand(MCOperand::createReg(DestReg)); 1491 MovK.addOperand(MCOperand::createReg(DestReg)); 1492 MovK.addOperand(Lo_MCSym); 1493 MovK.addOperand(MCOperand::createImm(0)); 1494 EmitToStreamer(*OutStreamer, MovK); 1495 return; 1496 } 1497 case AArch64::MOVIv2d_ns: 1498 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0". 1499 // as movi is more efficient across all cores. Newer cores can eliminate 1500 // fmovs early and there is no difference with movi, but this not true for 1501 // all implementations. 1502 // 1503 // The floating-point version doesn't quite work in rare cases on older 1504 // CPUs, so on those targets we lower this instruction to movi.16b instead. 1505 if (STI->hasZeroCycleZeroingFPWorkaround() && 1506 MI->getOperand(1).getImm() == 0) { 1507 MCInst TmpInst; 1508 TmpInst.setOpcode(AArch64::MOVIv16b_ns); 1509 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); 1510 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm())); 1511 EmitToStreamer(*OutStreamer, TmpInst); 1512 return; 1513 } 1514 break; 1515 1516 case AArch64::DBG_VALUE: 1517 case AArch64::DBG_VALUE_LIST: 1518 if (isVerbose() && OutStreamer->hasRawTextSupport()) { 1519 SmallString<128> TmpStr; 1520 raw_svector_ostream OS(TmpStr); 1521 PrintDebugValueComment(MI, OS); 1522 OutStreamer->emitRawText(StringRef(OS.str())); 1523 } 1524 return; 1525 1526 case AArch64::EMITBKEY: { 1527 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); 1528 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && 1529 ExceptionHandlingType != ExceptionHandling::ARM) 1530 return; 1531 1532 if (getFunctionCFISectionType(*MF) == CFISection::None) 1533 return; 1534 1535 OutStreamer->emitCFIBKeyFrame(); 1536 return; 1537 } 1538 1539 case AArch64::EMITMTETAGGED: { 1540 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); 1541 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && 1542 ExceptionHandlingType != ExceptionHandling::ARM) 1543 return; 1544 1545 if (getFunctionCFISectionType(*MF) != CFISection::None) 1546 OutStreamer->emitCFIMTETaggedFrame(); 1547 return; 1548 } 1549 1550 // Tail calls use pseudo instructions so they have the proper code-gen 1551 // attributes (isCall, isReturn, etc.). We lower them to the real 1552 // instruction here. 1553 case AArch64::TCRETURNri: 1554 case AArch64::TCRETURNriBTI: 1555 case AArch64::TCRETURNriALL: { 1556 MCInst TmpInst; 1557 TmpInst.setOpcode(AArch64::BR); 1558 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); 1559 EmitToStreamer(*OutStreamer, TmpInst); 1560 return; 1561 } 1562 case AArch64::TCRETURNdi: { 1563 MCOperand Dest; 1564 MCInstLowering.lowerOperand(MI->getOperand(0), Dest); 1565 MCInst TmpInst; 1566 TmpInst.setOpcode(AArch64::B); 1567 TmpInst.addOperand(Dest); 1568 EmitToStreamer(*OutStreamer, TmpInst); 1569 return; 1570 } 1571 case AArch64::SpeculationBarrierISBDSBEndBB: { 1572 // Print DSB SYS + ISB 1573 MCInst TmpInstDSB; 1574 TmpInstDSB.setOpcode(AArch64::DSB); 1575 TmpInstDSB.addOperand(MCOperand::createImm(0xf)); 1576 EmitToStreamer(*OutStreamer, TmpInstDSB); 1577 MCInst TmpInstISB; 1578 TmpInstISB.setOpcode(AArch64::ISB); 1579 TmpInstISB.addOperand(MCOperand::createImm(0xf)); 1580 EmitToStreamer(*OutStreamer, TmpInstISB); 1581 return; 1582 } 1583 case AArch64::SpeculationBarrierSBEndBB: { 1584 // Print SB 1585 MCInst TmpInstSB; 1586 TmpInstSB.setOpcode(AArch64::SB); 1587 EmitToStreamer(*OutStreamer, TmpInstSB); 1588 return; 1589 } 1590 case AArch64::TLSDESC_CALLSEQ: { 1591 /// lower this to: 1592 /// adrp x0, :tlsdesc:var 1593 /// ldr x1, [x0, #:tlsdesc_lo12:var] 1594 /// add x0, x0, #:tlsdesc_lo12:var 1595 /// .tlsdesccall var 1596 /// blr x1 1597 /// (TPIDR_EL0 offset now in x0) 1598 const MachineOperand &MO_Sym = MI->getOperand(0); 1599 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym); 1600 MCOperand Sym, SymTLSDescLo12, SymTLSDesc; 1601 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF); 1602 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE); 1603 MCInstLowering.lowerOperand(MO_Sym, Sym); 1604 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12); 1605 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc); 1606 1607 MCInst Adrp; 1608 Adrp.setOpcode(AArch64::ADRP); 1609 Adrp.addOperand(MCOperand::createReg(AArch64::X0)); 1610 Adrp.addOperand(SymTLSDesc); 1611 EmitToStreamer(*OutStreamer, Adrp); 1612 1613 MCInst Ldr; 1614 if (STI->isTargetILP32()) { 1615 Ldr.setOpcode(AArch64::LDRWui); 1616 Ldr.addOperand(MCOperand::createReg(AArch64::W1)); 1617 } else { 1618 Ldr.setOpcode(AArch64::LDRXui); 1619 Ldr.addOperand(MCOperand::createReg(AArch64::X1)); 1620 } 1621 Ldr.addOperand(MCOperand::createReg(AArch64::X0)); 1622 Ldr.addOperand(SymTLSDescLo12); 1623 Ldr.addOperand(MCOperand::createImm(0)); 1624 EmitToStreamer(*OutStreamer, Ldr); 1625 1626 MCInst Add; 1627 if (STI->isTargetILP32()) { 1628 Add.setOpcode(AArch64::ADDWri); 1629 Add.addOperand(MCOperand::createReg(AArch64::W0)); 1630 Add.addOperand(MCOperand::createReg(AArch64::W0)); 1631 } else { 1632 Add.setOpcode(AArch64::ADDXri); 1633 Add.addOperand(MCOperand::createReg(AArch64::X0)); 1634 Add.addOperand(MCOperand::createReg(AArch64::X0)); 1635 } 1636 Add.addOperand(SymTLSDescLo12); 1637 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0))); 1638 EmitToStreamer(*OutStreamer, Add); 1639 1640 // Emit a relocation-annotation. This expands to no code, but requests 1641 // the following instruction gets an R_AARCH64_TLSDESC_CALL. 1642 MCInst TLSDescCall; 1643 TLSDescCall.setOpcode(AArch64::TLSDESCCALL); 1644 TLSDescCall.addOperand(Sym); 1645 EmitToStreamer(*OutStreamer, TLSDescCall); 1646 1647 MCInst Blr; 1648 Blr.setOpcode(AArch64::BLR); 1649 Blr.addOperand(MCOperand::createReg(AArch64::X1)); 1650 EmitToStreamer(*OutStreamer, Blr); 1651 1652 return; 1653 } 1654 1655 case AArch64::JumpTableDest32: 1656 case AArch64::JumpTableDest16: 1657 case AArch64::JumpTableDest8: 1658 LowerJumpTableDest(*OutStreamer, *MI); 1659 return; 1660 1661 case AArch64::FMOVH0: 1662 case AArch64::FMOVS0: 1663 case AArch64::FMOVD0: 1664 emitFMov0(*MI); 1665 return; 1666 1667 case AArch64::MOPSMemoryCopyPseudo: 1668 case AArch64::MOPSMemoryMovePseudo: 1669 case AArch64::MOPSMemorySetPseudo: 1670 case AArch64::MOPSMemorySetTaggingPseudo: 1671 LowerMOPS(*OutStreamer, *MI); 1672 return; 1673 1674 case TargetOpcode::STACKMAP: 1675 return LowerSTACKMAP(*OutStreamer, SM, *MI); 1676 1677 case TargetOpcode::PATCHPOINT: 1678 return LowerPATCHPOINT(*OutStreamer, SM, *MI); 1679 1680 case TargetOpcode::STATEPOINT: 1681 return LowerSTATEPOINT(*OutStreamer, SM, *MI); 1682 1683 case TargetOpcode::FAULTING_OP: 1684 return LowerFAULTING_OP(*MI); 1685 1686 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: 1687 LowerPATCHABLE_FUNCTION_ENTER(*MI); 1688 return; 1689 1690 case TargetOpcode::PATCHABLE_FUNCTION_EXIT: 1691 LowerPATCHABLE_FUNCTION_EXIT(*MI); 1692 return; 1693 1694 case TargetOpcode::PATCHABLE_TAIL_CALL: 1695 LowerPATCHABLE_TAIL_CALL(*MI); 1696 return; 1697 case TargetOpcode::PATCHABLE_EVENT_CALL: 1698 return LowerPATCHABLE_EVENT_CALL(*MI, false); 1699 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL: 1700 return LowerPATCHABLE_EVENT_CALL(*MI, true); 1701 1702 case AArch64::KCFI_CHECK: 1703 LowerKCFI_CHECK(*MI); 1704 return; 1705 1706 case AArch64::HWASAN_CHECK_MEMACCESS: 1707 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: 1708 LowerHWASAN_CHECK_MEMACCESS(*MI); 1709 return; 1710 1711 case AArch64::SEH_StackAlloc: 1712 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm()); 1713 return; 1714 1715 case AArch64::SEH_SaveFPLR: 1716 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm()); 1717 return; 1718 1719 case AArch64::SEH_SaveFPLR_X: 1720 assert(MI->getOperand(0).getImm() < 0 && 1721 "Pre increment SEH opcode must have a negative offset"); 1722 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm()); 1723 return; 1724 1725 case AArch64::SEH_SaveReg: 1726 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(), 1727 MI->getOperand(1).getImm()); 1728 return; 1729 1730 case AArch64::SEH_SaveReg_X: 1731 assert(MI->getOperand(1).getImm() < 0 && 1732 "Pre increment SEH opcode must have a negative offset"); 1733 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(), 1734 -MI->getOperand(1).getImm()); 1735 return; 1736 1737 case AArch64::SEH_SaveRegP: 1738 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 && 1739 MI->getOperand(0).getImm() <= 28) { 1740 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 && 1741 "Register paired with LR must be odd"); 1742 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(), 1743 MI->getOperand(2).getImm()); 1744 return; 1745 } 1746 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1747 "Non-consecutive registers not allowed for save_regp"); 1748 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(), 1749 MI->getOperand(2).getImm()); 1750 return; 1751 1752 case AArch64::SEH_SaveRegP_X: 1753 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1754 "Non-consecutive registers not allowed for save_regp_x"); 1755 assert(MI->getOperand(2).getImm() < 0 && 1756 "Pre increment SEH opcode must have a negative offset"); 1757 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(), 1758 -MI->getOperand(2).getImm()); 1759 return; 1760 1761 case AArch64::SEH_SaveFReg: 1762 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(), 1763 MI->getOperand(1).getImm()); 1764 return; 1765 1766 case AArch64::SEH_SaveFReg_X: 1767 assert(MI->getOperand(1).getImm() < 0 && 1768 "Pre increment SEH opcode must have a negative offset"); 1769 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(), 1770 -MI->getOperand(1).getImm()); 1771 return; 1772 1773 case AArch64::SEH_SaveFRegP: 1774 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1775 "Non-consecutive registers not allowed for save_regp"); 1776 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(), 1777 MI->getOperand(2).getImm()); 1778 return; 1779 1780 case AArch64::SEH_SaveFRegP_X: 1781 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1782 "Non-consecutive registers not allowed for save_regp_x"); 1783 assert(MI->getOperand(2).getImm() < 0 && 1784 "Pre increment SEH opcode must have a negative offset"); 1785 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(), 1786 -MI->getOperand(2).getImm()); 1787 return; 1788 1789 case AArch64::SEH_SetFP: 1790 TS->emitARM64WinCFISetFP(); 1791 return; 1792 1793 case AArch64::SEH_AddFP: 1794 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm()); 1795 return; 1796 1797 case AArch64::SEH_Nop: 1798 TS->emitARM64WinCFINop(); 1799 return; 1800 1801 case AArch64::SEH_PrologEnd: 1802 TS->emitARM64WinCFIPrologEnd(); 1803 return; 1804 1805 case AArch64::SEH_EpilogStart: 1806 TS->emitARM64WinCFIEpilogStart(); 1807 return; 1808 1809 case AArch64::SEH_EpilogEnd: 1810 TS->emitARM64WinCFIEpilogEnd(); 1811 return; 1812 1813 case AArch64::SEH_PACSignLR: 1814 TS->emitARM64WinCFIPACSignLR(); 1815 return; 1816 } 1817 1818 // Finally, do the automated lowerings for everything else. 1819 MCInst TmpInst; 1820 MCInstLowering.Lower(MI, TmpInst); 1821 EmitToStreamer(*OutStreamer, TmpInst); 1822 } 1823 1824 void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, 1825 MCSymbol *LazyPointer) { 1826 // _ifunc: 1827 // adrp x16, lazy_pointer@GOTPAGE 1828 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] 1829 // ldr x16, [x16] 1830 // br x16 1831 1832 { 1833 MCInst Adrp; 1834 Adrp.setOpcode(AArch64::ADRP); 1835 Adrp.addOperand(MCOperand::createReg(AArch64::X16)); 1836 MCOperand SymPage; 1837 MCInstLowering.lowerOperand( 1838 MachineOperand::CreateMCSymbol(LazyPointer, 1839 AArch64II::MO_GOT | AArch64II::MO_PAGE), 1840 SymPage); 1841 Adrp.addOperand(SymPage); 1842 OutStreamer->emitInstruction(Adrp, *STI); 1843 } 1844 1845 { 1846 MCInst Ldr; 1847 Ldr.setOpcode(AArch64::LDRXui); 1848 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1849 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1850 MCOperand SymPageOff; 1851 MCInstLowering.lowerOperand( 1852 MachineOperand::CreateMCSymbol(LazyPointer, AArch64II::MO_GOT | 1853 AArch64II::MO_PAGEOFF), 1854 SymPageOff); 1855 Ldr.addOperand(SymPageOff); 1856 Ldr.addOperand(MCOperand::createImm(0)); 1857 OutStreamer->emitInstruction(Ldr, *STI); 1858 } 1859 1860 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui) 1861 .addReg(AArch64::X16) 1862 .addReg(AArch64::X16) 1863 .addImm(0), 1864 *STI); 1865 1866 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() 1867 ? AArch64::BRAAZ 1868 : AArch64::BR) 1869 .addReg(AArch64::X16), 1870 *STI); 1871 } 1872 1873 void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M, 1874 const GlobalIFunc &GI, 1875 MCSymbol *LazyPointer) { 1876 // These stub helpers are only ever called once, so here we're optimizing for 1877 // minimum size by using the pre-indexed store variants, which saves a few 1878 // bytes of instructions to bump & restore sp. 1879 1880 // _ifunc.stub_helper: 1881 // stp fp, lr, [sp, #-16]! 1882 // mov fp, sp 1883 // stp x1, x0, [sp, #-16]! 1884 // stp x3, x2, [sp, #-16]! 1885 // stp x5, x4, [sp, #-16]! 1886 // stp x7, x6, [sp, #-16]! 1887 // stp d1, d0, [sp, #-16]! 1888 // stp d3, d2, [sp, #-16]! 1889 // stp d5, d4, [sp, #-16]! 1890 // stp d7, d6, [sp, #-16]! 1891 // bl _resolver 1892 // adrp x16, lazy_pointer@GOTPAGE 1893 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] 1894 // str x0, [x16] 1895 // mov x16, x0 1896 // ldp d7, d6, [sp], #16 1897 // ldp d5, d4, [sp], #16 1898 // ldp d3, d2, [sp], #16 1899 // ldp d1, d0, [sp], #16 1900 // ldp x7, x6, [sp], #16 1901 // ldp x5, x4, [sp], #16 1902 // ldp x3, x2, [sp], #16 1903 // ldp x1, x0, [sp], #16 1904 // ldp fp, lr, [sp], #16 1905 // br x16 1906 1907 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 1908 .addReg(AArch64::SP) 1909 .addReg(AArch64::FP) 1910 .addReg(AArch64::LR) 1911 .addReg(AArch64::SP) 1912 .addImm(-2), 1913 *STI); 1914 1915 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 1916 .addReg(AArch64::FP) 1917 .addReg(AArch64::SP) 1918 .addImm(0) 1919 .addImm(0), 1920 *STI); 1921 1922 for (int I = 0; I != 4; ++I) 1923 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 1924 .addReg(AArch64::SP) 1925 .addReg(AArch64::X1 + 2 * I) 1926 .addReg(AArch64::X0 + 2 * I) 1927 .addReg(AArch64::SP) 1928 .addImm(-2), 1929 *STI); 1930 1931 for (int I = 0; I != 4; ++I) 1932 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) 1933 .addReg(AArch64::SP) 1934 .addReg(AArch64::D1 + 2 * I) 1935 .addReg(AArch64::D0 + 2 * I) 1936 .addReg(AArch64::SP) 1937 .addImm(-2), 1938 *STI); 1939 1940 OutStreamer->emitInstruction( 1941 MCInstBuilder(AArch64::BL) 1942 .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), 1943 *STI); 1944 1945 { 1946 MCInst Adrp; 1947 Adrp.setOpcode(AArch64::ADRP); 1948 Adrp.addOperand(MCOperand::createReg(AArch64::X16)); 1949 MCOperand SymPage; 1950 MCInstLowering.lowerOperand( 1951 MachineOperand::CreateES(LazyPointer->getName().data() + 1, 1952 AArch64II::MO_GOT | AArch64II::MO_PAGE), 1953 SymPage); 1954 Adrp.addOperand(SymPage); 1955 OutStreamer->emitInstruction(Adrp, *STI); 1956 } 1957 1958 { 1959 MCInst Ldr; 1960 Ldr.setOpcode(AArch64::LDRXui); 1961 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1962 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1963 MCOperand SymPageOff; 1964 MCInstLowering.lowerOperand( 1965 MachineOperand::CreateES(LazyPointer->getName().data() + 1, 1966 AArch64II::MO_GOT | AArch64II::MO_PAGEOFF), 1967 SymPageOff); 1968 Ldr.addOperand(SymPageOff); 1969 Ldr.addOperand(MCOperand::createImm(0)); 1970 OutStreamer->emitInstruction(Ldr, *STI); 1971 } 1972 1973 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui) 1974 .addReg(AArch64::X0) 1975 .addReg(AArch64::X16) 1976 .addImm(0), 1977 *STI); 1978 1979 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 1980 .addReg(AArch64::X16) 1981 .addReg(AArch64::X0) 1982 .addImm(0) 1983 .addImm(0), 1984 *STI); 1985 1986 for (int I = 3; I != -1; --I) 1987 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) 1988 .addReg(AArch64::SP) 1989 .addReg(AArch64::D1 + 2 * I) 1990 .addReg(AArch64::D0 + 2 * I) 1991 .addReg(AArch64::SP) 1992 .addImm(2), 1993 *STI); 1994 1995 for (int I = 3; I != -1; --I) 1996 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) 1997 .addReg(AArch64::SP) 1998 .addReg(AArch64::X1 + 2 * I) 1999 .addReg(AArch64::X0 + 2 * I) 2000 .addReg(AArch64::SP) 2001 .addImm(2), 2002 *STI); 2003 2004 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) 2005 .addReg(AArch64::SP) 2006 .addReg(AArch64::FP) 2007 .addReg(AArch64::LR) 2008 .addReg(AArch64::SP) 2009 .addImm(2), 2010 *STI); 2011 2012 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() 2013 ? AArch64::BRAAZ 2014 : AArch64::BR) 2015 .addReg(AArch64::X16), 2016 *STI); 2017 } 2018 2019 // Force static initialization. 2020 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { 2021 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); 2022 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget()); 2023 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target()); 2024 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target()); 2025 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target()); 2026 } 2027