1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a printer that converts from our internal representation 10 // of machine-dependent LLVM code to the AArch64 assembly language. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64.h" 15 #include "AArch64MCInstLower.h" 16 #include "AArch64MachineFunctionInfo.h" 17 #include "AArch64RegisterInfo.h" 18 #include "AArch64Subtarget.h" 19 #include "AArch64TargetObjectFile.h" 20 #include "MCTargetDesc/AArch64AddressingModes.h" 21 #include "MCTargetDesc/AArch64InstPrinter.h" 22 #include "MCTargetDesc/AArch64MCExpr.h" 23 #include "MCTargetDesc/AArch64MCTargetDesc.h" 24 #include "MCTargetDesc/AArch64TargetStreamer.h" 25 #include "TargetInfo/AArch64TargetInfo.h" 26 #include "Utils/AArch64BaseInfo.h" 27 #include "llvm/ADT/SmallString.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/ADT/Twine.h" 31 #include "llvm/BinaryFormat/COFF.h" 32 #include "llvm/BinaryFormat/ELF.h" 33 #include "llvm/BinaryFormat/MachO.h" 34 #include "llvm/CodeGen/AsmPrinter.h" 35 #include "llvm/CodeGen/FaultMaps.h" 36 #include "llvm/CodeGen/MachineBasicBlock.h" 37 #include "llvm/CodeGen/MachineFunction.h" 38 #include "llvm/CodeGen/MachineInstr.h" 39 #include "llvm/CodeGen/MachineJumpTableInfo.h" 40 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 41 #include "llvm/CodeGen/MachineOperand.h" 42 #include "llvm/CodeGen/StackMaps.h" 43 #include "llvm/CodeGen/TargetRegisterInfo.h" 44 #include "llvm/IR/DataLayout.h" 45 #include "llvm/IR/DebugInfoMetadata.h" 46 #include "llvm/MC/MCAsmInfo.h" 47 #include "llvm/MC/MCContext.h" 48 #include "llvm/MC/MCInst.h" 49 #include "llvm/MC/MCInstBuilder.h" 50 #include "llvm/MC/MCSectionELF.h" 51 #include "llvm/MC/MCSectionMachO.h" 52 #include "llvm/MC/MCStreamer.h" 53 #include "llvm/MC/MCSymbol.h" 54 #include "llvm/MC/TargetRegistry.h" 55 #include "llvm/Support/Casting.h" 56 #include "llvm/Support/CommandLine.h" 57 #include "llvm/Support/ErrorHandling.h" 58 #include "llvm/Support/raw_ostream.h" 59 #include "llvm/Target/TargetMachine.h" 60 #include "llvm/TargetParser/Triple.h" 61 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 62 #include <algorithm> 63 #include <cassert> 64 #include <cstdint> 65 #include <map> 66 #include <memory> 67 68 using namespace llvm; 69 70 #define DEBUG_TYPE "asm-printer" 71 72 namespace { 73 74 class AArch64AsmPrinter : public AsmPrinter { 75 AArch64MCInstLower MCInstLowering; 76 FaultMaps FM; 77 const AArch64Subtarget *STI; 78 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false; 79 80 public: 81 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) 82 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this), 83 FM(*this) {} 84 85 StringRef getPassName() const override { return "AArch64 Assembly Printer"; } 86 87 /// Wrapper for MCInstLowering.lowerOperand() for the 88 /// tblgen'erated pseudo lowering. 89 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const { 90 return MCInstLowering.lowerOperand(MO, MCOp); 91 } 92 93 void emitStartOfAsmFile(Module &M) override; 94 void emitJumpTableInfo() override; 95 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *, 96 codeview::JumpTableEntrySize> 97 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, 98 const MCSymbol *BranchLabel) const override; 99 100 void emitFunctionEntryLabel() override; 101 102 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI); 103 104 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI); 105 106 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 107 const MachineInstr &MI); 108 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 109 const MachineInstr &MI); 110 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 111 const MachineInstr &MI); 112 void LowerFAULTING_OP(const MachineInstr &MI); 113 114 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); 115 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); 116 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); 117 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed); 118 119 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple; 120 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; 121 void LowerKCFI_CHECK(const MachineInstr &MI); 122 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); 123 void emitHwasanMemaccessSymbols(Module &M); 124 125 void emitSled(const MachineInstr &MI, SledKind Kind); 126 127 /// tblgen'erated driver function for lowering simple MI->MC 128 /// pseudo instructions. 129 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, 130 const MachineInstr *MI); 131 132 void emitInstruction(const MachineInstr *MI) override; 133 134 void emitFunctionHeaderComment() override; 135 136 void getAnalysisUsage(AnalysisUsage &AU) const override { 137 AsmPrinter::getAnalysisUsage(AU); 138 AU.setPreservesAll(); 139 } 140 141 bool runOnMachineFunction(MachineFunction &MF) override { 142 AArch64FI = MF.getInfo<AArch64FunctionInfo>(); 143 STI = &MF.getSubtarget<AArch64Subtarget>(); 144 145 SetupMachineFunction(MF); 146 147 if (STI->isTargetCOFF()) { 148 bool Local = MF.getFunction().hasLocalLinkage(); 149 COFF::SymbolStorageClass Scl = 150 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL; 151 int Type = 152 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT; 153 154 OutStreamer->beginCOFFSymbolDef(CurrentFnSym); 155 OutStreamer->emitCOFFSymbolStorageClass(Scl); 156 OutStreamer->emitCOFFSymbolType(Type); 157 OutStreamer->endCOFFSymbolDef(); 158 } 159 160 // Emit the rest of the function body. 161 emitFunctionBody(); 162 163 // Emit the XRay table for this function. 164 emitXRayTable(); 165 166 // We didn't modify anything. 167 return false; 168 } 169 170 private: 171 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); 172 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O); 173 bool printAsmRegInClass(const MachineOperand &MO, 174 const TargetRegisterClass *RC, unsigned AltName, 175 raw_ostream &O); 176 177 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, 178 const char *ExtraCode, raw_ostream &O) override; 179 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, 180 const char *ExtraCode, raw_ostream &O) override; 181 182 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS); 183 184 void emitFunctionBodyEnd() override; 185 186 MCSymbol *GetCPISymbol(unsigned CPID) const override; 187 void emitEndOfAsmFile(Module &M) override; 188 189 AArch64FunctionInfo *AArch64FI = nullptr; 190 191 /// Emit the LOHs contained in AArch64FI. 192 void emitLOHs(); 193 194 /// Emit instruction to set float register to zero. 195 void emitFMov0(const MachineInstr &MI); 196 197 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; 198 199 MInstToMCSymbol LOHInstToLabel; 200 201 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { 202 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; 203 } 204 205 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override { 206 assert(STI); 207 return STI; 208 } 209 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, 210 MCSymbol *LazyPointer) override; 211 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, 212 MCSymbol *LazyPointer) override; 213 }; 214 215 } // end anonymous namespace 216 217 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) { 218 const Triple &TT = TM.getTargetTriple(); 219 220 if (TT.isOSBinFormatCOFF()) { 221 // Emit an absolute @feat.00 symbol 222 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00")); 223 OutStreamer->beginCOFFSymbolDef(S); 224 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC); 225 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL); 226 OutStreamer->endCOFFSymbolDef(); 227 int64_t Feat00Value = 0; 228 229 if (M.getModuleFlag("cfguard")) { 230 // Object is CFG-aware. 231 Feat00Value |= COFF::Feat00Flags::GuardCF; 232 } 233 234 if (M.getModuleFlag("ehcontguard")) { 235 // Object also has EHCont. 236 Feat00Value |= COFF::Feat00Flags::GuardEHCont; 237 } 238 239 if (M.getModuleFlag("ms-kernel")) { 240 // Object is compiled with /kernel. 241 Feat00Value |= COFF::Feat00Flags::Kernel; 242 } 243 244 OutStreamer->emitSymbolAttribute(S, MCSA_Global); 245 OutStreamer->emitAssignment( 246 S, MCConstantExpr::create(Feat00Value, MMI->getContext())); 247 } 248 249 if (!TT.isOSBinFormatELF()) 250 return; 251 252 // Assemble feature flags that may require creation of a note section. 253 unsigned Flags = 0; 254 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>( 255 M.getModuleFlag("branch-target-enforcement"))) 256 if (BTE->getZExtValue()) 257 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI; 258 259 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>( 260 M.getModuleFlag("guarded-control-stack"))) 261 if (GCS->getZExtValue()) 262 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS; 263 264 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>( 265 M.getModuleFlag("sign-return-address"))) 266 if (Sign->getZExtValue()) 267 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC; 268 269 if (Flags == 0) 270 return; 271 272 // Emit a .note.gnu.property section with the flags. 273 auto *TS = 274 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 275 TS->emitNoteSection(Flags); 276 } 277 278 void AArch64AsmPrinter::emitFunctionHeaderComment() { 279 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>(); 280 std::optional<std::string> OutlinerString = FI->getOutliningStyle(); 281 if (OutlinerString != std::nullopt) 282 OutStreamer->getCommentOS() << ' ' << OutlinerString; 283 } 284 285 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) 286 { 287 const Function &F = MF->getFunction(); 288 if (F.hasFnAttribute("patchable-function-entry")) { 289 unsigned Num; 290 if (F.getFnAttribute("patchable-function-entry") 291 .getValueAsString() 292 .getAsInteger(10, Num)) 293 return; 294 emitNops(Num); 295 return; 296 } 297 298 emitSled(MI, SledKind::FUNCTION_ENTER); 299 } 300 301 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) { 302 emitSled(MI, SledKind::FUNCTION_EXIT); 303 } 304 305 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) { 306 emitSled(MI, SledKind::TAIL_CALL); 307 } 308 309 void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) { 310 static const int8_t NoopsInSledCount = 7; 311 // We want to emit the following pattern: 312 // 313 // .Lxray_sled_N: 314 // ALIGN 315 // B #32 316 // ; 7 NOP instructions (28 bytes) 317 // .tmpN 318 // 319 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching 320 // over the full 32 bytes (8 instructions) with the following pattern: 321 // 322 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack 323 // LDR W17, #12 ; W17 := function ID 324 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit 325 // BLR X16 ; call the tracing trampoline 326 // ;DATA: 32 bits of function ID 327 // ;DATA: lower 32 bits of the address of the trampoline 328 // ;DATA: higher 32 bits of the address of the trampoline 329 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack 330 // 331 OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo()); 332 auto CurSled = OutContext.createTempSymbol("xray_sled_", true); 333 OutStreamer->emitLabel(CurSled); 334 auto Target = OutContext.createTempSymbol(); 335 336 // Emit "B #32" instruction, which jumps over the next 28 bytes. 337 // The operand has to be the number of 4-byte instructions to jump over, 338 // including the current instruction. 339 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8)); 340 341 for (int8_t I = 0; I < NoopsInSledCount; I++) 342 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 343 344 OutStreamer->emitLabel(Target); 345 recordSled(CurSled, MI, Kind, 2); 346 } 347 348 // Emit the following code for Intrinsic::{xray_customevent,xray_typedevent} 349 // (built-in functions __xray_customevent/__xray_typedevent). 350 // 351 // .Lxray_event_sled_N: 352 // b 1f 353 // save x0 and x1 (and also x2 for TYPED_EVENT_CALL) 354 // set up x0 and x1 (and also x2 for TYPED_EVENT_CALL) 355 // bl __xray_CustomEvent or __xray_TypedEvent 356 // restore x0 and x1 (and also x2 for TYPED_EVENT_CALL) 357 // 1: 358 // 359 // There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL. 360 // 361 // Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT. 362 // After patching, b .+N will become a nop. 363 void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, 364 bool Typed) { 365 auto &O = *OutStreamer; 366 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true); 367 O.emitLabel(CurSled); 368 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs) 369 .addReg(AArch64::X0) 370 .addReg(AArch64::XZR) 371 .addReg(MI.getOperand(0).getReg()) 372 .addImm(0); 373 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs) 374 .addReg(AArch64::X1) 375 .addReg(AArch64::XZR) 376 .addReg(MI.getOperand(1).getReg()) 377 .addImm(0); 378 bool MachO = TM.getTargetTriple().isOSBinFormatMachO(); 379 auto *Sym = MCSymbolRefExpr::create( 380 OutContext.getOrCreateSymbol( 381 Twine(MachO ? "_" : "") + 382 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")), 383 OutContext); 384 if (Typed) { 385 O.AddComment("Begin XRay typed event"); 386 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9)); 387 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre) 388 .addReg(AArch64::SP) 389 .addReg(AArch64::X0) 390 .addReg(AArch64::X1) 391 .addReg(AArch64::SP) 392 .addImm(-4)); 393 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui) 394 .addReg(AArch64::X2) 395 .addReg(AArch64::SP) 396 .addImm(2)); 397 EmitToStreamer(O, MovX0Op0); 398 EmitToStreamer(O, MovX1Op1); 399 EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs) 400 .addReg(AArch64::X2) 401 .addReg(AArch64::XZR) 402 .addReg(MI.getOperand(2).getReg()) 403 .addImm(0)); 404 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym)); 405 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui) 406 .addReg(AArch64::X2) 407 .addReg(AArch64::SP) 408 .addImm(2)); 409 O.AddComment("End XRay typed event"); 410 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost) 411 .addReg(AArch64::SP) 412 .addReg(AArch64::X0) 413 .addReg(AArch64::X1) 414 .addReg(AArch64::SP) 415 .addImm(4)); 416 417 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2); 418 } else { 419 O.AddComment("Begin XRay custom event"); 420 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6)); 421 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre) 422 .addReg(AArch64::SP) 423 .addReg(AArch64::X0) 424 .addReg(AArch64::X1) 425 .addReg(AArch64::SP) 426 .addImm(-2)); 427 EmitToStreamer(O, MovX0Op0); 428 EmitToStreamer(O, MovX1Op1); 429 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym)); 430 O.AddComment("End XRay custom event"); 431 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost) 432 .addReg(AArch64::SP) 433 .addReg(AArch64::X0) 434 .addReg(AArch64::X1) 435 .addReg(AArch64::SP) 436 .addImm(2)); 437 438 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2); 439 } 440 } 441 442 void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { 443 Register AddrReg = MI.getOperand(0).getReg(); 444 assert(std::next(MI.getIterator())->isCall() && 445 "KCFI_CHECK not followed by a call instruction"); 446 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg && 447 "KCFI_CHECK call target doesn't match call operand"); 448 449 // Default to using the intra-procedure-call temporary registers for 450 // comparing the hashes. 451 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17}; 452 if (AddrReg == AArch64::XZR) { 453 // Checking XZR makes no sense. Instead of emitting a load, zero 454 // ScratchRegs[0] and use it for the ESR AddrIndex below. 455 AddrReg = getXRegFromWReg(ScratchRegs[0]); 456 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs) 457 .addReg(AddrReg) 458 .addReg(AArch64::XZR) 459 .addReg(AArch64::XZR) 460 .addImm(0)); 461 } else { 462 // If one of the scratch registers is used for the call target (e.g. 463 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved 464 // temporary register instead (in this case, AArch64::W9) as the check 465 // is immediately followed by the call instruction. 466 for (auto &Reg : ScratchRegs) { 467 if (Reg == getWRegFromXReg(AddrReg)) { 468 Reg = AArch64::W9; 469 break; 470 } 471 } 472 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg && 473 "Invalid scratch registers for KCFI_CHECK"); 474 475 // Adjust the offset for patchable-function-prefix. This assumes that 476 // patchable-function-prefix is the same for all functions. 477 int64_t PrefixNops = 0; 478 (void)MI.getMF() 479 ->getFunction() 480 .getFnAttribute("patchable-function-prefix") 481 .getValueAsString() 482 .getAsInteger(10, PrefixNops); 483 484 // Load the target function type hash. 485 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi) 486 .addReg(ScratchRegs[0]) 487 .addReg(AddrReg) 488 .addImm(-(PrefixNops * 4 + 4))); 489 } 490 491 // Load the expected type hash. 492 const int64_t Type = MI.getOperand(1).getImm(); 493 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) 494 .addReg(ScratchRegs[1]) 495 .addReg(ScratchRegs[1]) 496 .addImm(Type & 0xFFFF) 497 .addImm(0)); 498 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) 499 .addReg(ScratchRegs[1]) 500 .addReg(ScratchRegs[1]) 501 .addImm((Type >> 16) & 0xFFFF) 502 .addImm(16)); 503 504 // Compare the hashes and trap if there's a mismatch. 505 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs) 506 .addReg(AArch64::WZR) 507 .addReg(ScratchRegs[0]) 508 .addReg(ScratchRegs[1]) 509 .addImm(0)); 510 511 MCSymbol *Pass = OutContext.createTempSymbol(); 512 EmitToStreamer(*OutStreamer, 513 MCInstBuilder(AArch64::Bcc) 514 .addImm(AArch64CC::EQ) 515 .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); 516 517 // The base ESR is 0x8000 and the register information is encoded in bits 518 // 0-9 as follows: 519 // - 0-4: n, where the register Xn contains the target address 520 // - 5-9: m, where the register Wm contains the expected type hash 521 // Where n, m are in [0, 30]. 522 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0; 523 unsigned AddrIndex; 524 switch (AddrReg) { 525 default: 526 AddrIndex = AddrReg - AArch64::X0; 527 break; 528 case AArch64::FP: 529 AddrIndex = 29; 530 break; 531 case AArch64::LR: 532 AddrIndex = 30; 533 break; 534 } 535 536 assert(AddrIndex < 31 && TypeIndex < 31); 537 538 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31); 539 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR)); 540 OutStreamer->emitLabel(Pass); 541 } 542 543 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 544 Register Reg = MI.getOperand(0).getReg(); 545 bool IsShort = 546 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES; 547 uint32_t AccessInfo = MI.getOperand(1).getImm(); 548 MCSymbol *&Sym = 549 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)]; 550 if (!Sym) { 551 // FIXME: Make this work on non-ELF. 552 if (!TM.getTargetTriple().isOSBinFormatELF()) 553 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); 554 555 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" + 556 utostr(AccessInfo); 557 if (IsShort) 558 SymName += "_short_v2"; 559 Sym = OutContext.getOrCreateSymbol(SymName); 560 } 561 562 EmitToStreamer(*OutStreamer, 563 MCInstBuilder(AArch64::BL) 564 .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); 565 } 566 567 void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) { 568 if (HwasanMemaccessSymbols.empty()) 569 return; 570 571 const Triple &TT = TM.getTargetTriple(); 572 assert(TT.isOSBinFormatELF()); 573 std::unique_ptr<MCSubtargetInfo> STI( 574 TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); 575 assert(STI && "Unable to create subtarget info"); 576 577 MCSymbol *HwasanTagMismatchV1Sym = 578 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch"); 579 MCSymbol *HwasanTagMismatchV2Sym = 580 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2"); 581 582 const MCSymbolRefExpr *HwasanTagMismatchV1Ref = 583 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext); 584 const MCSymbolRefExpr *HwasanTagMismatchV2Ref = 585 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext); 586 587 for (auto &P : HwasanMemaccessSymbols) { 588 unsigned Reg = std::get<0>(P.first); 589 bool IsShort = std::get<1>(P.first); 590 uint32_t AccessInfo = std::get<2>(P.first); 591 const MCSymbolRefExpr *HwasanTagMismatchRef = 592 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref; 593 MCSymbol *Sym = P.second; 594 595 bool HasMatchAllTag = 596 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1; 597 uint8_t MatchAllTag = 598 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff; 599 unsigned Size = 600 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); 601 bool CompileKernel = 602 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1; 603 604 OutStreamer->switchSection(OutContext.getELFSection( 605 ".text.hot", ELF::SHT_PROGBITS, 606 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(), 607 /*IsComdat=*/true)); 608 609 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); 610 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); 611 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); 612 OutStreamer->emitLabel(Sym); 613 614 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri) 615 .addReg(AArch64::X16) 616 .addReg(Reg) 617 .addImm(4) 618 .addImm(55), 619 *STI); 620 OutStreamer->emitInstruction( 621 MCInstBuilder(AArch64::LDRBBroX) 622 .addReg(AArch64::W16) 623 .addReg(IsShort ? AArch64::X20 : AArch64::X9) 624 .addReg(AArch64::X16) 625 .addImm(0) 626 .addImm(0), 627 *STI); 628 OutStreamer->emitInstruction( 629 MCInstBuilder(AArch64::SUBSXrs) 630 .addReg(AArch64::XZR) 631 .addReg(AArch64::X16) 632 .addReg(Reg) 633 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), 634 *STI); 635 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); 636 OutStreamer->emitInstruction( 637 MCInstBuilder(AArch64::Bcc) 638 .addImm(AArch64CC::NE) 639 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, 640 OutContext)), 641 *STI); 642 MCSymbol *ReturnSym = OutContext.createTempSymbol(); 643 OutStreamer->emitLabel(ReturnSym); 644 OutStreamer->emitInstruction( 645 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); 646 OutStreamer->emitLabel(HandleMismatchOrPartialSym); 647 648 if (HasMatchAllTag) { 649 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri) 650 .addReg(AArch64::X17) 651 .addReg(Reg) 652 .addImm(56) 653 .addImm(63), 654 *STI); 655 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri) 656 .addReg(AArch64::XZR) 657 .addReg(AArch64::X17) 658 .addImm(MatchAllTag) 659 .addImm(0), 660 *STI); 661 OutStreamer->emitInstruction( 662 MCInstBuilder(AArch64::Bcc) 663 .addImm(AArch64CC::EQ) 664 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 665 *STI); 666 } 667 668 if (IsShort) { 669 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri) 670 .addReg(AArch64::WZR) 671 .addReg(AArch64::W16) 672 .addImm(15) 673 .addImm(0), 674 *STI); 675 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); 676 OutStreamer->emitInstruction( 677 MCInstBuilder(AArch64::Bcc) 678 .addImm(AArch64CC::HI) 679 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 680 *STI); 681 682 OutStreamer->emitInstruction( 683 MCInstBuilder(AArch64::ANDXri) 684 .addReg(AArch64::X17) 685 .addReg(Reg) 686 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), 687 *STI); 688 if (Size != 1) 689 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 690 .addReg(AArch64::X17) 691 .addReg(AArch64::X17) 692 .addImm(Size - 1) 693 .addImm(0), 694 *STI); 695 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs) 696 .addReg(AArch64::WZR) 697 .addReg(AArch64::W16) 698 .addReg(AArch64::W17) 699 .addImm(0), 700 *STI); 701 OutStreamer->emitInstruction( 702 MCInstBuilder(AArch64::Bcc) 703 .addImm(AArch64CC::LS) 704 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 705 *STI); 706 707 OutStreamer->emitInstruction( 708 MCInstBuilder(AArch64::ORRXri) 709 .addReg(AArch64::X16) 710 .addReg(Reg) 711 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), 712 *STI); 713 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui) 714 .addReg(AArch64::W16) 715 .addReg(AArch64::X16) 716 .addImm(0), 717 *STI); 718 OutStreamer->emitInstruction( 719 MCInstBuilder(AArch64::SUBSXrs) 720 .addReg(AArch64::XZR) 721 .addReg(AArch64::X16) 722 .addReg(Reg) 723 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), 724 *STI); 725 OutStreamer->emitInstruction( 726 MCInstBuilder(AArch64::Bcc) 727 .addImm(AArch64CC::EQ) 728 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 729 *STI); 730 731 OutStreamer->emitLabel(HandleMismatchSym); 732 } 733 734 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 735 .addReg(AArch64::SP) 736 .addReg(AArch64::X0) 737 .addReg(AArch64::X1) 738 .addReg(AArch64::SP) 739 .addImm(-32), 740 *STI); 741 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) 742 .addReg(AArch64::FP) 743 .addReg(AArch64::LR) 744 .addReg(AArch64::SP) 745 .addImm(29), 746 *STI); 747 748 if (Reg != AArch64::X0) 749 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs) 750 .addReg(AArch64::X0) 751 .addReg(AArch64::XZR) 752 .addReg(Reg) 753 .addImm(0), 754 *STI); 755 OutStreamer->emitInstruction( 756 MCInstBuilder(AArch64::MOVZXi) 757 .addReg(AArch64::X1) 758 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask) 759 .addImm(0), 760 *STI); 761 762 if (CompileKernel) { 763 // The Linux kernel's dynamic loader doesn't support GOT relative 764 // relocations, but it doesn't support late binding either, so just call 765 // the function directly. 766 OutStreamer->emitInstruction( 767 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI); 768 } else { 769 // Intentionally load the GOT entry and branch to it, rather than possibly 770 // late binding the function, which may clobber the registers before we 771 // have a chance to save them. 772 OutStreamer->emitInstruction( 773 MCInstBuilder(AArch64::ADRP) 774 .addReg(AArch64::X16) 775 .addExpr(AArch64MCExpr::create( 776 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE, 777 OutContext)), 778 *STI); 779 OutStreamer->emitInstruction( 780 MCInstBuilder(AArch64::LDRXui) 781 .addReg(AArch64::X16) 782 .addReg(AArch64::X16) 783 .addExpr(AArch64MCExpr::create( 784 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12, 785 OutContext)), 786 *STI); 787 OutStreamer->emitInstruction( 788 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); 789 } 790 } 791 } 792 793 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) { 794 emitHwasanMemaccessSymbols(M); 795 796 const Triple &TT = TM.getTargetTriple(); 797 if (TT.isOSBinFormatMachO()) { 798 // Funny Darwin hack: This flag tells the linker that no global symbols 799 // contain code that falls through to other global symbols (e.g. the obvious 800 // implementation of multiple entry points). If this doesn't occur, the 801 // linker can safely perform dead code stripping. Since LLVM never 802 // generates code that does this, it is always safe to set. 803 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols); 804 } 805 806 // Emit stack and fault map information. 807 FM.serializeToFaultMapSection(); 808 809 } 810 811 void AArch64AsmPrinter::emitLOHs() { 812 SmallVector<MCSymbol *, 3> MCArgs; 813 814 for (const auto &D : AArch64FI->getLOHContainer()) { 815 for (const MachineInstr *MI : D.getArgs()) { 816 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI); 817 assert(LabelIt != LOHInstToLabel.end() && 818 "Label hasn't been inserted for LOH related instruction"); 819 MCArgs.push_back(LabelIt->second); 820 } 821 OutStreamer->emitLOHDirective(D.getKind(), MCArgs); 822 MCArgs.clear(); 823 } 824 } 825 826 void AArch64AsmPrinter::emitFunctionBodyEnd() { 827 if (!AArch64FI->getLOHRelated().empty()) 828 emitLOHs(); 829 } 830 831 /// GetCPISymbol - Return the symbol for the specified constant pool entry. 832 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const { 833 // Darwin uses a linker-private symbol name for constant-pools (to 834 // avoid addends on the relocation?), ELF has no such concept and 835 // uses a normal private symbol. 836 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty()) 837 return OutContext.getOrCreateSymbol( 838 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" + 839 Twine(getFunctionNumber()) + "_" + Twine(CPID)); 840 841 return AsmPrinter::GetCPISymbol(CPID); 842 } 843 844 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, 845 raw_ostream &O) { 846 const MachineOperand &MO = MI->getOperand(OpNum); 847 switch (MO.getType()) { 848 default: 849 llvm_unreachable("<unknown operand type>"); 850 case MachineOperand::MO_Register: { 851 Register Reg = MO.getReg(); 852 assert(Reg.isPhysical()); 853 assert(!MO.getSubReg() && "Subregs should be eliminated!"); 854 O << AArch64InstPrinter::getRegisterName(Reg); 855 break; 856 } 857 case MachineOperand::MO_Immediate: { 858 O << MO.getImm(); 859 break; 860 } 861 case MachineOperand::MO_GlobalAddress: { 862 PrintSymbolOperand(MO, O); 863 break; 864 } 865 case MachineOperand::MO_BlockAddress: { 866 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); 867 Sym->print(O, MAI); 868 break; 869 } 870 } 871 } 872 873 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, 874 raw_ostream &O) { 875 Register Reg = MO.getReg(); 876 switch (Mode) { 877 default: 878 return true; // Unknown mode. 879 case 'w': 880 Reg = getWRegFromXReg(Reg); 881 break; 882 case 'x': 883 Reg = getXRegFromWReg(Reg); 884 break; 885 case 't': 886 Reg = getXRegFromXRegTuple(Reg); 887 break; 888 } 889 890 O << AArch64InstPrinter::getRegisterName(Reg); 891 return false; 892 } 893 894 // Prints the register in MO using class RC using the offset in the 895 // new register class. This should not be used for cross class 896 // printing. 897 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO, 898 const TargetRegisterClass *RC, 899 unsigned AltName, raw_ostream &O) { 900 assert(MO.isReg() && "Should only get here with a register!"); 901 const TargetRegisterInfo *RI = STI->getRegisterInfo(); 902 Register Reg = MO.getReg(); 903 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg)); 904 if (!RI->regsOverlap(RegToPrint, Reg)) 905 return true; 906 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName); 907 return false; 908 } 909 910 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, 911 const char *ExtraCode, raw_ostream &O) { 912 const MachineOperand &MO = MI->getOperand(OpNum); 913 914 // First try the generic code, which knows about modifiers like 'c' and 'n'. 915 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O)) 916 return false; 917 918 // Does this asm operand have a single letter operand modifier? 919 if (ExtraCode && ExtraCode[0]) { 920 if (ExtraCode[1] != 0) 921 return true; // Unknown modifier. 922 923 switch (ExtraCode[0]) { 924 default: 925 return true; // Unknown modifier. 926 case 'w': // Print W register 927 case 'x': // Print X register 928 if (MO.isReg()) 929 return printAsmMRegister(MO, ExtraCode[0], O); 930 if (MO.isImm() && MO.getImm() == 0) { 931 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR; 932 O << AArch64InstPrinter::getRegisterName(Reg); 933 return false; 934 } 935 printOperand(MI, OpNum, O); 936 return false; 937 case 'b': // Print B register. 938 case 'h': // Print H register. 939 case 's': // Print S register. 940 case 'd': // Print D register. 941 case 'q': // Print Q register. 942 case 'z': // Print Z register. 943 if (MO.isReg()) { 944 const TargetRegisterClass *RC; 945 switch (ExtraCode[0]) { 946 case 'b': 947 RC = &AArch64::FPR8RegClass; 948 break; 949 case 'h': 950 RC = &AArch64::FPR16RegClass; 951 break; 952 case 's': 953 RC = &AArch64::FPR32RegClass; 954 break; 955 case 'd': 956 RC = &AArch64::FPR64RegClass; 957 break; 958 case 'q': 959 RC = &AArch64::FPR128RegClass; 960 break; 961 case 'z': 962 RC = &AArch64::ZPRRegClass; 963 break; 964 default: 965 return true; 966 } 967 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O); 968 } 969 printOperand(MI, OpNum, O); 970 return false; 971 } 972 } 973 974 // According to ARM, we should emit x and v registers unless we have a 975 // modifier. 976 if (MO.isReg()) { 977 Register Reg = MO.getReg(); 978 979 // If this is a w or x register, print an x register. 980 if (AArch64::GPR32allRegClass.contains(Reg) || 981 AArch64::GPR64allRegClass.contains(Reg)) 982 return printAsmMRegister(MO, 'x', O); 983 984 // If this is an x register tuple, print an x register. 985 if (AArch64::GPR64x8ClassRegClass.contains(Reg)) 986 return printAsmMRegister(MO, 't', O); 987 988 unsigned AltName = AArch64::NoRegAltName; 989 const TargetRegisterClass *RegClass; 990 if (AArch64::ZPRRegClass.contains(Reg)) { 991 RegClass = &AArch64::ZPRRegClass; 992 } else if (AArch64::PPRRegClass.contains(Reg)) { 993 RegClass = &AArch64::PPRRegClass; 994 } else if (AArch64::PNRRegClass.contains(Reg)) { 995 RegClass = &AArch64::PNRRegClass; 996 } else { 997 RegClass = &AArch64::FPR128RegClass; 998 AltName = AArch64::vreg; 999 } 1000 1001 // If this is a b, h, s, d, or q register, print it as a v register. 1002 return printAsmRegInClass(MO, RegClass, AltName, O); 1003 } 1004 1005 printOperand(MI, OpNum, O); 1006 return false; 1007 } 1008 1009 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, 1010 unsigned OpNum, 1011 const char *ExtraCode, 1012 raw_ostream &O) { 1013 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a') 1014 return true; // Unknown modifier. 1015 1016 const MachineOperand &MO = MI->getOperand(OpNum); 1017 assert(MO.isReg() && "unexpected inline asm memory operand"); 1018 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]"; 1019 return false; 1020 } 1021 1022 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI, 1023 raw_ostream &OS) { 1024 unsigned NOps = MI->getNumOperands(); 1025 assert(NOps == 4); 1026 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: "; 1027 // cast away const; DIetc do not take const operands for some reason. 1028 OS << MI->getDebugVariable()->getName(); 1029 OS << " <- "; 1030 // Frame address. Currently handles register +- offset only. 1031 assert(MI->isIndirectDebugValue()); 1032 OS << '['; 1033 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(), 1034 MI->debug_operands().end()); 1035 I < E; ++I) { 1036 if (I != 0) 1037 OS << ", "; 1038 printOperand(MI, I, OS); 1039 } 1040 OS << ']'; 1041 OS << "+"; 1042 printOperand(MI, NOps - 2, OS); 1043 } 1044 1045 void AArch64AsmPrinter::emitJumpTableInfo() { 1046 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); 1047 if (!MJTI) return; 1048 1049 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1050 if (JT.empty()) return; 1051 1052 const TargetLoweringObjectFile &TLOF = getObjFileLowering(); 1053 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM); 1054 OutStreamer->switchSection(ReadOnlySec); 1055 1056 auto AFI = MF->getInfo<AArch64FunctionInfo>(); 1057 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) { 1058 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs; 1059 1060 // If this jump table was deleted, ignore it. 1061 if (JTBBs.empty()) continue; 1062 1063 unsigned Size = AFI->getJumpTableEntrySize(JTI); 1064 emitAlignment(Align(Size)); 1065 OutStreamer->emitLabel(GetJTISymbol(JTI)); 1066 1067 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI); 1068 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext); 1069 1070 for (auto *JTBB : JTBBs) { 1071 const MCExpr *Value = 1072 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext); 1073 1074 // Each entry is: 1075 // .byte/.hword (LBB - Lbase)>>2 1076 // or plain: 1077 // .word LBB - Lbase 1078 Value = MCBinaryExpr::createSub(Value, Base, OutContext); 1079 if (Size != 4) 1080 Value = MCBinaryExpr::createLShr( 1081 Value, MCConstantExpr::create(2, OutContext), OutContext); 1082 1083 OutStreamer->emitValue(Value, Size); 1084 } 1085 } 1086 } 1087 1088 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *, 1089 codeview::JumpTableEntrySize> 1090 AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI, 1091 const MachineInstr *BranchInstr, 1092 const MCSymbol *BranchLabel) const { 1093 const auto AFI = MF->getInfo<AArch64FunctionInfo>(); 1094 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI); 1095 codeview::JumpTableEntrySize EntrySize; 1096 switch (AFI->getJumpTableEntrySize(JTI)) { 1097 case 1: 1098 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft; 1099 break; 1100 case 2: 1101 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft; 1102 break; 1103 case 4: 1104 EntrySize = codeview::JumpTableEntrySize::Int32; 1105 break; 1106 default: 1107 llvm_unreachable("Unexpected jump table entry size"); 1108 } 1109 return std::make_tuple(Base, 0, BranchLabel, EntrySize); 1110 } 1111 1112 void AArch64AsmPrinter::emitFunctionEntryLabel() { 1113 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall || 1114 MF->getFunction().getCallingConv() == 1115 CallingConv::AArch64_SVE_VectorCall || 1116 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) { 1117 auto *TS = 1118 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 1119 TS->emitDirectiveVariantPCS(CurrentFnSym); 1120 } 1121 1122 return AsmPrinter::emitFunctionEntryLabel(); 1123 } 1124 1125 /// Small jump tables contain an unsigned byte or half, representing the offset 1126 /// from the lowest-addressed possible destination to the desired basic 1127 /// block. Since all instructions are 4-byte aligned, this is further compressed 1128 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to 1129 /// materialize the correct destination we need: 1130 /// 1131 /// adr xDest, .LBB0_0 1132 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh). 1133 /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries) 1134 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer, 1135 const llvm::MachineInstr &MI) { 1136 Register DestReg = MI.getOperand(0).getReg(); 1137 Register ScratchReg = MI.getOperand(1).getReg(); 1138 Register ScratchRegW = 1139 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32); 1140 Register TableReg = MI.getOperand(2).getReg(); 1141 Register EntryReg = MI.getOperand(3).getReg(); 1142 int JTIdx = MI.getOperand(4).getIndex(); 1143 int Size = AArch64FI->getJumpTableEntrySize(JTIdx); 1144 1145 // This has to be first because the compression pass based its reachability 1146 // calculations on the start of the JumpTableDest instruction. 1147 auto Label = 1148 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx); 1149 1150 // If we don't already have a symbol to use as the base, use the ADR 1151 // instruction itself. 1152 if (!Label) { 1153 Label = MF->getContext().createTempSymbol(); 1154 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label); 1155 OutStreamer.emitLabel(Label); 1156 } 1157 1158 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext()); 1159 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR) 1160 .addReg(DestReg) 1161 .addExpr(LabelExpr)); 1162 1163 // Load the number of instruction-steps to offset from the label. 1164 unsigned LdrOpcode; 1165 switch (Size) { 1166 case 1: LdrOpcode = AArch64::LDRBBroX; break; 1167 case 2: LdrOpcode = AArch64::LDRHHroX; break; 1168 case 4: LdrOpcode = AArch64::LDRSWroX; break; 1169 default: 1170 llvm_unreachable("Unknown jump table size"); 1171 } 1172 1173 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode) 1174 .addReg(Size == 4 ? ScratchReg : ScratchRegW) 1175 .addReg(TableReg) 1176 .addReg(EntryReg) 1177 .addImm(0) 1178 .addImm(Size == 1 ? 0 : 1)); 1179 1180 // Add to the already materialized base label address, multiplying by 4 if 1181 // compressed. 1182 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs) 1183 .addReg(DestReg) 1184 .addReg(DestReg) 1185 .addReg(ScratchReg) 1186 .addImm(Size == 4 ? 0 : 2)); 1187 } 1188 1189 void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer, 1190 const llvm::MachineInstr &MI) { 1191 unsigned Opcode = MI.getOpcode(); 1192 assert(STI->hasMOPS()); 1193 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo); 1194 1195 const auto Ops = [Opcode]() -> std::array<unsigned, 3> { 1196 if (Opcode == AArch64::MOPSMemoryCopyPseudo) 1197 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE}; 1198 if (Opcode == AArch64::MOPSMemoryMovePseudo) 1199 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE}; 1200 if (Opcode == AArch64::MOPSMemorySetPseudo) 1201 return {AArch64::SETP, AArch64::SETM, AArch64::SETE}; 1202 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo) 1203 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE}; 1204 llvm_unreachable("Unhandled memory operation pseudo"); 1205 }(); 1206 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo || 1207 Opcode == AArch64::MOPSMemorySetTaggingPseudo; 1208 1209 for (auto Op : Ops) { 1210 int i = 0; 1211 auto MCIB = MCInstBuilder(Op); 1212 // Destination registers 1213 MCIB.addReg(MI.getOperand(i++).getReg()); 1214 MCIB.addReg(MI.getOperand(i++).getReg()); 1215 if (!IsSet) 1216 MCIB.addReg(MI.getOperand(i++).getReg()); 1217 // Input registers 1218 MCIB.addReg(MI.getOperand(i++).getReg()); 1219 MCIB.addReg(MI.getOperand(i++).getReg()); 1220 MCIB.addReg(MI.getOperand(i++).getReg()); 1221 1222 EmitToStreamer(OutStreamer, MCIB); 1223 } 1224 } 1225 1226 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 1227 const MachineInstr &MI) { 1228 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); 1229 1230 auto &Ctx = OutStreamer.getContext(); 1231 MCSymbol *MILabel = Ctx.createTempSymbol(); 1232 OutStreamer.emitLabel(MILabel); 1233 1234 SM.recordStackMap(*MILabel, MI); 1235 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); 1236 1237 // Scan ahead to trim the shadow. 1238 const MachineBasicBlock &MBB = *MI.getParent(); 1239 MachineBasicBlock::const_iterator MII(MI); 1240 ++MII; 1241 while (NumNOPBytes > 0) { 1242 if (MII == MBB.end() || MII->isCall() || 1243 MII->getOpcode() == AArch64::DBG_VALUE || 1244 MII->getOpcode() == TargetOpcode::PATCHPOINT || 1245 MII->getOpcode() == TargetOpcode::STACKMAP) 1246 break; 1247 ++MII; 1248 NumNOPBytes -= 4; 1249 } 1250 1251 // Emit nops. 1252 for (unsigned i = 0; i < NumNOPBytes; i += 4) 1253 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1254 } 1255 1256 // Lower a patchpoint of the form: 1257 // [<def>], <id>, <numBytes>, <target>, <numArgs> 1258 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 1259 const MachineInstr &MI) { 1260 auto &Ctx = OutStreamer.getContext(); 1261 MCSymbol *MILabel = Ctx.createTempSymbol(); 1262 OutStreamer.emitLabel(MILabel); 1263 SM.recordPatchPoint(*MILabel, MI); 1264 1265 PatchPointOpers Opers(&MI); 1266 1267 int64_t CallTarget = Opers.getCallTarget().getImm(); 1268 unsigned EncodedBytes = 0; 1269 if (CallTarget) { 1270 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && 1271 "High 16 bits of call target should be zero."); 1272 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg(); 1273 EncodedBytes = 16; 1274 // Materialize the jump address: 1275 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi) 1276 .addReg(ScratchReg) 1277 .addImm((CallTarget >> 32) & 0xFFFF) 1278 .addImm(32)); 1279 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) 1280 .addReg(ScratchReg) 1281 .addReg(ScratchReg) 1282 .addImm((CallTarget >> 16) & 0xFFFF) 1283 .addImm(16)); 1284 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) 1285 .addReg(ScratchReg) 1286 .addReg(ScratchReg) 1287 .addImm(CallTarget & 0xFFFF) 1288 .addImm(0)); 1289 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg)); 1290 } 1291 // Emit padding. 1292 unsigned NumBytes = Opers.getNumPatchBytes(); 1293 assert(NumBytes >= EncodedBytes && 1294 "Patchpoint can't request size less than the length of a call."); 1295 assert((NumBytes - EncodedBytes) % 4 == 0 && 1296 "Invalid number of NOP bytes requested!"); 1297 for (unsigned i = EncodedBytes; i < NumBytes; i += 4) 1298 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1299 } 1300 1301 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 1302 const MachineInstr &MI) { 1303 StatepointOpers SOpers(&MI); 1304 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 1305 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); 1306 for (unsigned i = 0; i < PatchBytes; i += 4) 1307 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); 1308 } else { 1309 // Lower call target and choose correct opcode 1310 const MachineOperand &CallTarget = SOpers.getCallTarget(); 1311 MCOperand CallTargetMCOp; 1312 unsigned CallOpcode; 1313 switch (CallTarget.getType()) { 1314 case MachineOperand::MO_GlobalAddress: 1315 case MachineOperand::MO_ExternalSymbol: 1316 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp); 1317 CallOpcode = AArch64::BL; 1318 break; 1319 case MachineOperand::MO_Immediate: 1320 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm()); 1321 CallOpcode = AArch64::BL; 1322 break; 1323 case MachineOperand::MO_Register: 1324 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg()); 1325 CallOpcode = AArch64::BLR; 1326 break; 1327 default: 1328 llvm_unreachable("Unsupported operand type in statepoint call target"); 1329 break; 1330 } 1331 1332 EmitToStreamer(OutStreamer, 1333 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp)); 1334 } 1335 1336 auto &Ctx = OutStreamer.getContext(); 1337 MCSymbol *MILabel = Ctx.createTempSymbol(); 1338 OutStreamer.emitLabel(MILabel); 1339 SM.recordStatepoint(*MILabel, MI); 1340 } 1341 1342 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) { 1343 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>, 1344 // <opcode>, <operands> 1345 1346 Register DefRegister = FaultingMI.getOperand(0).getReg(); 1347 FaultMaps::FaultKind FK = 1348 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm()); 1349 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol(); 1350 unsigned Opcode = FaultingMI.getOperand(3).getImm(); 1351 unsigned OperandsBeginIdx = 4; 1352 1353 auto &Ctx = OutStreamer->getContext(); 1354 MCSymbol *FaultingLabel = Ctx.createTempSymbol(); 1355 OutStreamer->emitLabel(FaultingLabel); 1356 1357 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!"); 1358 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel); 1359 1360 MCInst MI; 1361 MI.setOpcode(Opcode); 1362 1363 if (DefRegister != (Register)0) 1364 MI.addOperand(MCOperand::createReg(DefRegister)); 1365 1366 for (const MachineOperand &MO : 1367 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) { 1368 MCOperand Dest; 1369 lowerOperand(MO, Dest); 1370 MI.addOperand(Dest); 1371 } 1372 1373 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName()); 1374 OutStreamer->emitInstruction(MI, getSubtargetInfo()); 1375 } 1376 1377 void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) { 1378 Register DestReg = MI.getOperand(0).getReg(); 1379 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() && 1380 STI->isNeonAvailable()) { 1381 // Convert H/S register to corresponding D register 1382 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31) 1383 DestReg = AArch64::D0 + (DestReg - AArch64::H0); 1384 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31) 1385 DestReg = AArch64::D0 + (DestReg - AArch64::S0); 1386 else 1387 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); 1388 1389 MCInst MOVI; 1390 MOVI.setOpcode(AArch64::MOVID); 1391 MOVI.addOperand(MCOperand::createReg(DestReg)); 1392 MOVI.addOperand(MCOperand::createImm(0)); 1393 EmitToStreamer(*OutStreamer, MOVI); 1394 } else { 1395 MCInst FMov; 1396 switch (MI.getOpcode()) { 1397 default: llvm_unreachable("Unexpected opcode"); 1398 case AArch64::FMOVH0: 1399 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); 1400 if (!STI->hasFullFP16()) 1401 DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); 1402 FMov.addOperand(MCOperand::createReg(DestReg)); 1403 FMov.addOperand(MCOperand::createReg(AArch64::WZR)); 1404 break; 1405 case AArch64::FMOVS0: 1406 FMov.setOpcode(AArch64::FMOVWSr); 1407 FMov.addOperand(MCOperand::createReg(DestReg)); 1408 FMov.addOperand(MCOperand::createReg(AArch64::WZR)); 1409 break; 1410 case AArch64::FMOVD0: 1411 FMov.setOpcode(AArch64::FMOVXDr); 1412 FMov.addOperand(MCOperand::createReg(DestReg)); 1413 FMov.addOperand(MCOperand::createReg(AArch64::XZR)); 1414 break; 1415 } 1416 EmitToStreamer(*OutStreamer, FMov); 1417 } 1418 } 1419 1420 // Simple pseudo-instructions have their lowering (with expansion to real 1421 // instructions) auto-generated. 1422 #include "AArch64GenMCPseudoLowering.inc" 1423 1424 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { 1425 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits()); 1426 1427 // Do any auto-generated pseudo lowerings. 1428 if (emitPseudoExpansionLowering(*OutStreamer, MI)) 1429 return; 1430 1431 if (MI->getOpcode() == AArch64::ADRP) { 1432 for (auto &Opd : MI->operands()) { 1433 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) == 1434 "swift_async_extendedFramePointerFlags") { 1435 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true; 1436 } 1437 } 1438 } 1439 1440 if (AArch64FI->getLOHRelated().count(MI)) { 1441 // Generate a label for LOH related instruction 1442 MCSymbol *LOHLabel = createTempSymbol("loh"); 1443 // Associate the instruction with the label 1444 LOHInstToLabel[MI] = LOHLabel; 1445 OutStreamer->emitLabel(LOHLabel); 1446 } 1447 1448 AArch64TargetStreamer *TS = 1449 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); 1450 // Do any manual lowerings. 1451 switch (MI->getOpcode()) { 1452 default: 1453 break; 1454 case AArch64::HINT: { 1455 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for 1456 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be 1457 // non-empty. If MI is the initial BTI, place the 1458 // __patchable_function_entries label after BTI. 1459 if (CurrentPatchableFunctionEntrySym && 1460 CurrentPatchableFunctionEntrySym == CurrentFnBegin && 1461 MI == &MF->front().front()) { 1462 int64_t Imm = MI->getOperand(0).getImm(); 1463 if ((Imm & 32) && (Imm & 6)) { 1464 MCInst Inst; 1465 MCInstLowering.Lower(MI, Inst); 1466 EmitToStreamer(*OutStreamer, Inst); 1467 CurrentPatchableFunctionEntrySym = createTempSymbol("patch"); 1468 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym); 1469 return; 1470 } 1471 } 1472 break; 1473 } 1474 case AArch64::MOVMCSym: { 1475 Register DestReg = MI->getOperand(0).getReg(); 1476 const MachineOperand &MO_Sym = MI->getOperand(1); 1477 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym); 1478 MCOperand Hi_MCSym, Lo_MCSym; 1479 1480 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S); 1481 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC); 1482 1483 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym); 1484 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym); 1485 1486 MCInst MovZ; 1487 MovZ.setOpcode(AArch64::MOVZXi); 1488 MovZ.addOperand(MCOperand::createReg(DestReg)); 1489 MovZ.addOperand(Hi_MCSym); 1490 MovZ.addOperand(MCOperand::createImm(16)); 1491 EmitToStreamer(*OutStreamer, MovZ); 1492 1493 MCInst MovK; 1494 MovK.setOpcode(AArch64::MOVKXi); 1495 MovK.addOperand(MCOperand::createReg(DestReg)); 1496 MovK.addOperand(MCOperand::createReg(DestReg)); 1497 MovK.addOperand(Lo_MCSym); 1498 MovK.addOperand(MCOperand::createImm(0)); 1499 EmitToStreamer(*OutStreamer, MovK); 1500 return; 1501 } 1502 case AArch64::MOVIv2d_ns: 1503 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0". 1504 // as movi is more efficient across all cores. Newer cores can eliminate 1505 // fmovs early and there is no difference with movi, but this not true for 1506 // all implementations. 1507 // 1508 // The floating-point version doesn't quite work in rare cases on older 1509 // CPUs, so on those targets we lower this instruction to movi.16b instead. 1510 if (STI->hasZeroCycleZeroingFPWorkaround() && 1511 MI->getOperand(1).getImm() == 0) { 1512 MCInst TmpInst; 1513 TmpInst.setOpcode(AArch64::MOVIv16b_ns); 1514 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); 1515 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm())); 1516 EmitToStreamer(*OutStreamer, TmpInst); 1517 return; 1518 } 1519 break; 1520 1521 case AArch64::DBG_VALUE: 1522 case AArch64::DBG_VALUE_LIST: 1523 if (isVerbose() && OutStreamer->hasRawTextSupport()) { 1524 SmallString<128> TmpStr; 1525 raw_svector_ostream OS(TmpStr); 1526 PrintDebugValueComment(MI, OS); 1527 OutStreamer->emitRawText(StringRef(OS.str())); 1528 } 1529 return; 1530 1531 case AArch64::EMITBKEY: { 1532 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); 1533 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && 1534 ExceptionHandlingType != ExceptionHandling::ARM) 1535 return; 1536 1537 if (getFunctionCFISectionType(*MF) == CFISection::None) 1538 return; 1539 1540 OutStreamer->emitCFIBKeyFrame(); 1541 return; 1542 } 1543 1544 case AArch64::EMITMTETAGGED: { 1545 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); 1546 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && 1547 ExceptionHandlingType != ExceptionHandling::ARM) 1548 return; 1549 1550 if (getFunctionCFISectionType(*MF) != CFISection::None) 1551 OutStreamer->emitCFIMTETaggedFrame(); 1552 return; 1553 } 1554 1555 // Tail calls use pseudo instructions so they have the proper code-gen 1556 // attributes (isCall, isReturn, etc.). We lower them to the real 1557 // instruction here. 1558 case AArch64::TCRETURNri: 1559 case AArch64::TCRETURNriBTI: 1560 case AArch64::TCRETURNriALL: { 1561 MCInst TmpInst; 1562 TmpInst.setOpcode(AArch64::BR); 1563 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); 1564 EmitToStreamer(*OutStreamer, TmpInst); 1565 return; 1566 } 1567 case AArch64::TCRETURNdi: { 1568 MCOperand Dest; 1569 MCInstLowering.lowerOperand(MI->getOperand(0), Dest); 1570 MCInst TmpInst; 1571 TmpInst.setOpcode(AArch64::B); 1572 TmpInst.addOperand(Dest); 1573 EmitToStreamer(*OutStreamer, TmpInst); 1574 return; 1575 } 1576 case AArch64::SpeculationBarrierISBDSBEndBB: { 1577 // Print DSB SYS + ISB 1578 MCInst TmpInstDSB; 1579 TmpInstDSB.setOpcode(AArch64::DSB); 1580 TmpInstDSB.addOperand(MCOperand::createImm(0xf)); 1581 EmitToStreamer(*OutStreamer, TmpInstDSB); 1582 MCInst TmpInstISB; 1583 TmpInstISB.setOpcode(AArch64::ISB); 1584 TmpInstISB.addOperand(MCOperand::createImm(0xf)); 1585 EmitToStreamer(*OutStreamer, TmpInstISB); 1586 return; 1587 } 1588 case AArch64::SpeculationBarrierSBEndBB: { 1589 // Print SB 1590 MCInst TmpInstSB; 1591 TmpInstSB.setOpcode(AArch64::SB); 1592 EmitToStreamer(*OutStreamer, TmpInstSB); 1593 return; 1594 } 1595 case AArch64::TLSDESC_CALLSEQ: { 1596 /// lower this to: 1597 /// adrp x0, :tlsdesc:var 1598 /// ldr x1, [x0, #:tlsdesc_lo12:var] 1599 /// add x0, x0, #:tlsdesc_lo12:var 1600 /// .tlsdesccall var 1601 /// blr x1 1602 /// (TPIDR_EL0 offset now in x0) 1603 const MachineOperand &MO_Sym = MI->getOperand(0); 1604 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym); 1605 MCOperand Sym, SymTLSDescLo12, SymTLSDesc; 1606 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF); 1607 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE); 1608 MCInstLowering.lowerOperand(MO_Sym, Sym); 1609 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12); 1610 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc); 1611 1612 MCInst Adrp; 1613 Adrp.setOpcode(AArch64::ADRP); 1614 Adrp.addOperand(MCOperand::createReg(AArch64::X0)); 1615 Adrp.addOperand(SymTLSDesc); 1616 EmitToStreamer(*OutStreamer, Adrp); 1617 1618 MCInst Ldr; 1619 if (STI->isTargetILP32()) { 1620 Ldr.setOpcode(AArch64::LDRWui); 1621 Ldr.addOperand(MCOperand::createReg(AArch64::W1)); 1622 } else { 1623 Ldr.setOpcode(AArch64::LDRXui); 1624 Ldr.addOperand(MCOperand::createReg(AArch64::X1)); 1625 } 1626 Ldr.addOperand(MCOperand::createReg(AArch64::X0)); 1627 Ldr.addOperand(SymTLSDescLo12); 1628 Ldr.addOperand(MCOperand::createImm(0)); 1629 EmitToStreamer(*OutStreamer, Ldr); 1630 1631 MCInst Add; 1632 if (STI->isTargetILP32()) { 1633 Add.setOpcode(AArch64::ADDWri); 1634 Add.addOperand(MCOperand::createReg(AArch64::W0)); 1635 Add.addOperand(MCOperand::createReg(AArch64::W0)); 1636 } else { 1637 Add.setOpcode(AArch64::ADDXri); 1638 Add.addOperand(MCOperand::createReg(AArch64::X0)); 1639 Add.addOperand(MCOperand::createReg(AArch64::X0)); 1640 } 1641 Add.addOperand(SymTLSDescLo12); 1642 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0))); 1643 EmitToStreamer(*OutStreamer, Add); 1644 1645 // Emit a relocation-annotation. This expands to no code, but requests 1646 // the following instruction gets an R_AARCH64_TLSDESC_CALL. 1647 MCInst TLSDescCall; 1648 TLSDescCall.setOpcode(AArch64::TLSDESCCALL); 1649 TLSDescCall.addOperand(Sym); 1650 EmitToStreamer(*OutStreamer, TLSDescCall); 1651 1652 MCInst Blr; 1653 Blr.setOpcode(AArch64::BLR); 1654 Blr.addOperand(MCOperand::createReg(AArch64::X1)); 1655 EmitToStreamer(*OutStreamer, Blr); 1656 1657 return; 1658 } 1659 1660 case AArch64::JumpTableDest32: 1661 case AArch64::JumpTableDest16: 1662 case AArch64::JumpTableDest8: 1663 LowerJumpTableDest(*OutStreamer, *MI); 1664 return; 1665 1666 case AArch64::FMOVH0: 1667 case AArch64::FMOVS0: 1668 case AArch64::FMOVD0: 1669 emitFMov0(*MI); 1670 return; 1671 1672 case AArch64::MOPSMemoryCopyPseudo: 1673 case AArch64::MOPSMemoryMovePseudo: 1674 case AArch64::MOPSMemorySetPseudo: 1675 case AArch64::MOPSMemorySetTaggingPseudo: 1676 LowerMOPS(*OutStreamer, *MI); 1677 return; 1678 1679 case TargetOpcode::STACKMAP: 1680 return LowerSTACKMAP(*OutStreamer, SM, *MI); 1681 1682 case TargetOpcode::PATCHPOINT: 1683 return LowerPATCHPOINT(*OutStreamer, SM, *MI); 1684 1685 case TargetOpcode::STATEPOINT: 1686 return LowerSTATEPOINT(*OutStreamer, SM, *MI); 1687 1688 case TargetOpcode::FAULTING_OP: 1689 return LowerFAULTING_OP(*MI); 1690 1691 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: 1692 LowerPATCHABLE_FUNCTION_ENTER(*MI); 1693 return; 1694 1695 case TargetOpcode::PATCHABLE_FUNCTION_EXIT: 1696 LowerPATCHABLE_FUNCTION_EXIT(*MI); 1697 return; 1698 1699 case TargetOpcode::PATCHABLE_TAIL_CALL: 1700 LowerPATCHABLE_TAIL_CALL(*MI); 1701 return; 1702 case TargetOpcode::PATCHABLE_EVENT_CALL: 1703 return LowerPATCHABLE_EVENT_CALL(*MI, false); 1704 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL: 1705 return LowerPATCHABLE_EVENT_CALL(*MI, true); 1706 1707 case AArch64::KCFI_CHECK: 1708 LowerKCFI_CHECK(*MI); 1709 return; 1710 1711 case AArch64::HWASAN_CHECK_MEMACCESS: 1712 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: 1713 LowerHWASAN_CHECK_MEMACCESS(*MI); 1714 return; 1715 1716 case AArch64::SEH_StackAlloc: 1717 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm()); 1718 return; 1719 1720 case AArch64::SEH_SaveFPLR: 1721 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm()); 1722 return; 1723 1724 case AArch64::SEH_SaveFPLR_X: 1725 assert(MI->getOperand(0).getImm() < 0 && 1726 "Pre increment SEH opcode must have a negative offset"); 1727 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm()); 1728 return; 1729 1730 case AArch64::SEH_SaveReg: 1731 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(), 1732 MI->getOperand(1).getImm()); 1733 return; 1734 1735 case AArch64::SEH_SaveReg_X: 1736 assert(MI->getOperand(1).getImm() < 0 && 1737 "Pre increment SEH opcode must have a negative offset"); 1738 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(), 1739 -MI->getOperand(1).getImm()); 1740 return; 1741 1742 case AArch64::SEH_SaveRegP: 1743 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 && 1744 MI->getOperand(0).getImm() <= 28) { 1745 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 && 1746 "Register paired with LR must be odd"); 1747 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(), 1748 MI->getOperand(2).getImm()); 1749 return; 1750 } 1751 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1752 "Non-consecutive registers not allowed for save_regp"); 1753 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(), 1754 MI->getOperand(2).getImm()); 1755 return; 1756 1757 case AArch64::SEH_SaveRegP_X: 1758 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1759 "Non-consecutive registers not allowed for save_regp_x"); 1760 assert(MI->getOperand(2).getImm() < 0 && 1761 "Pre increment SEH opcode must have a negative offset"); 1762 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(), 1763 -MI->getOperand(2).getImm()); 1764 return; 1765 1766 case AArch64::SEH_SaveFReg: 1767 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(), 1768 MI->getOperand(1).getImm()); 1769 return; 1770 1771 case AArch64::SEH_SaveFReg_X: 1772 assert(MI->getOperand(1).getImm() < 0 && 1773 "Pre increment SEH opcode must have a negative offset"); 1774 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(), 1775 -MI->getOperand(1).getImm()); 1776 return; 1777 1778 case AArch64::SEH_SaveFRegP: 1779 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1780 "Non-consecutive registers not allowed for save_regp"); 1781 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(), 1782 MI->getOperand(2).getImm()); 1783 return; 1784 1785 case AArch64::SEH_SaveFRegP_X: 1786 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && 1787 "Non-consecutive registers not allowed for save_regp_x"); 1788 assert(MI->getOperand(2).getImm() < 0 && 1789 "Pre increment SEH opcode must have a negative offset"); 1790 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(), 1791 -MI->getOperand(2).getImm()); 1792 return; 1793 1794 case AArch64::SEH_SetFP: 1795 TS->emitARM64WinCFISetFP(); 1796 return; 1797 1798 case AArch64::SEH_AddFP: 1799 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm()); 1800 return; 1801 1802 case AArch64::SEH_Nop: 1803 TS->emitARM64WinCFINop(); 1804 return; 1805 1806 case AArch64::SEH_PrologEnd: 1807 TS->emitARM64WinCFIPrologEnd(); 1808 return; 1809 1810 case AArch64::SEH_EpilogStart: 1811 TS->emitARM64WinCFIEpilogStart(); 1812 return; 1813 1814 case AArch64::SEH_EpilogEnd: 1815 TS->emitARM64WinCFIEpilogEnd(); 1816 return; 1817 1818 case AArch64::SEH_PACSignLR: 1819 TS->emitARM64WinCFIPACSignLR(); 1820 return; 1821 } 1822 1823 // Finally, do the automated lowerings for everything else. 1824 MCInst TmpInst; 1825 MCInstLowering.Lower(MI, TmpInst); 1826 EmitToStreamer(*OutStreamer, TmpInst); 1827 } 1828 1829 void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, 1830 MCSymbol *LazyPointer) { 1831 // _ifunc: 1832 // adrp x16, lazy_pointer@GOTPAGE 1833 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] 1834 // ldr x16, [x16] 1835 // br x16 1836 1837 { 1838 MCInst Adrp; 1839 Adrp.setOpcode(AArch64::ADRP); 1840 Adrp.addOperand(MCOperand::createReg(AArch64::X16)); 1841 MCOperand SymPage; 1842 MCInstLowering.lowerOperand( 1843 MachineOperand::CreateMCSymbol(LazyPointer, 1844 AArch64II::MO_GOT | AArch64II::MO_PAGE), 1845 SymPage); 1846 Adrp.addOperand(SymPage); 1847 OutStreamer->emitInstruction(Adrp, *STI); 1848 } 1849 1850 { 1851 MCInst Ldr; 1852 Ldr.setOpcode(AArch64::LDRXui); 1853 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1854 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1855 MCOperand SymPageOff; 1856 MCInstLowering.lowerOperand( 1857 MachineOperand::CreateMCSymbol(LazyPointer, AArch64II::MO_GOT | 1858 AArch64II::MO_PAGEOFF), 1859 SymPageOff); 1860 Ldr.addOperand(SymPageOff); 1861 Ldr.addOperand(MCOperand::createImm(0)); 1862 OutStreamer->emitInstruction(Ldr, *STI); 1863 } 1864 1865 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui) 1866 .addReg(AArch64::X16) 1867 .addReg(AArch64::X16) 1868 .addImm(0), 1869 *STI); 1870 1871 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() 1872 ? AArch64::BRAAZ 1873 : AArch64::BR) 1874 .addReg(AArch64::X16), 1875 *STI); 1876 } 1877 1878 void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M, 1879 const GlobalIFunc &GI, 1880 MCSymbol *LazyPointer) { 1881 // These stub helpers are only ever called once, so here we're optimizing for 1882 // minimum size by using the pre-indexed store variants, which saves a few 1883 // bytes of instructions to bump & restore sp. 1884 1885 // _ifunc.stub_helper: 1886 // stp fp, lr, [sp, #-16]! 1887 // mov fp, sp 1888 // stp x1, x0, [sp, #-16]! 1889 // stp x3, x2, [sp, #-16]! 1890 // stp x5, x4, [sp, #-16]! 1891 // stp x7, x6, [sp, #-16]! 1892 // stp d1, d0, [sp, #-16]! 1893 // stp d3, d2, [sp, #-16]! 1894 // stp d5, d4, [sp, #-16]! 1895 // stp d7, d6, [sp, #-16]! 1896 // bl _resolver 1897 // adrp x16, lazy_pointer@GOTPAGE 1898 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] 1899 // str x0, [x16] 1900 // mov x16, x0 1901 // ldp d7, d6, [sp], #16 1902 // ldp d5, d4, [sp], #16 1903 // ldp d3, d2, [sp], #16 1904 // ldp d1, d0, [sp], #16 1905 // ldp x7, x6, [sp], #16 1906 // ldp x5, x4, [sp], #16 1907 // ldp x3, x2, [sp], #16 1908 // ldp x1, x0, [sp], #16 1909 // ldp fp, lr, [sp], #16 1910 // br x16 1911 1912 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 1913 .addReg(AArch64::SP) 1914 .addReg(AArch64::FP) 1915 .addReg(AArch64::LR) 1916 .addReg(AArch64::SP) 1917 .addImm(-2), 1918 *STI); 1919 1920 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 1921 .addReg(AArch64::FP) 1922 .addReg(AArch64::SP) 1923 .addImm(0) 1924 .addImm(0), 1925 *STI); 1926 1927 for (int I = 0; I != 4; ++I) 1928 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) 1929 .addReg(AArch64::SP) 1930 .addReg(AArch64::X1 + 2 * I) 1931 .addReg(AArch64::X0 + 2 * I) 1932 .addReg(AArch64::SP) 1933 .addImm(-2), 1934 *STI); 1935 1936 for (int I = 0; I != 4; ++I) 1937 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) 1938 .addReg(AArch64::SP) 1939 .addReg(AArch64::D1 + 2 * I) 1940 .addReg(AArch64::D0 + 2 * I) 1941 .addReg(AArch64::SP) 1942 .addImm(-2), 1943 *STI); 1944 1945 OutStreamer->emitInstruction( 1946 MCInstBuilder(AArch64::BL) 1947 .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), 1948 *STI); 1949 1950 { 1951 MCInst Adrp; 1952 Adrp.setOpcode(AArch64::ADRP); 1953 Adrp.addOperand(MCOperand::createReg(AArch64::X16)); 1954 MCOperand SymPage; 1955 MCInstLowering.lowerOperand( 1956 MachineOperand::CreateES(LazyPointer->getName().data() + 1, 1957 AArch64II::MO_GOT | AArch64II::MO_PAGE), 1958 SymPage); 1959 Adrp.addOperand(SymPage); 1960 OutStreamer->emitInstruction(Adrp, *STI); 1961 } 1962 1963 { 1964 MCInst Ldr; 1965 Ldr.setOpcode(AArch64::LDRXui); 1966 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1967 Ldr.addOperand(MCOperand::createReg(AArch64::X16)); 1968 MCOperand SymPageOff; 1969 MCInstLowering.lowerOperand( 1970 MachineOperand::CreateES(LazyPointer->getName().data() + 1, 1971 AArch64II::MO_GOT | AArch64II::MO_PAGEOFF), 1972 SymPageOff); 1973 Ldr.addOperand(SymPageOff); 1974 Ldr.addOperand(MCOperand::createImm(0)); 1975 OutStreamer->emitInstruction(Ldr, *STI); 1976 } 1977 1978 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui) 1979 .addReg(AArch64::X0) 1980 .addReg(AArch64::X16) 1981 .addImm(0), 1982 *STI); 1983 1984 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) 1985 .addReg(AArch64::X16) 1986 .addReg(AArch64::X0) 1987 .addImm(0) 1988 .addImm(0), 1989 *STI); 1990 1991 for (int I = 3; I != -1; --I) 1992 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) 1993 .addReg(AArch64::SP) 1994 .addReg(AArch64::D1 + 2 * I) 1995 .addReg(AArch64::D0 + 2 * I) 1996 .addReg(AArch64::SP) 1997 .addImm(2), 1998 *STI); 1999 2000 for (int I = 3; I != -1; --I) 2001 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) 2002 .addReg(AArch64::SP) 2003 .addReg(AArch64::X1 + 2 * I) 2004 .addReg(AArch64::X0 + 2 * I) 2005 .addReg(AArch64::SP) 2006 .addImm(2), 2007 *STI); 2008 2009 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) 2010 .addReg(AArch64::SP) 2011 .addReg(AArch64::FP) 2012 .addReg(AArch64::LR) 2013 .addReg(AArch64::SP) 2014 .addImm(2), 2015 *STI); 2016 2017 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() 2018 ? AArch64::BRAAZ 2019 : AArch64::BR) 2020 .addReg(AArch64::X16), 2021 *STI); 2022 } 2023 2024 // Force static initialization. 2025 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { 2026 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); 2027 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget()); 2028 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target()); 2029 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target()); 2030 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target()); 2031 } 2032