1 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a printer that converts from our internal representation 10 // of machine-dependent LLVM code to the RISC-V assembly language. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "MCTargetDesc/RISCVBaseInfo.h" 15 #include "MCTargetDesc/RISCVInstPrinter.h" 16 #include "MCTargetDesc/RISCVMCExpr.h" 17 #include "MCTargetDesc/RISCVTargetStreamer.h" 18 #include "RISCV.h" 19 #include "RISCVMachineFunctionInfo.h" 20 #include "RISCVTargetMachine.h" 21 #include "TargetInfo/RISCVTargetInfo.h" 22 #include "llvm/ADT/APInt.h" 23 #include "llvm/ADT/Statistic.h" 24 #include "llvm/BinaryFormat/ELF.h" 25 #include "llvm/CodeGen/AsmPrinter.h" 26 #include "llvm/CodeGen/MachineConstantPool.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/MachineInstr.h" 29 #include "llvm/CodeGen/MachineModuleInfo.h" 30 #include "llvm/MC/MCAsmInfo.h" 31 #include "llvm/MC/MCContext.h" 32 #include "llvm/MC/MCInst.h" 33 #include "llvm/MC/MCInstBuilder.h" 34 #include "llvm/MC/MCObjectFileInfo.h" 35 #include "llvm/MC/MCSectionELF.h" 36 #include "llvm/MC/MCStreamer.h" 37 #include "llvm/MC/MCSymbol.h" 38 #include "llvm/MC/TargetRegistry.h" 39 #include "llvm/Support/RISCVISAInfo.h" 40 #include "llvm/Support/raw_ostream.h" 41 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 42 43 using namespace llvm; 44 45 #define DEBUG_TYPE "asm-printer" 46 47 STATISTIC(RISCVNumInstrsCompressed, 48 "Number of RISC-V Compressed instructions emitted"); 49 50 namespace llvm { 51 extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures]; 52 } // namespace llvm 53 54 namespace { 55 class RISCVAsmPrinter : public AsmPrinter { 56 const RISCVSubtarget *STI; 57 58 public: 59 explicit RISCVAsmPrinter(TargetMachine &TM, 60 std::unique_ptr<MCStreamer> Streamer) 61 : AsmPrinter(TM, std::move(Streamer)) {} 62 63 StringRef getPassName() const override { return "RISC-V Assembly Printer"; } 64 65 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 66 const MachineInstr &MI); 67 68 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 69 const MachineInstr &MI); 70 71 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 72 const MachineInstr &MI); 73 74 bool runOnMachineFunction(MachineFunction &MF) override; 75 76 void emitInstruction(const MachineInstr *MI) override; 77 78 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 79 const char *ExtraCode, raw_ostream &OS) override; 80 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, 81 const char *ExtraCode, raw_ostream &OS) override; 82 83 void EmitToStreamer(MCStreamer &S, const MCInst &Inst); 84 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, 85 const MachineInstr *MI); 86 87 typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple; 88 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; 89 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); 90 void LowerKCFI_CHECK(const MachineInstr &MI); 91 void EmitHwasanMemaccessSymbols(Module &M); 92 93 // Wrapper needed for tblgenned pseudo lowering. 94 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const; 95 96 void emitStartOfAsmFile(Module &M) override; 97 void emitEndOfAsmFile(Module &M) override; 98 99 void emitFunctionEntryLabel() override; 100 bool emitDirectiveOptionArch(); 101 102 private: 103 void emitAttributes(); 104 105 void emitNTLHint(const MachineInstr *MI); 106 107 bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI); 108 }; 109 } 110 111 void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 112 const MachineInstr &MI) { 113 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 114 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); 115 116 auto &Ctx = OutStreamer.getContext(); 117 MCSymbol *MILabel = Ctx.createTempSymbol(); 118 OutStreamer.emitLabel(MILabel); 119 120 SM.recordStackMap(*MILabel, MI); 121 assert(NumNOPBytes % NOPBytes == 0 && 122 "Invalid number of NOP bytes requested!"); 123 124 // Scan ahead to trim the shadow. 125 const MachineBasicBlock &MBB = *MI.getParent(); 126 MachineBasicBlock::const_iterator MII(MI); 127 ++MII; 128 while (NumNOPBytes > 0) { 129 if (MII == MBB.end() || MII->isCall() || 130 MII->getOpcode() == RISCV::DBG_VALUE || 131 MII->getOpcode() == TargetOpcode::PATCHPOINT || 132 MII->getOpcode() == TargetOpcode::STACKMAP) 133 break; 134 ++MII; 135 NumNOPBytes -= 4; 136 } 137 138 // Emit nops. 139 emitNops(NumNOPBytes / NOPBytes); 140 } 141 142 // Lower a patchpoint of the form: 143 // [<def>], <id>, <numBytes>, <target>, <numArgs> 144 void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 145 const MachineInstr &MI) { 146 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 147 148 auto &Ctx = OutStreamer.getContext(); 149 MCSymbol *MILabel = Ctx.createTempSymbol(); 150 OutStreamer.emitLabel(MILabel); 151 SM.recordPatchPoint(*MILabel, MI); 152 153 PatchPointOpers Opers(&MI); 154 155 unsigned EncodedBytes = 0; 156 157 // Emit padding. 158 unsigned NumBytes = Opers.getNumPatchBytes(); 159 assert(NumBytes >= EncodedBytes && 160 "Patchpoint can't request size less than the length of a call."); 161 assert((NumBytes - EncodedBytes) % NOPBytes == 0 && 162 "Invalid number of NOP bytes requested!"); 163 emitNops((NumBytes - EncodedBytes) / NOPBytes); 164 } 165 166 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 167 const MachineInstr &MI) { 168 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 169 170 StatepointOpers SOpers(&MI); 171 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 172 assert(PatchBytes % NOPBytes == 0 && 173 "Invalid number of NOP bytes requested!"); 174 emitNops(PatchBytes / NOPBytes); 175 } 176 177 auto &Ctx = OutStreamer.getContext(); 178 MCSymbol *MILabel = Ctx.createTempSymbol(); 179 OutStreamer.emitLabel(MILabel); 180 SM.recordStatepoint(*MILabel, MI); 181 } 182 183 void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) { 184 MCInst CInst; 185 bool Res = RISCVRVC::compress(CInst, Inst, *STI); 186 if (Res) 187 ++RISCVNumInstrsCompressed; 188 AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst); 189 } 190 191 // Simple pseudo-instructions have their lowering (with expansion to real 192 // instructions) auto-generated. 193 #include "RISCVGenMCPseudoLowering.inc" 194 195 // If the target supports Zihintntl and the instruction has a nontemporal 196 // MachineMemOperand, emit an NTLH hint instruction before it. 197 void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) { 198 if (!STI->hasStdExtZihintntl()) 199 return; 200 201 if (MI->memoperands_empty()) 202 return; 203 204 MachineMemOperand *MMO = *(MI->memoperands_begin()); 205 if (!MMO->isNonTemporal()) 206 return; 207 208 unsigned NontemporalMode = 0; 209 if (MMO->getFlags() & MONontemporalBit0) 210 NontemporalMode += 0b1; 211 if (MMO->getFlags() & MONontemporalBit1) 212 NontemporalMode += 0b10; 213 214 MCInst Hint; 215 if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs()) 216 Hint.setOpcode(RISCV::C_ADD_HINT); 217 else 218 Hint.setOpcode(RISCV::ADD); 219 220 Hint.addOperand(MCOperand::createReg(RISCV::X0)); 221 Hint.addOperand(MCOperand::createReg(RISCV::X0)); 222 Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode)); 223 224 EmitToStreamer(*OutStreamer, Hint); 225 } 226 227 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) { 228 RISCV_MC::verifyInstructionPredicates(MI->getOpcode(), 229 getSubtargetInfo().getFeatureBits()); 230 231 emitNTLHint(MI); 232 233 // Do any auto-generated pseudo lowerings. 234 if (emitPseudoExpansionLowering(*OutStreamer, MI)) 235 return; 236 237 238 switch (MI->getOpcode()) { 239 case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: 240 LowerHWASAN_CHECK_MEMACCESS(*MI); 241 return; 242 case RISCV::KCFI_CHECK: 243 LowerKCFI_CHECK(*MI); 244 return; 245 case RISCV::PseudoRVVInitUndefM1: 246 case RISCV::PseudoRVVInitUndefM2: 247 case RISCV::PseudoRVVInitUndefM4: 248 case RISCV::PseudoRVVInitUndefM8: 249 return; 250 case TargetOpcode::STACKMAP: 251 return LowerSTACKMAP(*OutStreamer, SM, *MI); 252 case TargetOpcode::PATCHPOINT: 253 return LowerPATCHPOINT(*OutStreamer, SM, *MI); 254 case TargetOpcode::STATEPOINT: 255 return LowerSTATEPOINT(*OutStreamer, SM, *MI); 256 } 257 258 MCInst OutInst; 259 if (!lowerToMCInst(MI, OutInst)) 260 EmitToStreamer(*OutStreamer, OutInst); 261 } 262 263 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 264 const char *ExtraCode, raw_ostream &OS) { 265 // First try the generic code, which knows about modifiers like 'c' and 'n'. 266 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS)) 267 return false; 268 269 const MachineOperand &MO = MI->getOperand(OpNo); 270 if (ExtraCode && ExtraCode[0]) { 271 if (ExtraCode[1] != 0) 272 return true; // Unknown modifier. 273 274 switch (ExtraCode[0]) { 275 default: 276 return true; // Unknown modifier. 277 case 'z': // Print zero register if zero, regular printing otherwise. 278 if (MO.isImm() && MO.getImm() == 0) { 279 OS << RISCVInstPrinter::getRegisterName(RISCV::X0); 280 return false; 281 } 282 break; 283 case 'i': // Literal 'i' if operand is not a register. 284 if (!MO.isReg()) 285 OS << 'i'; 286 return false; 287 } 288 } 289 290 switch (MO.getType()) { 291 case MachineOperand::MO_Immediate: 292 OS << MO.getImm(); 293 return false; 294 case MachineOperand::MO_Register: 295 OS << RISCVInstPrinter::getRegisterName(MO.getReg()); 296 return false; 297 case MachineOperand::MO_GlobalAddress: 298 PrintSymbolOperand(MO, OS); 299 return false; 300 case MachineOperand::MO_BlockAddress: { 301 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); 302 Sym->print(OS, MAI); 303 return false; 304 } 305 default: 306 break; 307 } 308 309 return true; 310 } 311 312 bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, 313 unsigned OpNo, 314 const char *ExtraCode, 315 raw_ostream &OS) { 316 if (ExtraCode) 317 return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS); 318 319 const MachineOperand &AddrReg = MI->getOperand(OpNo); 320 assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand"); 321 const MachineOperand &Offset = MI->getOperand(OpNo + 1); 322 // All memory operands should have a register and an immediate operand (see 323 // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand). 324 if (!AddrReg.isReg()) 325 return true; 326 if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress()) 327 return true; 328 329 MCOperand MCO; 330 if (!lowerOperand(Offset, MCO)) 331 return true; 332 333 if (Offset.isImm()) 334 OS << MCO.getImm(); 335 else if (Offset.isGlobal() || Offset.isBlockAddress()) 336 OS << *MCO.getExpr(); 337 OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")"; 338 return false; 339 } 340 341 bool RISCVAsmPrinter::emitDirectiveOptionArch() { 342 RISCVTargetStreamer &RTS = 343 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 344 SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs; 345 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo(); 346 for (const auto &Feature : RISCVFeatureKV) { 347 if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value)) 348 continue; 349 350 if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key)) 351 continue; 352 353 auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus 354 : RISCVOptionArchArgType::Minus; 355 NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key); 356 } 357 if (!NeedEmitStdOptionArgs.empty()) { 358 RTS.emitDirectiveOptionPush(); 359 RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs); 360 return true; 361 } 362 363 return false; 364 } 365 366 bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) { 367 STI = &MF.getSubtarget<RISCVSubtarget>(); 368 RISCVTargetStreamer &RTS = 369 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 370 371 bool EmittedOptionArch = emitDirectiveOptionArch(); 372 373 SetupMachineFunction(MF); 374 emitFunctionBody(); 375 376 if (EmittedOptionArch) 377 RTS.emitDirectiveOptionPop(); 378 return false; 379 } 380 381 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) { 382 RISCVTargetStreamer &RTS = 383 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 384 if (const MDString *ModuleTargetABI = 385 dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi"))) 386 RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString())); 387 if (TM.getTargetTriple().isOSBinFormatELF()) 388 emitAttributes(); 389 } 390 391 void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) { 392 RISCVTargetStreamer &RTS = 393 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 394 395 if (TM.getTargetTriple().isOSBinFormatELF()) 396 RTS.finishAttributeSection(); 397 EmitHwasanMemaccessSymbols(M); 398 } 399 400 void RISCVAsmPrinter::emitAttributes() { 401 RISCVTargetStreamer &RTS = 402 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 403 // Use MCSubtargetInfo from TargetMachine. Individual functions may have 404 // attributes that differ from other functions in the module and we have no 405 // way to know which function is correct. 406 RTS.emitTargetAttributes(*TM.getMCSubtargetInfo(), /*EmitStackAlign*/ true); 407 } 408 409 void RISCVAsmPrinter::emitFunctionEntryLabel() { 410 const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>(); 411 if (RMFI->isVectorCall()) { 412 auto &RTS = 413 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 414 RTS.emitDirectiveVariantCC(*CurrentFnSym); 415 } 416 return AsmPrinter::emitFunctionEntryLabel(); 417 } 418 419 // Force static initialization. 420 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() { 421 RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target()); 422 RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target()); 423 } 424 425 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 426 Register Reg = MI.getOperand(0).getReg(); 427 uint32_t AccessInfo = MI.getOperand(1).getImm(); 428 MCSymbol *&Sym = 429 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)]; 430 if (!Sym) { 431 // FIXME: Make this work on non-ELF. 432 if (!TM.getTargetTriple().isOSBinFormatELF()) 433 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); 434 435 std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" + 436 utostr(AccessInfo) + "_short"; 437 Sym = OutContext.getOrCreateSymbol(SymName); 438 } 439 auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext); 440 auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext); 441 442 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr)); 443 } 444 445 void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { 446 Register AddrReg = MI.getOperand(0).getReg(); 447 assert(std::next(MI.getIterator())->isCall() && 448 "KCFI_CHECK not followed by a call instruction"); 449 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg && 450 "KCFI_CHECK call target doesn't match call operand"); 451 452 // Temporary registers for comparing the hashes. If a register is used 453 // for the call target, or reserved by the user, we can clobber another 454 // temporary register as the check is immediately followed by the 455 // call. The check defaults to X6/X7, but can fall back to X28-X31 if 456 // needed. 457 unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7}; 458 unsigned NextReg = RISCV::X28; 459 auto isRegAvailable = [&](unsigned Reg) { 460 return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg); 461 }; 462 for (auto &Reg : ScratchRegs) { 463 if (isRegAvailable(Reg)) 464 continue; 465 while (!isRegAvailable(NextReg)) 466 ++NextReg; 467 Reg = NextReg++; 468 if (Reg > RISCV::X31) 469 report_fatal_error("Unable to find scratch registers for KCFI_CHECK"); 470 } 471 472 if (AddrReg == RISCV::X0) { 473 // Checking X0 makes no sense. Instead of emitting a load, zero 474 // ScratchRegs[0]. 475 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI) 476 .addReg(ScratchRegs[0]) 477 .addReg(RISCV::X0) 478 .addImm(0)); 479 } else { 480 // Adjust the offset for patchable-function-prefix. This assumes that 481 // patchable-function-prefix is the same for all functions. 482 int NopSize = STI->hasStdExtCOrZca() ? 2 : 4; 483 int64_t PrefixNops = 0; 484 (void)MI.getMF() 485 ->getFunction() 486 .getFnAttribute("patchable-function-prefix") 487 .getValueAsString() 488 .getAsInteger(10, PrefixNops); 489 490 // Load the target function type hash. 491 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW) 492 .addReg(ScratchRegs[0]) 493 .addReg(AddrReg) 494 .addImm(-(PrefixNops * NopSize + 4))); 495 } 496 497 // Load the expected 32-bit type hash. 498 const int64_t Type = MI.getOperand(1).getImm(); 499 const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF; 500 const int64_t Lo12 = SignExtend64<12>(Type); 501 if (Hi20) { 502 EmitToStreamer( 503 *OutStreamer, 504 MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20)); 505 } 506 if (Lo12 || Hi20 == 0) { 507 EmitToStreamer(*OutStreamer, 508 MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20) 509 ? RISCV::ADDIW 510 : RISCV::ADDI) 511 .addReg(ScratchRegs[1]) 512 .addReg(ScratchRegs[1]) 513 .addImm(Lo12)); 514 } 515 516 // Compare the hashes and trap if there's a mismatch. 517 MCSymbol *Pass = OutContext.createTempSymbol(); 518 EmitToStreamer(*OutStreamer, 519 MCInstBuilder(RISCV::BEQ) 520 .addReg(ScratchRegs[0]) 521 .addReg(ScratchRegs[1]) 522 .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); 523 524 MCSymbol *Trap = OutContext.createTempSymbol(); 525 OutStreamer->emitLabel(Trap); 526 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK)); 527 emitKCFITrapEntry(*MI.getMF(), Trap); 528 OutStreamer->emitLabel(Pass); 529 } 530 531 void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) { 532 if (HwasanMemaccessSymbols.empty()) 533 return; 534 535 assert(TM.getTargetTriple().isOSBinFormatELF()); 536 // Use MCSubtargetInfo from TargetMachine. Individual functions may have 537 // attributes that differ from other functions in the module and we have no 538 // way to know which function is correct. 539 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo(); 540 541 MCSymbol *HwasanTagMismatchV2Sym = 542 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2"); 543 // Annotate symbol as one having incompatible calling convention, so 544 // run-time linkers can instead eagerly bind this function. 545 auto &RTS = 546 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 547 RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym); 548 549 const MCSymbolRefExpr *HwasanTagMismatchV2Ref = 550 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext); 551 auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref, 552 RISCVMCExpr::VK_RISCV_CALL, OutContext); 553 554 for (auto &P : HwasanMemaccessSymbols) { 555 unsigned Reg = std::get<0>(P.first); 556 uint32_t AccessInfo = std::get<1>(P.first); 557 MCSymbol *Sym = P.second; 558 559 unsigned Size = 560 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); 561 OutStreamer->switchSection(OutContext.getELFSection( 562 ".text.hot", ELF::SHT_PROGBITS, 563 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(), 564 /*IsComdat=*/true)); 565 566 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); 567 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); 568 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); 569 OutStreamer->emitLabel(Sym); 570 571 // Extract shadow offset from ptr 572 OutStreamer->emitInstruction( 573 MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8), 574 MCSTI); 575 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI) 576 .addReg(RISCV::X6) 577 .addReg(RISCV::X6) 578 .addImm(12), 579 MCSTI); 580 // load shadow tag in X6, X5 contains shadow base 581 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD) 582 .addReg(RISCV::X6) 583 .addReg(RISCV::X5) 584 .addReg(RISCV::X6), 585 MCSTI); 586 OutStreamer->emitInstruction( 587 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0), 588 MCSTI); 589 // Extract tag from X5 and compare it with loaded tag from shadow 590 OutStreamer->emitInstruction( 591 MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56), 592 MCSTI); 593 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); 594 // X7 contains tag from memory, while X6 contains tag from the pointer 595 OutStreamer->emitInstruction( 596 MCInstBuilder(RISCV::BNE) 597 .addReg(RISCV::X7) 598 .addReg(RISCV::X6) 599 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, 600 OutContext)), 601 MCSTI); 602 MCSymbol *ReturnSym = OutContext.createTempSymbol(); 603 OutStreamer->emitLabel(ReturnSym); 604 OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR) 605 .addReg(RISCV::X0) 606 .addReg(RISCV::X1) 607 .addImm(0), 608 MCSTI); 609 OutStreamer->emitLabel(HandleMismatchOrPartialSym); 610 611 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 612 .addReg(RISCV::X28) 613 .addReg(RISCV::X0) 614 .addImm(16), 615 MCSTI); 616 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); 617 OutStreamer->emitInstruction( 618 MCInstBuilder(RISCV::BGEU) 619 .addReg(RISCV::X6) 620 .addReg(RISCV::X28) 621 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 622 MCSTI); 623 624 OutStreamer->emitInstruction( 625 MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF), 626 MCSTI); 627 628 if (Size != 1) 629 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 630 .addReg(RISCV::X28) 631 .addReg(RISCV::X28) 632 .addImm(Size - 1), 633 MCSTI); 634 OutStreamer->emitInstruction( 635 MCInstBuilder(RISCV::BGE) 636 .addReg(RISCV::X28) 637 .addReg(RISCV::X6) 638 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 639 MCSTI); 640 641 OutStreamer->emitInstruction( 642 MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF), 643 MCSTI); 644 OutStreamer->emitInstruction( 645 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0), 646 MCSTI); 647 OutStreamer->emitInstruction( 648 MCInstBuilder(RISCV::BEQ) 649 .addReg(RISCV::X6) 650 .addReg(RISCV::X7) 651 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 652 MCSTI); 653 654 OutStreamer->emitLabel(HandleMismatchSym); 655 656 // | Previous stack frames... | 657 // +=================================+ <-- [SP + 256] 658 // | ... | 659 // | | 660 // | Stack frame space for x12 - x31.| 661 // | | 662 // | ... | 663 // +---------------------------------+ <-- [SP + 96] 664 // | Saved x11(arg1), as | 665 // | __hwasan_check_* clobbers it. | 666 // +---------------------------------+ <-- [SP + 88] 667 // | Saved x10(arg0), as | 668 // | __hwasan_check_* clobbers it. | 669 // +---------------------------------+ <-- [SP + 80] 670 // | | 671 // | Stack frame space for x9. | 672 // +---------------------------------+ <-- [SP + 72] 673 // | | 674 // | Saved x8(fp), as | 675 // | __hwasan_check_* clobbers it. | 676 // +---------------------------------+ <-- [SP + 64] 677 // | ... | 678 // | | 679 // | Stack frame space for x2 - x7. | 680 // | | 681 // | ... | 682 // +---------------------------------+ <-- [SP + 16] 683 // | Return address (x1) for caller | 684 // | of __hwasan_check_*. | 685 // +---------------------------------+ <-- [SP + 8] 686 // | Reserved place for x0, possibly | 687 // | junk, since we don't save it. | 688 // +---------------------------------+ <-- [x2 / SP] 689 690 // Adjust sp 691 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 692 .addReg(RISCV::X2) 693 .addReg(RISCV::X2) 694 .addImm(-256), 695 MCSTI); 696 697 // store x10(arg0) by new sp 698 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD) 699 .addReg(RISCV::X10) 700 .addReg(RISCV::X2) 701 .addImm(8 * 10), 702 MCSTI); 703 // store x11(arg1) by new sp 704 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD) 705 .addReg(RISCV::X11) 706 .addReg(RISCV::X2) 707 .addImm(8 * 11), 708 MCSTI); 709 710 // store x8(fp) by new sp 711 OutStreamer->emitInstruction( 712 MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 * 713 8), 714 MCSTI); 715 // store x1(ra) by new sp 716 OutStreamer->emitInstruction( 717 MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 * 718 8), 719 MCSTI); 720 if (Reg != RISCV::X10) 721 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 722 .addReg(RISCV::X10) 723 .addReg(Reg) 724 .addImm(0), 725 MCSTI); 726 OutStreamer->emitInstruction( 727 MCInstBuilder(RISCV::ADDI) 728 .addReg(RISCV::X11) 729 .addReg(RISCV::X0) 730 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask), 731 MCSTI); 732 733 OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr), 734 MCSTI); 735 } 736 } 737 738 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym, 739 const AsmPrinter &AP) { 740 MCContext &Ctx = AP.OutContext; 741 RISCVMCExpr::VariantKind Kind; 742 743 switch (MO.getTargetFlags()) { 744 default: 745 llvm_unreachable("Unknown target flag on GV operand"); 746 case RISCVII::MO_None: 747 Kind = RISCVMCExpr::VK_RISCV_None; 748 break; 749 case RISCVII::MO_CALL: 750 Kind = RISCVMCExpr::VK_RISCV_CALL_PLT; 751 break; 752 case RISCVII::MO_LO: 753 Kind = RISCVMCExpr::VK_RISCV_LO; 754 break; 755 case RISCVII::MO_HI: 756 Kind = RISCVMCExpr::VK_RISCV_HI; 757 break; 758 case RISCVII::MO_PCREL_LO: 759 Kind = RISCVMCExpr::VK_RISCV_PCREL_LO; 760 break; 761 case RISCVII::MO_PCREL_HI: 762 Kind = RISCVMCExpr::VK_RISCV_PCREL_HI; 763 break; 764 case RISCVII::MO_GOT_HI: 765 Kind = RISCVMCExpr::VK_RISCV_GOT_HI; 766 break; 767 case RISCVII::MO_TPREL_LO: 768 Kind = RISCVMCExpr::VK_RISCV_TPREL_LO; 769 break; 770 case RISCVII::MO_TPREL_HI: 771 Kind = RISCVMCExpr::VK_RISCV_TPREL_HI; 772 break; 773 case RISCVII::MO_TPREL_ADD: 774 Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD; 775 break; 776 case RISCVII::MO_TLS_GOT_HI: 777 Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI; 778 break; 779 case RISCVII::MO_TLS_GD_HI: 780 Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI; 781 break; 782 } 783 784 const MCExpr *ME = 785 MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx); 786 787 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset()) 788 ME = MCBinaryExpr::createAdd( 789 ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx); 790 791 if (Kind != RISCVMCExpr::VK_RISCV_None) 792 ME = RISCVMCExpr::create(ME, Kind, Ctx); 793 return MCOperand::createExpr(ME); 794 } 795 796 bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO, 797 MCOperand &MCOp) const { 798 switch (MO.getType()) { 799 default: 800 report_fatal_error("lowerOperand: unknown operand type"); 801 case MachineOperand::MO_Register: 802 // Ignore all implicit register operands. 803 if (MO.isImplicit()) 804 return false; 805 MCOp = MCOperand::createReg(MO.getReg()); 806 break; 807 case MachineOperand::MO_RegisterMask: 808 // Regmasks are like implicit defs. 809 return false; 810 case MachineOperand::MO_Immediate: 811 MCOp = MCOperand::createImm(MO.getImm()); 812 break; 813 case MachineOperand::MO_MachineBasicBlock: 814 MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this); 815 break; 816 case MachineOperand::MO_GlobalAddress: 817 MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this); 818 break; 819 case MachineOperand::MO_BlockAddress: 820 MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()), 821 *this); 822 break; 823 case MachineOperand::MO_ExternalSymbol: 824 MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()), 825 *this); 826 break; 827 case MachineOperand::MO_ConstantPoolIndex: 828 MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this); 829 break; 830 case MachineOperand::MO_JumpTableIndex: 831 MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this); 832 break; 833 case MachineOperand::MO_MCSymbol: 834 MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this); 835 break; 836 } 837 return true; 838 } 839 840 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI, 841 MCInst &OutMI) { 842 const RISCVVPseudosTable::PseudoInfo *RVV = 843 RISCVVPseudosTable::getPseudoInfo(MI->getOpcode()); 844 if (!RVV) 845 return false; 846 847 OutMI.setOpcode(RVV->BaseInstr); 848 849 const MachineBasicBlock *MBB = MI->getParent(); 850 assert(MBB && "MI expected to be in a basic block"); 851 const MachineFunction *MF = MBB->getParent(); 852 assert(MF && "MBB expected to be in a machine function"); 853 854 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>(); 855 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 856 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 857 assert(TRI && "TargetRegisterInfo expected"); 858 859 const MCInstrDesc &MCID = MI->getDesc(); 860 uint64_t TSFlags = MCID.TSFlags; 861 unsigned NumOps = MI->getNumExplicitOperands(); 862 863 // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if 864 // present. 865 if (RISCVII::hasVecPolicyOp(TSFlags)) 866 --NumOps; 867 if (RISCVII::hasSEWOp(TSFlags)) 868 --NumOps; 869 if (RISCVII::hasVLOp(TSFlags)) 870 --NumOps; 871 if (RISCVII::hasRoundModeOp(TSFlags)) 872 --NumOps; 873 874 bool hasVLOutput = RISCV::isFaultFirstLoad(*MI); 875 for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) { 876 const MachineOperand &MO = MI->getOperand(OpNo); 877 // Skip vl ouput. It should be the second output. 878 if (hasVLOutput && OpNo == 1) 879 continue; 880 881 // Skip merge op. It should be the first operand after the defs. 882 if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) { 883 assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 && 884 "Expected tied to first def."); 885 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode()); 886 // Skip if the next operand in OutMI is not supposed to be tied. Unless it 887 // is a _TIED instruction. 888 if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) < 889 0 && 890 !RISCVII::isTiedPseudo(TSFlags)) 891 continue; 892 } 893 894 MCOperand MCOp; 895 switch (MO.getType()) { 896 default: 897 llvm_unreachable("Unknown operand type"); 898 case MachineOperand::MO_Register: { 899 Register Reg = MO.getReg(); 900 901 if (RISCV::VRM2RegClass.contains(Reg) || 902 RISCV::VRM4RegClass.contains(Reg) || 903 RISCV::VRM8RegClass.contains(Reg)) { 904 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0); 905 assert(Reg && "Subregister does not exist"); 906 } else if (RISCV::FPR16RegClass.contains(Reg)) { 907 Reg = 908 TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass); 909 assert(Reg && "Subregister does not exist"); 910 } else if (RISCV::FPR64RegClass.contains(Reg)) { 911 Reg = TRI->getSubReg(Reg, RISCV::sub_32); 912 assert(Reg && "Superregister does not exist"); 913 } else if (RISCV::VRN2M1RegClass.contains(Reg) || 914 RISCV::VRN2M2RegClass.contains(Reg) || 915 RISCV::VRN2M4RegClass.contains(Reg) || 916 RISCV::VRN3M1RegClass.contains(Reg) || 917 RISCV::VRN3M2RegClass.contains(Reg) || 918 RISCV::VRN4M1RegClass.contains(Reg) || 919 RISCV::VRN4M2RegClass.contains(Reg) || 920 RISCV::VRN5M1RegClass.contains(Reg) || 921 RISCV::VRN6M1RegClass.contains(Reg) || 922 RISCV::VRN7M1RegClass.contains(Reg) || 923 RISCV::VRN8M1RegClass.contains(Reg)) { 924 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0); 925 assert(Reg && "Subregister does not exist"); 926 } 927 928 MCOp = MCOperand::createReg(Reg); 929 break; 930 } 931 case MachineOperand::MO_Immediate: 932 MCOp = MCOperand::createImm(MO.getImm()); 933 break; 934 } 935 OutMI.addOperand(MCOp); 936 } 937 938 // Unmasked pseudo instructions need to append dummy mask operand to 939 // V instructions. All V instructions are modeled as the masked version. 940 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode()); 941 if (OutMI.getNumOperands() < OutMCID.getNumOperands()) { 942 assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass == 943 RISCV::VMV0RegClassID && 944 "Expected only mask operand to be missing"); 945 OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister)); 946 } 947 948 assert(OutMI.getNumOperands() == OutMCID.getNumOperands()); 949 return true; 950 } 951 952 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) { 953 if (lowerRISCVVMachineInstrToMCInst(MI, OutMI)) 954 return false; 955 956 OutMI.setOpcode(MI->getOpcode()); 957 958 for (const MachineOperand &MO : MI->operands()) { 959 MCOperand MCOp; 960 if (lowerOperand(MO, MCOp)) 961 OutMI.addOperand(MCOp); 962 } 963 964 switch (OutMI.getOpcode()) { 965 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: { 966 const Function &F = MI->getParent()->getParent()->getFunction(); 967 if (F.hasFnAttribute("patchable-function-entry")) { 968 unsigned Num; 969 if (F.getFnAttribute("patchable-function-entry") 970 .getValueAsString() 971 .getAsInteger(10, Num)) 972 return false; 973 emitNops(Num); 974 return true; 975 } 976 break; 977 } 978 } 979 return false; 980 } 981