1 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a printer that converts from our internal representation 10 // of machine-dependent LLVM code to the RISC-V assembly language. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "MCTargetDesc/RISCVBaseInfo.h" 15 #include "MCTargetDesc/RISCVInstPrinter.h" 16 #include "MCTargetDesc/RISCVMCExpr.h" 17 #include "MCTargetDesc/RISCVTargetStreamer.h" 18 #include "RISCV.h" 19 #include "RISCVMachineFunctionInfo.h" 20 #include "RISCVTargetMachine.h" 21 #include "TargetInfo/RISCVTargetInfo.h" 22 #include "llvm/ADT/APInt.h" 23 #include "llvm/ADT/Statistic.h" 24 #include "llvm/BinaryFormat/ELF.h" 25 #include "llvm/CodeGen/AsmPrinter.h" 26 #include "llvm/CodeGen/MachineConstantPool.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/MachineInstr.h" 29 #include "llvm/CodeGen/MachineModuleInfo.h" 30 #include "llvm/MC/MCAsmInfo.h" 31 #include "llvm/MC/MCContext.h" 32 #include "llvm/MC/MCInst.h" 33 #include "llvm/MC/MCInstBuilder.h" 34 #include "llvm/MC/MCObjectFileInfo.h" 35 #include "llvm/MC/MCSectionELF.h" 36 #include "llvm/MC/MCStreamer.h" 37 #include "llvm/MC/MCSymbol.h" 38 #include "llvm/MC/TargetRegistry.h" 39 #include "llvm/Support/RISCVISAInfo.h" 40 #include "llvm/Support/raw_ostream.h" 41 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 42 43 using namespace llvm; 44 45 #define DEBUG_TYPE "asm-printer" 46 47 STATISTIC(RISCVNumInstrsCompressed, 48 "Number of RISC-V Compressed instructions emitted"); 49 50 namespace llvm { 51 extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures]; 52 } // namespace llvm 53 54 namespace { 55 class RISCVAsmPrinter : public AsmPrinter { 56 const RISCVSubtarget *STI; 57 58 public: 59 explicit RISCVAsmPrinter(TargetMachine &TM, 60 std::unique_ptr<MCStreamer> Streamer) 61 : AsmPrinter(TM, std::move(Streamer)) {} 62 63 StringRef getPassName() const override { return "RISC-V Assembly Printer"; } 64 65 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 66 const MachineInstr &MI); 67 68 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 69 const MachineInstr &MI); 70 71 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 72 const MachineInstr &MI); 73 74 bool runOnMachineFunction(MachineFunction &MF) override; 75 76 void emitInstruction(const MachineInstr *MI) override; 77 78 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 79 const char *ExtraCode, raw_ostream &OS) override; 80 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, 81 const char *ExtraCode, raw_ostream &OS) override; 82 83 void EmitToStreamer(MCStreamer &S, const MCInst &Inst); 84 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, 85 const MachineInstr *MI); 86 87 typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple; 88 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; 89 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); 90 void LowerKCFI_CHECK(const MachineInstr &MI); 91 void EmitHwasanMemaccessSymbols(Module &M); 92 93 // Wrapper needed for tblgenned pseudo lowering. 94 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const; 95 96 void emitStartOfAsmFile(Module &M) override; 97 void emitEndOfAsmFile(Module &M) override; 98 99 void emitFunctionEntryLabel() override; 100 bool emitDirectiveOptionArch(); 101 102 private: 103 void emitAttributes(const MCSubtargetInfo &SubtargetInfo); 104 105 void emitNTLHint(const MachineInstr *MI); 106 107 bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI); 108 }; 109 } 110 111 void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, 112 const MachineInstr &MI) { 113 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 114 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); 115 116 auto &Ctx = OutStreamer.getContext(); 117 MCSymbol *MILabel = Ctx.createTempSymbol(); 118 OutStreamer.emitLabel(MILabel); 119 120 SM.recordStackMap(*MILabel, MI); 121 assert(NumNOPBytes % NOPBytes == 0 && 122 "Invalid number of NOP bytes requested!"); 123 124 // Scan ahead to trim the shadow. 125 const MachineBasicBlock &MBB = *MI.getParent(); 126 MachineBasicBlock::const_iterator MII(MI); 127 ++MII; 128 while (NumNOPBytes > 0) { 129 if (MII == MBB.end() || MII->isCall() || 130 MII->getOpcode() == RISCV::DBG_VALUE || 131 MII->getOpcode() == TargetOpcode::PATCHPOINT || 132 MII->getOpcode() == TargetOpcode::STACKMAP) 133 break; 134 ++MII; 135 NumNOPBytes -= 4; 136 } 137 138 // Emit nops. 139 emitNops(NumNOPBytes / NOPBytes); 140 } 141 142 // Lower a patchpoint of the form: 143 // [<def>], <id>, <numBytes>, <target>, <numArgs> 144 void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, 145 const MachineInstr &MI) { 146 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 147 148 auto &Ctx = OutStreamer.getContext(); 149 MCSymbol *MILabel = Ctx.createTempSymbol(); 150 OutStreamer.emitLabel(MILabel); 151 SM.recordPatchPoint(*MILabel, MI); 152 153 PatchPointOpers Opers(&MI); 154 155 unsigned EncodedBytes = 0; 156 157 // Emit padding. 158 unsigned NumBytes = Opers.getNumPatchBytes(); 159 assert(NumBytes >= EncodedBytes && 160 "Patchpoint can't request size less than the length of a call."); 161 assert((NumBytes - EncodedBytes) % NOPBytes == 0 && 162 "Invalid number of NOP bytes requested!"); 163 emitNops((NumBytes - EncodedBytes) / NOPBytes); 164 } 165 166 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, 167 const MachineInstr &MI) { 168 unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4; 169 170 StatepointOpers SOpers(&MI); 171 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 172 assert(PatchBytes % NOPBytes == 0 && 173 "Invalid number of NOP bytes requested!"); 174 emitNops(PatchBytes / NOPBytes); 175 } 176 177 auto &Ctx = OutStreamer.getContext(); 178 MCSymbol *MILabel = Ctx.createTempSymbol(); 179 OutStreamer.emitLabel(MILabel); 180 SM.recordStatepoint(*MILabel, MI); 181 } 182 183 void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) { 184 MCInst CInst; 185 bool Res = RISCVRVC::compress(CInst, Inst, *STI); 186 if (Res) 187 ++RISCVNumInstrsCompressed; 188 AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst); 189 } 190 191 // Simple pseudo-instructions have their lowering (with expansion to real 192 // instructions) auto-generated. 193 #include "RISCVGenMCPseudoLowering.inc" 194 195 // If the target supports Zihintntl and the instruction has a nontemporal 196 // MachineMemOperand, emit an NTLH hint instruction before it. 197 void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) { 198 if (!STI->hasStdExtZihintntl()) 199 return; 200 201 if (MI->memoperands_empty()) 202 return; 203 204 MachineMemOperand *MMO = *(MI->memoperands_begin()); 205 if (!MMO->isNonTemporal()) 206 return; 207 208 unsigned NontemporalMode = 0; 209 if (MMO->getFlags() & MONontemporalBit0) 210 NontemporalMode += 0b1; 211 if (MMO->getFlags() & MONontemporalBit1) 212 NontemporalMode += 0b10; 213 214 MCInst Hint; 215 if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs()) 216 Hint.setOpcode(RISCV::C_ADD_HINT); 217 else 218 Hint.setOpcode(RISCV::ADD); 219 220 Hint.addOperand(MCOperand::createReg(RISCV::X0)); 221 Hint.addOperand(MCOperand::createReg(RISCV::X0)); 222 Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode)); 223 224 EmitToStreamer(*OutStreamer, Hint); 225 } 226 227 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) { 228 RISCV_MC::verifyInstructionPredicates(MI->getOpcode(), 229 getSubtargetInfo().getFeatureBits()); 230 231 emitNTLHint(MI); 232 233 // Do any auto-generated pseudo lowerings. 234 if (emitPseudoExpansionLowering(*OutStreamer, MI)) 235 return; 236 237 238 switch (MI->getOpcode()) { 239 case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: 240 LowerHWASAN_CHECK_MEMACCESS(*MI); 241 return; 242 case RISCV::KCFI_CHECK: 243 LowerKCFI_CHECK(*MI); 244 return; 245 case RISCV::PseudoRVVInitUndefM1: 246 case RISCV::PseudoRVVInitUndefM2: 247 case RISCV::PseudoRVVInitUndefM4: 248 case RISCV::PseudoRVVInitUndefM8: 249 return; 250 case TargetOpcode::STACKMAP: 251 return LowerSTACKMAP(*OutStreamer, SM, *MI); 252 case TargetOpcode::PATCHPOINT: 253 return LowerPATCHPOINT(*OutStreamer, SM, *MI); 254 case TargetOpcode::STATEPOINT: 255 return LowerSTATEPOINT(*OutStreamer, SM, *MI); 256 } 257 258 MCInst OutInst; 259 if (!lowerToMCInst(MI, OutInst)) 260 EmitToStreamer(*OutStreamer, OutInst); 261 } 262 263 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, 264 const char *ExtraCode, raw_ostream &OS) { 265 // First try the generic code, which knows about modifiers like 'c' and 'n'. 266 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS)) 267 return false; 268 269 const MachineOperand &MO = MI->getOperand(OpNo); 270 if (ExtraCode && ExtraCode[0]) { 271 if (ExtraCode[1] != 0) 272 return true; // Unknown modifier. 273 274 switch (ExtraCode[0]) { 275 default: 276 return true; // Unknown modifier. 277 case 'z': // Print zero register if zero, regular printing otherwise. 278 if (MO.isImm() && MO.getImm() == 0) { 279 OS << RISCVInstPrinter::getRegisterName(RISCV::X0); 280 return false; 281 } 282 break; 283 case 'i': // Literal 'i' if operand is not a register. 284 if (!MO.isReg()) 285 OS << 'i'; 286 return false; 287 } 288 } 289 290 switch (MO.getType()) { 291 case MachineOperand::MO_Immediate: 292 OS << MO.getImm(); 293 return false; 294 case MachineOperand::MO_Register: 295 OS << RISCVInstPrinter::getRegisterName(MO.getReg()); 296 return false; 297 case MachineOperand::MO_GlobalAddress: 298 PrintSymbolOperand(MO, OS); 299 return false; 300 case MachineOperand::MO_BlockAddress: { 301 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); 302 Sym->print(OS, MAI); 303 return false; 304 } 305 default: 306 break; 307 } 308 309 return true; 310 } 311 312 bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, 313 unsigned OpNo, 314 const char *ExtraCode, 315 raw_ostream &OS) { 316 if (ExtraCode) 317 return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS); 318 319 const MachineOperand &AddrReg = MI->getOperand(OpNo); 320 assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand"); 321 const MachineOperand &Offset = MI->getOperand(OpNo + 1); 322 // All memory operands should have a register and an immediate operand (see 323 // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand). 324 if (!AddrReg.isReg()) 325 return true; 326 if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() && 327 !Offset.isMCSymbol()) 328 return true; 329 330 MCOperand MCO; 331 if (!lowerOperand(Offset, MCO)) 332 return true; 333 334 if (Offset.isImm()) 335 OS << MCO.getImm(); 336 else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol()) 337 OS << *MCO.getExpr(); 338 OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")"; 339 return false; 340 } 341 342 bool RISCVAsmPrinter::emitDirectiveOptionArch() { 343 RISCVTargetStreamer &RTS = 344 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 345 SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs; 346 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo(); 347 for (const auto &Feature : RISCVFeatureKV) { 348 if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value)) 349 continue; 350 351 if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key)) 352 continue; 353 354 auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus 355 : RISCVOptionArchArgType::Minus; 356 NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key); 357 } 358 if (!NeedEmitStdOptionArgs.empty()) { 359 RTS.emitDirectiveOptionPush(); 360 RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs); 361 return true; 362 } 363 364 return false; 365 } 366 367 bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) { 368 STI = &MF.getSubtarget<RISCVSubtarget>(); 369 RISCVTargetStreamer &RTS = 370 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 371 372 bool EmittedOptionArch = emitDirectiveOptionArch(); 373 374 SetupMachineFunction(MF); 375 emitFunctionBody(); 376 377 if (EmittedOptionArch) 378 RTS.emitDirectiveOptionPop(); 379 return false; 380 } 381 382 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) { 383 RISCVTargetStreamer &RTS = 384 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 385 if (const MDString *ModuleTargetABI = 386 dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi"))) 387 RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString())); 388 389 MCSubtargetInfo SubtargetInfo = *TM.getMCSubtargetInfo(); 390 391 // Use module flag to update feature bits. 392 if (auto *MD = dyn_cast_or_null<MDNode>(M.getModuleFlag("riscv-isa"))) { 393 for (auto &ISA : MD->operands()) { 394 if (auto *ISAString = dyn_cast_or_null<MDString>(ISA)) { 395 auto ParseResult = llvm::RISCVISAInfo::parseArchString( 396 ISAString->getString(), /*EnableExperimentalExtension=*/true, 397 /*ExperimentalExtensionVersionCheck=*/true); 398 if (!errorToBool(ParseResult.takeError())) { 399 auto &ISAInfo = *ParseResult; 400 for (const auto &Feature : RISCVFeatureKV) { 401 if (ISAInfo->hasExtension(Feature.Key) && 402 !SubtargetInfo.hasFeature(Feature.Value)) 403 SubtargetInfo.ToggleFeature(Feature.Key); 404 } 405 } 406 } 407 } 408 409 RTS.setFlagsFromFeatures(SubtargetInfo); 410 } 411 412 if (TM.getTargetTriple().isOSBinFormatELF()) 413 emitAttributes(SubtargetInfo); 414 } 415 416 void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) { 417 RISCVTargetStreamer &RTS = 418 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 419 420 if (TM.getTargetTriple().isOSBinFormatELF()) 421 RTS.finishAttributeSection(); 422 EmitHwasanMemaccessSymbols(M); 423 } 424 425 void RISCVAsmPrinter::emitAttributes(const MCSubtargetInfo &SubtargetInfo) { 426 RISCVTargetStreamer &RTS = 427 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 428 // Use MCSubtargetInfo from TargetMachine. Individual functions may have 429 // attributes that differ from other functions in the module and we have no 430 // way to know which function is correct. 431 RTS.emitTargetAttributes(SubtargetInfo, /*EmitStackAlign*/ true); 432 } 433 434 void RISCVAsmPrinter::emitFunctionEntryLabel() { 435 const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>(); 436 if (RMFI->isVectorCall()) { 437 auto &RTS = 438 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 439 RTS.emitDirectiveVariantCC(*CurrentFnSym); 440 } 441 return AsmPrinter::emitFunctionEntryLabel(); 442 } 443 444 // Force static initialization. 445 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() { 446 RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target()); 447 RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target()); 448 } 449 450 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 451 Register Reg = MI.getOperand(0).getReg(); 452 uint32_t AccessInfo = MI.getOperand(1).getImm(); 453 MCSymbol *&Sym = 454 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)]; 455 if (!Sym) { 456 // FIXME: Make this work on non-ELF. 457 if (!TM.getTargetTriple().isOSBinFormatELF()) 458 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); 459 460 std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" + 461 utostr(AccessInfo) + "_short"; 462 Sym = OutContext.getOrCreateSymbol(SymName); 463 } 464 auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext); 465 auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext); 466 467 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr)); 468 } 469 470 void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { 471 Register AddrReg = MI.getOperand(0).getReg(); 472 assert(std::next(MI.getIterator())->isCall() && 473 "KCFI_CHECK not followed by a call instruction"); 474 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg && 475 "KCFI_CHECK call target doesn't match call operand"); 476 477 // Temporary registers for comparing the hashes. If a register is used 478 // for the call target, or reserved by the user, we can clobber another 479 // temporary register as the check is immediately followed by the 480 // call. The check defaults to X6/X7, but can fall back to X28-X31 if 481 // needed. 482 unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7}; 483 unsigned NextReg = RISCV::X28; 484 auto isRegAvailable = [&](unsigned Reg) { 485 return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg); 486 }; 487 for (auto &Reg : ScratchRegs) { 488 if (isRegAvailable(Reg)) 489 continue; 490 while (!isRegAvailable(NextReg)) 491 ++NextReg; 492 Reg = NextReg++; 493 if (Reg > RISCV::X31) 494 report_fatal_error("Unable to find scratch registers for KCFI_CHECK"); 495 } 496 497 if (AddrReg == RISCV::X0) { 498 // Checking X0 makes no sense. Instead of emitting a load, zero 499 // ScratchRegs[0]. 500 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI) 501 .addReg(ScratchRegs[0]) 502 .addReg(RISCV::X0) 503 .addImm(0)); 504 } else { 505 // Adjust the offset for patchable-function-prefix. This assumes that 506 // patchable-function-prefix is the same for all functions. 507 int NopSize = STI->hasStdExtCOrZca() ? 2 : 4; 508 int64_t PrefixNops = 0; 509 (void)MI.getMF() 510 ->getFunction() 511 .getFnAttribute("patchable-function-prefix") 512 .getValueAsString() 513 .getAsInteger(10, PrefixNops); 514 515 // Load the target function type hash. 516 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW) 517 .addReg(ScratchRegs[0]) 518 .addReg(AddrReg) 519 .addImm(-(PrefixNops * NopSize + 4))); 520 } 521 522 // Load the expected 32-bit type hash. 523 const int64_t Type = MI.getOperand(1).getImm(); 524 const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF; 525 const int64_t Lo12 = SignExtend64<12>(Type); 526 if (Hi20) { 527 EmitToStreamer( 528 *OutStreamer, 529 MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20)); 530 } 531 if (Lo12 || Hi20 == 0) { 532 EmitToStreamer(*OutStreamer, 533 MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20) 534 ? RISCV::ADDIW 535 : RISCV::ADDI) 536 .addReg(ScratchRegs[1]) 537 .addReg(ScratchRegs[1]) 538 .addImm(Lo12)); 539 } 540 541 // Compare the hashes and trap if there's a mismatch. 542 MCSymbol *Pass = OutContext.createTempSymbol(); 543 EmitToStreamer(*OutStreamer, 544 MCInstBuilder(RISCV::BEQ) 545 .addReg(ScratchRegs[0]) 546 .addReg(ScratchRegs[1]) 547 .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); 548 549 MCSymbol *Trap = OutContext.createTempSymbol(); 550 OutStreamer->emitLabel(Trap); 551 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK)); 552 emitKCFITrapEntry(*MI.getMF(), Trap); 553 OutStreamer->emitLabel(Pass); 554 } 555 556 void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) { 557 if (HwasanMemaccessSymbols.empty()) 558 return; 559 560 assert(TM.getTargetTriple().isOSBinFormatELF()); 561 // Use MCSubtargetInfo from TargetMachine. Individual functions may have 562 // attributes that differ from other functions in the module and we have no 563 // way to know which function is correct. 564 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo(); 565 566 MCSymbol *HwasanTagMismatchV2Sym = 567 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2"); 568 // Annotate symbol as one having incompatible calling convention, so 569 // run-time linkers can instead eagerly bind this function. 570 auto &RTS = 571 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer()); 572 RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym); 573 574 const MCSymbolRefExpr *HwasanTagMismatchV2Ref = 575 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext); 576 auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref, 577 RISCVMCExpr::VK_RISCV_CALL, OutContext); 578 579 for (auto &P : HwasanMemaccessSymbols) { 580 unsigned Reg = std::get<0>(P.first); 581 uint32_t AccessInfo = std::get<1>(P.first); 582 MCSymbol *Sym = P.second; 583 584 unsigned Size = 585 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); 586 OutStreamer->switchSection(OutContext.getELFSection( 587 ".text.hot", ELF::SHT_PROGBITS, 588 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(), 589 /*IsComdat=*/true)); 590 591 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); 592 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); 593 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); 594 OutStreamer->emitLabel(Sym); 595 596 // Extract shadow offset from ptr 597 OutStreamer->emitInstruction( 598 MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8), 599 MCSTI); 600 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI) 601 .addReg(RISCV::X6) 602 .addReg(RISCV::X6) 603 .addImm(12), 604 MCSTI); 605 // load shadow tag in X6, X5 contains shadow base 606 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD) 607 .addReg(RISCV::X6) 608 .addReg(RISCV::X5) 609 .addReg(RISCV::X6), 610 MCSTI); 611 OutStreamer->emitInstruction( 612 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0), 613 MCSTI); 614 // Extract tag from X5 and compare it with loaded tag from shadow 615 OutStreamer->emitInstruction( 616 MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56), 617 MCSTI); 618 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); 619 // X7 contains tag from memory, while X6 contains tag from the pointer 620 OutStreamer->emitInstruction( 621 MCInstBuilder(RISCV::BNE) 622 .addReg(RISCV::X7) 623 .addReg(RISCV::X6) 624 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, 625 OutContext)), 626 MCSTI); 627 MCSymbol *ReturnSym = OutContext.createTempSymbol(); 628 OutStreamer->emitLabel(ReturnSym); 629 OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR) 630 .addReg(RISCV::X0) 631 .addReg(RISCV::X1) 632 .addImm(0), 633 MCSTI); 634 OutStreamer->emitLabel(HandleMismatchOrPartialSym); 635 636 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 637 .addReg(RISCV::X28) 638 .addReg(RISCV::X0) 639 .addImm(16), 640 MCSTI); 641 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); 642 OutStreamer->emitInstruction( 643 MCInstBuilder(RISCV::BGEU) 644 .addReg(RISCV::X6) 645 .addReg(RISCV::X28) 646 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 647 MCSTI); 648 649 OutStreamer->emitInstruction( 650 MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF), 651 MCSTI); 652 653 if (Size != 1) 654 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 655 .addReg(RISCV::X28) 656 .addReg(RISCV::X28) 657 .addImm(Size - 1), 658 MCSTI); 659 OutStreamer->emitInstruction( 660 MCInstBuilder(RISCV::BGE) 661 .addReg(RISCV::X28) 662 .addReg(RISCV::X6) 663 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), 664 MCSTI); 665 666 OutStreamer->emitInstruction( 667 MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF), 668 MCSTI); 669 OutStreamer->emitInstruction( 670 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0), 671 MCSTI); 672 OutStreamer->emitInstruction( 673 MCInstBuilder(RISCV::BEQ) 674 .addReg(RISCV::X6) 675 .addReg(RISCV::X7) 676 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), 677 MCSTI); 678 679 OutStreamer->emitLabel(HandleMismatchSym); 680 681 // | Previous stack frames... | 682 // +=================================+ <-- [SP + 256] 683 // | ... | 684 // | | 685 // | Stack frame space for x12 - x31.| 686 // | | 687 // | ... | 688 // +---------------------------------+ <-- [SP + 96] 689 // | Saved x11(arg1), as | 690 // | __hwasan_check_* clobbers it. | 691 // +---------------------------------+ <-- [SP + 88] 692 // | Saved x10(arg0), as | 693 // | __hwasan_check_* clobbers it. | 694 // +---------------------------------+ <-- [SP + 80] 695 // | | 696 // | Stack frame space for x9. | 697 // +---------------------------------+ <-- [SP + 72] 698 // | | 699 // | Saved x8(fp), as | 700 // | __hwasan_check_* clobbers it. | 701 // +---------------------------------+ <-- [SP + 64] 702 // | ... | 703 // | | 704 // | Stack frame space for x2 - x7. | 705 // | | 706 // | ... | 707 // +---------------------------------+ <-- [SP + 16] 708 // | Return address (x1) for caller | 709 // | of __hwasan_check_*. | 710 // +---------------------------------+ <-- [SP + 8] 711 // | Reserved place for x0, possibly | 712 // | junk, since we don't save it. | 713 // +---------------------------------+ <-- [x2 / SP] 714 715 // Adjust sp 716 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 717 .addReg(RISCV::X2) 718 .addReg(RISCV::X2) 719 .addImm(-256), 720 MCSTI); 721 722 // store x10(arg0) by new sp 723 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD) 724 .addReg(RISCV::X10) 725 .addReg(RISCV::X2) 726 .addImm(8 * 10), 727 MCSTI); 728 // store x11(arg1) by new sp 729 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD) 730 .addReg(RISCV::X11) 731 .addReg(RISCV::X2) 732 .addImm(8 * 11), 733 MCSTI); 734 735 // store x8(fp) by new sp 736 OutStreamer->emitInstruction( 737 MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 * 738 8), 739 MCSTI); 740 // store x1(ra) by new sp 741 OutStreamer->emitInstruction( 742 MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 * 743 8), 744 MCSTI); 745 if (Reg != RISCV::X10) 746 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI) 747 .addReg(RISCV::X10) 748 .addReg(Reg) 749 .addImm(0), 750 MCSTI); 751 OutStreamer->emitInstruction( 752 MCInstBuilder(RISCV::ADDI) 753 .addReg(RISCV::X11) 754 .addReg(RISCV::X0) 755 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask), 756 MCSTI); 757 758 OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr), 759 MCSTI); 760 } 761 } 762 763 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym, 764 const AsmPrinter &AP) { 765 MCContext &Ctx = AP.OutContext; 766 RISCVMCExpr::VariantKind Kind; 767 768 switch (MO.getTargetFlags()) { 769 default: 770 llvm_unreachable("Unknown target flag on GV operand"); 771 case RISCVII::MO_None: 772 Kind = RISCVMCExpr::VK_RISCV_None; 773 break; 774 case RISCVII::MO_CALL: 775 Kind = RISCVMCExpr::VK_RISCV_CALL_PLT; 776 break; 777 case RISCVII::MO_LO: 778 Kind = RISCVMCExpr::VK_RISCV_LO; 779 break; 780 case RISCVII::MO_HI: 781 Kind = RISCVMCExpr::VK_RISCV_HI; 782 break; 783 case RISCVII::MO_PCREL_LO: 784 Kind = RISCVMCExpr::VK_RISCV_PCREL_LO; 785 break; 786 case RISCVII::MO_PCREL_HI: 787 Kind = RISCVMCExpr::VK_RISCV_PCREL_HI; 788 break; 789 case RISCVII::MO_GOT_HI: 790 Kind = RISCVMCExpr::VK_RISCV_GOT_HI; 791 break; 792 case RISCVII::MO_TPREL_LO: 793 Kind = RISCVMCExpr::VK_RISCV_TPREL_LO; 794 break; 795 case RISCVII::MO_TPREL_HI: 796 Kind = RISCVMCExpr::VK_RISCV_TPREL_HI; 797 break; 798 case RISCVII::MO_TPREL_ADD: 799 Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD; 800 break; 801 case RISCVII::MO_TLS_GOT_HI: 802 Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI; 803 break; 804 case RISCVII::MO_TLS_GD_HI: 805 Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI; 806 break; 807 case RISCVII::MO_TLSDESC_HI: 808 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI; 809 break; 810 case RISCVII::MO_TLSDESC_LOAD_LO: 811 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO; 812 break; 813 case RISCVII::MO_TLSDESC_ADD_LO: 814 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO; 815 break; 816 case RISCVII::MO_TLSDESC_CALL: 817 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL; 818 break; 819 } 820 821 const MCExpr *ME = 822 MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx); 823 824 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset()) 825 ME = MCBinaryExpr::createAdd( 826 ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx); 827 828 if (Kind != RISCVMCExpr::VK_RISCV_None) 829 ME = RISCVMCExpr::create(ME, Kind, Ctx); 830 return MCOperand::createExpr(ME); 831 } 832 833 bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO, 834 MCOperand &MCOp) const { 835 switch (MO.getType()) { 836 default: 837 report_fatal_error("lowerOperand: unknown operand type"); 838 case MachineOperand::MO_Register: 839 // Ignore all implicit register operands. 840 if (MO.isImplicit()) 841 return false; 842 MCOp = MCOperand::createReg(MO.getReg()); 843 break; 844 case MachineOperand::MO_RegisterMask: 845 // Regmasks are like implicit defs. 846 return false; 847 case MachineOperand::MO_Immediate: 848 MCOp = MCOperand::createImm(MO.getImm()); 849 break; 850 case MachineOperand::MO_MachineBasicBlock: 851 MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this); 852 break; 853 case MachineOperand::MO_GlobalAddress: 854 MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this); 855 break; 856 case MachineOperand::MO_BlockAddress: 857 MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()), 858 *this); 859 break; 860 case MachineOperand::MO_ExternalSymbol: 861 MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()), 862 *this); 863 break; 864 case MachineOperand::MO_ConstantPoolIndex: 865 MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this); 866 break; 867 case MachineOperand::MO_JumpTableIndex: 868 MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this); 869 break; 870 case MachineOperand::MO_MCSymbol: 871 MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this); 872 break; 873 } 874 return true; 875 } 876 877 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI, 878 MCInst &OutMI) { 879 const RISCVVPseudosTable::PseudoInfo *RVV = 880 RISCVVPseudosTable::getPseudoInfo(MI->getOpcode()); 881 if (!RVV) 882 return false; 883 884 OutMI.setOpcode(RVV->BaseInstr); 885 886 const MachineBasicBlock *MBB = MI->getParent(); 887 assert(MBB && "MI expected to be in a basic block"); 888 const MachineFunction *MF = MBB->getParent(); 889 assert(MF && "MBB expected to be in a machine function"); 890 891 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>(); 892 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 893 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 894 assert(TRI && "TargetRegisterInfo expected"); 895 896 const MCInstrDesc &MCID = MI->getDesc(); 897 uint64_t TSFlags = MCID.TSFlags; 898 unsigned NumOps = MI->getNumExplicitOperands(); 899 900 // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if 901 // present. 902 if (RISCVII::hasVecPolicyOp(TSFlags)) 903 --NumOps; 904 if (RISCVII::hasSEWOp(TSFlags)) 905 --NumOps; 906 if (RISCVII::hasVLOp(TSFlags)) 907 --NumOps; 908 if (RISCVII::hasRoundModeOp(TSFlags)) 909 --NumOps; 910 911 bool hasVLOutput = RISCV::isFaultFirstLoad(*MI); 912 for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) { 913 const MachineOperand &MO = MI->getOperand(OpNo); 914 // Skip vl ouput. It should be the second output. 915 if (hasVLOutput && OpNo == 1) 916 continue; 917 918 // Skip merge op. It should be the first operand after the defs. 919 if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) { 920 assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 && 921 "Expected tied to first def."); 922 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode()); 923 // Skip if the next operand in OutMI is not supposed to be tied. Unless it 924 // is a _TIED instruction. 925 if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) < 926 0 && 927 !RISCVII::isTiedPseudo(TSFlags)) 928 continue; 929 } 930 931 MCOperand MCOp; 932 switch (MO.getType()) { 933 default: 934 llvm_unreachable("Unknown operand type"); 935 case MachineOperand::MO_Register: { 936 Register Reg = MO.getReg(); 937 938 if (RISCV::VRM2RegClass.contains(Reg) || 939 RISCV::VRM4RegClass.contains(Reg) || 940 RISCV::VRM8RegClass.contains(Reg)) { 941 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0); 942 assert(Reg && "Subregister does not exist"); 943 } else if (RISCV::FPR16RegClass.contains(Reg)) { 944 Reg = 945 TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass); 946 assert(Reg && "Subregister does not exist"); 947 } else if (RISCV::FPR64RegClass.contains(Reg)) { 948 Reg = TRI->getSubReg(Reg, RISCV::sub_32); 949 assert(Reg && "Superregister does not exist"); 950 } else if (RISCV::VRN2M1RegClass.contains(Reg) || 951 RISCV::VRN2M2RegClass.contains(Reg) || 952 RISCV::VRN2M4RegClass.contains(Reg) || 953 RISCV::VRN3M1RegClass.contains(Reg) || 954 RISCV::VRN3M2RegClass.contains(Reg) || 955 RISCV::VRN4M1RegClass.contains(Reg) || 956 RISCV::VRN4M2RegClass.contains(Reg) || 957 RISCV::VRN5M1RegClass.contains(Reg) || 958 RISCV::VRN6M1RegClass.contains(Reg) || 959 RISCV::VRN7M1RegClass.contains(Reg) || 960 RISCV::VRN8M1RegClass.contains(Reg)) { 961 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0); 962 assert(Reg && "Subregister does not exist"); 963 } 964 965 MCOp = MCOperand::createReg(Reg); 966 break; 967 } 968 case MachineOperand::MO_Immediate: 969 MCOp = MCOperand::createImm(MO.getImm()); 970 break; 971 } 972 OutMI.addOperand(MCOp); 973 } 974 975 // Unmasked pseudo instructions need to append dummy mask operand to 976 // V instructions. All V instructions are modeled as the masked version. 977 const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode()); 978 if (OutMI.getNumOperands() < OutMCID.getNumOperands()) { 979 assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass == 980 RISCV::VMV0RegClassID && 981 "Expected only mask operand to be missing"); 982 OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister)); 983 } 984 985 assert(OutMI.getNumOperands() == OutMCID.getNumOperands()); 986 return true; 987 } 988 989 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) { 990 if (lowerRISCVVMachineInstrToMCInst(MI, OutMI)) 991 return false; 992 993 OutMI.setOpcode(MI->getOpcode()); 994 995 for (const MachineOperand &MO : MI->operands()) { 996 MCOperand MCOp; 997 if (lowerOperand(MO, MCOp)) 998 OutMI.addOperand(MCOp); 999 } 1000 1001 switch (OutMI.getOpcode()) { 1002 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: { 1003 const Function &F = MI->getParent()->getParent()->getFunction(); 1004 if (F.hasFnAttribute("patchable-function-entry")) { 1005 unsigned Num; 1006 if (F.getFnAttribute("patchable-function-entry") 1007 .getValueAsString() 1008 .getAsInteger(10, Num)) 1009 return false; 1010 emitNops(Num); 1011 return true; 1012 } 1013 break; 1014 } 1015 } 1016 return false; 1017 } 1018