1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the interfaces that RISCV uses to lower LLVM code into a 10 // selection DAG. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "RISCVISelLowering.h" 15 #include "RISCV.h" 16 #include "RISCVMachineFunctionInfo.h" 17 #include "RISCVRegisterInfo.h" 18 #include "RISCVSubtarget.h" 19 #include "RISCVTargetMachine.h" 20 #include "Utils/RISCVMatInt.h" 21 #include "llvm/ADT/SmallSet.h" 22 #include "llvm/ADT/Statistic.h" 23 #include "llvm/CodeGen/CallingConvLower.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/MachineFunction.h" 26 #include "llvm/CodeGen/MachineInstrBuilder.h" 27 #include "llvm/CodeGen/MachineRegisterInfo.h" 28 #include "llvm/CodeGen/SelectionDAGISel.h" 29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 30 #include "llvm/CodeGen/ValueTypes.h" 31 #include "llvm/IR/DiagnosticInfo.h" 32 #include "llvm/IR/DiagnosticPrinter.h" 33 #include "llvm/IR/IntrinsicsRISCV.h" 34 #include "llvm/Support/Debug.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/MathExtras.h" 37 #include "llvm/Support/raw_ostream.h" 38 39 using namespace llvm; 40 41 #define DEBUG_TYPE "riscv-lower" 42 43 STATISTIC(NumTailCalls, "Number of tail calls"); 44 45 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, 46 const RISCVSubtarget &STI) 47 : TargetLowering(TM), Subtarget(STI) { 48 49 if (Subtarget.isRV32E()) 50 report_fatal_error("Codegen not yet implemented for RV32E"); 51 52 RISCVABI::ABI ABI = Subtarget.getTargetABI(); 53 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI"); 54 55 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) && 56 !Subtarget.hasStdExtF()) { 57 errs() << "Hard-float 'f' ABI can't be used for a target that " 58 "doesn't support the F instruction set extension (ignoring " 59 "target-abi)\n"; 60 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32; 61 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) && 62 !Subtarget.hasStdExtD()) { 63 errs() << "Hard-float 'd' ABI can't be used for a target that " 64 "doesn't support the D instruction set extension (ignoring " 65 "target-abi)\n"; 66 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32; 67 } 68 69 switch (ABI) { 70 default: 71 report_fatal_error("Don't know how to lower this ABI"); 72 case RISCVABI::ABI_ILP32: 73 case RISCVABI::ABI_ILP32F: 74 case RISCVABI::ABI_ILP32D: 75 case RISCVABI::ABI_LP64: 76 case RISCVABI::ABI_LP64F: 77 case RISCVABI::ABI_LP64D: 78 break; 79 } 80 81 MVT XLenVT = Subtarget.getXLenVT(); 82 83 // Set up the register classes. 84 addRegisterClass(XLenVT, &RISCV::GPRRegClass); 85 86 if (Subtarget.hasStdExtF()) 87 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass); 88 if (Subtarget.hasStdExtD()) 89 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass); 90 91 // Compute derived properties from the register classes. 92 computeRegisterProperties(STI.getRegisterInfo()); 93 94 setStackPointerRegisterToSaveRestore(RISCV::X2); 95 96 for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) 97 setLoadExtAction(N, XLenVT, MVT::i1, Promote); 98 99 // TODO: add all necessary setOperationAction calls. 100 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand); 101 102 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 103 setOperationAction(ISD::BR_CC, XLenVT, Expand); 104 setOperationAction(ISD::SELECT, XLenVT, Custom); 105 setOperationAction(ISD::SELECT_CC, XLenVT, Expand); 106 107 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 108 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 109 110 setOperationAction(ISD::VASTART, MVT::Other, Custom); 111 setOperationAction(ISD::VAARG, MVT::Other, Expand); 112 setOperationAction(ISD::VACOPY, MVT::Other, Expand); 113 setOperationAction(ISD::VAEND, MVT::Other, Expand); 114 115 for (auto VT : {MVT::i1, MVT::i8, MVT::i16}) 116 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 117 118 if (Subtarget.is64Bit()) { 119 setOperationAction(ISD::ADD, MVT::i32, Custom); 120 setOperationAction(ISD::SUB, MVT::i32, Custom); 121 setOperationAction(ISD::SHL, MVT::i32, Custom); 122 setOperationAction(ISD::SRA, MVT::i32, Custom); 123 setOperationAction(ISD::SRL, MVT::i32, Custom); 124 } 125 126 if (!Subtarget.hasStdExtM()) { 127 setOperationAction(ISD::MUL, XLenVT, Expand); 128 setOperationAction(ISD::MULHS, XLenVT, Expand); 129 setOperationAction(ISD::MULHU, XLenVT, Expand); 130 setOperationAction(ISD::SDIV, XLenVT, Expand); 131 setOperationAction(ISD::UDIV, XLenVT, Expand); 132 setOperationAction(ISD::SREM, XLenVT, Expand); 133 setOperationAction(ISD::UREM, XLenVT, Expand); 134 } 135 136 if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) { 137 setOperationAction(ISD::MUL, MVT::i32, Custom); 138 setOperationAction(ISD::SDIV, MVT::i32, Custom); 139 setOperationAction(ISD::UDIV, MVT::i32, Custom); 140 setOperationAction(ISD::UREM, MVT::i32, Custom); 141 } 142 143 setOperationAction(ISD::SDIVREM, XLenVT, Expand); 144 setOperationAction(ISD::UDIVREM, XLenVT, Expand); 145 setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand); 146 setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand); 147 148 setOperationAction(ISD::SHL_PARTS, XLenVT, Custom); 149 setOperationAction(ISD::SRL_PARTS, XLenVT, Custom); 150 setOperationAction(ISD::SRA_PARTS, XLenVT, Custom); 151 152 if (!(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp())) { 153 setOperationAction(ISD::ROTL, XLenVT, Expand); 154 setOperationAction(ISD::ROTR, XLenVT, Expand); 155 } 156 157 if (!Subtarget.hasStdExtZbp()) 158 setOperationAction(ISD::BSWAP, XLenVT, Expand); 159 160 if (!Subtarget.hasStdExtZbb()) { 161 setOperationAction(ISD::CTTZ, XLenVT, Expand); 162 setOperationAction(ISD::CTLZ, XLenVT, Expand); 163 setOperationAction(ISD::CTPOP, XLenVT, Expand); 164 } 165 166 if (Subtarget.hasStdExtZbp()) 167 setOperationAction(ISD::BITREVERSE, XLenVT, Legal); 168 169 if (Subtarget.hasStdExtZbt()) { 170 setOperationAction(ISD::FSHL, XLenVT, Legal); 171 setOperationAction(ISD::FSHR, XLenVT, Legal); 172 } 173 174 ISD::CondCode FPCCToExtend[] = { 175 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT, 176 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT, 177 ISD::SETGE, ISD::SETNE}; 178 179 ISD::NodeType FPOpToExtend[] = { 180 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FP16_TO_FP, 181 ISD::FP_TO_FP16}; 182 183 if (Subtarget.hasStdExtF()) { 184 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); 185 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); 186 for (auto CC : FPCCToExtend) 187 setCondCodeAction(CC, MVT::f32, Expand); 188 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); 189 setOperationAction(ISD::SELECT, MVT::f32, Custom); 190 setOperationAction(ISD::BR_CC, MVT::f32, Expand); 191 for (auto Op : FPOpToExtend) 192 setOperationAction(Op, MVT::f32, Expand); 193 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); 194 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 195 } 196 197 if (Subtarget.hasStdExtF() && Subtarget.is64Bit()) 198 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 199 200 if (Subtarget.hasStdExtD()) { 201 setOperationAction(ISD::FMINNUM, MVT::f64, Legal); 202 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); 203 for (auto CC : FPCCToExtend) 204 setCondCodeAction(CC, MVT::f64, Expand); 205 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); 206 setOperationAction(ISD::SELECT, MVT::f64, Custom); 207 setOperationAction(ISD::BR_CC, MVT::f64, Expand); 208 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); 209 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 210 for (auto Op : FPOpToExtend) 211 setOperationAction(Op, MVT::f64, Expand); 212 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); 213 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 214 } 215 216 if (Subtarget.is64Bit() && 217 !(Subtarget.hasStdExtD() || Subtarget.hasStdExtF())) { 218 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 219 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 220 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); 221 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); 222 } 223 224 setOperationAction(ISD::GlobalAddress, XLenVT, Custom); 225 setOperationAction(ISD::BlockAddress, XLenVT, Custom); 226 setOperationAction(ISD::ConstantPool, XLenVT, Custom); 227 228 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom); 229 230 // TODO: On M-mode only targets, the cycle[h] CSR may not be present. 231 // Unfortunately this can't be determined just from the ISA naming string. 232 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, 233 Subtarget.is64Bit() ? Legal : Custom); 234 235 setOperationAction(ISD::TRAP, MVT::Other, Legal); 236 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); 237 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 238 239 if (Subtarget.hasStdExtA()) { 240 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen()); 241 setMinCmpXchgSizeInBits(32); 242 } else { 243 setMaxAtomicSizeInBitsSupported(0); 244 } 245 246 setBooleanContents(ZeroOrOneBooleanContent); 247 248 // Function alignments. 249 const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4); 250 setMinFunctionAlignment(FunctionAlignment); 251 setPrefFunctionAlignment(FunctionAlignment); 252 253 // Effectively disable jump table generation. 254 setMinimumJumpTableEntries(INT_MAX); 255 256 // Jumps are expensive, compared to logic 257 setJumpIsExpensive(); 258 259 // We can use any register for comparisons 260 setHasMultipleConditionRegisters(); 261 } 262 263 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, 264 EVT VT) const { 265 if (!VT.isVector()) 266 return getPointerTy(DL); 267 return VT.changeVectorElementTypeToInteger(); 268 } 269 270 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 271 const CallInst &I, 272 MachineFunction &MF, 273 unsigned Intrinsic) const { 274 switch (Intrinsic) { 275 default: 276 return false; 277 case Intrinsic::riscv_masked_atomicrmw_xchg_i32: 278 case Intrinsic::riscv_masked_atomicrmw_add_i32: 279 case Intrinsic::riscv_masked_atomicrmw_sub_i32: 280 case Intrinsic::riscv_masked_atomicrmw_nand_i32: 281 case Intrinsic::riscv_masked_atomicrmw_max_i32: 282 case Intrinsic::riscv_masked_atomicrmw_min_i32: 283 case Intrinsic::riscv_masked_atomicrmw_umax_i32: 284 case Intrinsic::riscv_masked_atomicrmw_umin_i32: 285 case Intrinsic::riscv_masked_cmpxchg_i32: 286 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType()); 287 Info.opc = ISD::INTRINSIC_W_CHAIN; 288 Info.memVT = MVT::getVT(PtrTy->getElementType()); 289 Info.ptrVal = I.getArgOperand(0); 290 Info.offset = 0; 291 Info.align = Align(4); 292 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore | 293 MachineMemOperand::MOVolatile; 294 return true; 295 } 296 } 297 298 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL, 299 const AddrMode &AM, Type *Ty, 300 unsigned AS, 301 Instruction *I) const { 302 // No global is ever allowed as a base. 303 if (AM.BaseGV) 304 return false; 305 306 // Require a 12-bit signed offset. 307 if (!isInt<12>(AM.BaseOffs)) 308 return false; 309 310 switch (AM.Scale) { 311 case 0: // "r+i" or just "i", depending on HasBaseReg. 312 break; 313 case 1: 314 if (!AM.HasBaseReg) // allow "r+i". 315 break; 316 return false; // disallow "r+r" or "r+r+i". 317 default: 318 return false; 319 } 320 321 return true; 322 } 323 324 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 325 return isInt<12>(Imm); 326 } 327 328 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const { 329 return isInt<12>(Imm); 330 } 331 332 // On RV32, 64-bit integers are split into their high and low parts and held 333 // in two different registers, so the trunc is free since the low register can 334 // just be used. 335 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const { 336 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy()) 337 return false; 338 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits(); 339 unsigned DestBits = DstTy->getPrimitiveSizeInBits(); 340 return (SrcBits == 64 && DestBits == 32); 341 } 342 343 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const { 344 if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() || 345 !SrcVT.isInteger() || !DstVT.isInteger()) 346 return false; 347 unsigned SrcBits = SrcVT.getSizeInBits(); 348 unsigned DestBits = DstVT.getSizeInBits(); 349 return (SrcBits == 64 && DestBits == 32); 350 } 351 352 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 353 // Zexts are free if they can be combined with a load. 354 if (auto *LD = dyn_cast<LoadSDNode>(Val)) { 355 EVT MemVT = LD->getMemoryVT(); 356 if ((MemVT == MVT::i8 || MemVT == MVT::i16 || 357 (Subtarget.is64Bit() && MemVT == MVT::i32)) && 358 (LD->getExtensionType() == ISD::NON_EXTLOAD || 359 LD->getExtensionType() == ISD::ZEXTLOAD)) 360 return true; 361 } 362 363 return TargetLowering::isZExtFree(Val, VT2); 364 } 365 366 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const { 367 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64; 368 } 369 370 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, 371 bool ForCodeSize) const { 372 if (VT == MVT::f32 && !Subtarget.hasStdExtF()) 373 return false; 374 if (VT == MVT::f64 && !Subtarget.hasStdExtD()) 375 return false; 376 if (Imm.isNegZero()) 377 return false; 378 return Imm.isZero(); 379 } 380 381 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const { 382 return (VT == MVT::f32 && Subtarget.hasStdExtF()) || 383 (VT == MVT::f64 && Subtarget.hasStdExtD()); 384 } 385 386 // Changes the condition code and swaps operands if necessary, so the SetCC 387 // operation matches one of the comparisons supported directly in the RISC-V 388 // ISA. 389 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) { 390 switch (CC) { 391 default: 392 break; 393 case ISD::SETGT: 394 case ISD::SETLE: 395 case ISD::SETUGT: 396 case ISD::SETULE: 397 CC = ISD::getSetCCSwappedOperands(CC); 398 std::swap(LHS, RHS); 399 break; 400 } 401 } 402 403 // Return the RISC-V branch opcode that matches the given DAG integer 404 // condition code. The CondCode must be one of those supported by the RISC-V 405 // ISA (see normaliseSetCC). 406 static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC) { 407 switch (CC) { 408 default: 409 llvm_unreachable("Unsupported CondCode"); 410 case ISD::SETEQ: 411 return RISCV::BEQ; 412 case ISD::SETNE: 413 return RISCV::BNE; 414 case ISD::SETLT: 415 return RISCV::BLT; 416 case ISD::SETGE: 417 return RISCV::BGE; 418 case ISD::SETULT: 419 return RISCV::BLTU; 420 case ISD::SETUGE: 421 return RISCV::BGEU; 422 } 423 } 424 425 SDValue RISCVTargetLowering::LowerOperation(SDValue Op, 426 SelectionDAG &DAG) const { 427 switch (Op.getOpcode()) { 428 default: 429 report_fatal_error("unimplemented operand"); 430 case ISD::GlobalAddress: 431 return lowerGlobalAddress(Op, DAG); 432 case ISD::BlockAddress: 433 return lowerBlockAddress(Op, DAG); 434 case ISD::ConstantPool: 435 return lowerConstantPool(Op, DAG); 436 case ISD::GlobalTLSAddress: 437 return lowerGlobalTLSAddress(Op, DAG); 438 case ISD::SELECT: 439 return lowerSELECT(Op, DAG); 440 case ISD::VASTART: 441 return lowerVASTART(Op, DAG); 442 case ISD::FRAMEADDR: 443 return lowerFRAMEADDR(Op, DAG); 444 case ISD::RETURNADDR: 445 return lowerRETURNADDR(Op, DAG); 446 case ISD::SHL_PARTS: 447 return lowerShiftLeftParts(Op, DAG); 448 case ISD::SRA_PARTS: 449 return lowerShiftRightParts(Op, DAG, true); 450 case ISD::SRL_PARTS: 451 return lowerShiftRightParts(Op, DAG, false); 452 case ISD::BITCAST: { 453 assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() && 454 "Unexpected custom legalisation"); 455 SDLoc DL(Op); 456 SDValue Op0 = Op.getOperand(0); 457 if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32) 458 return SDValue(); 459 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); 460 SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0); 461 return FPConv; 462 } 463 case ISD::INTRINSIC_WO_CHAIN: 464 return LowerINTRINSIC_WO_CHAIN(Op, DAG); 465 } 466 } 467 468 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty, 469 SelectionDAG &DAG, unsigned Flags) { 470 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags); 471 } 472 473 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty, 474 SelectionDAG &DAG, unsigned Flags) { 475 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(), 476 Flags); 477 } 478 479 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty, 480 SelectionDAG &DAG, unsigned Flags) { 481 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(), 482 N->getOffset(), Flags); 483 } 484 485 template <class NodeTy> 486 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG, 487 bool IsLocal) const { 488 SDLoc DL(N); 489 EVT Ty = getPointerTy(DAG.getDataLayout()); 490 491 if (isPositionIndependent()) { 492 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0); 493 if (IsLocal) 494 // Use PC-relative addressing to access the symbol. This generates the 495 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym)) 496 // %pcrel_lo(auipc)). 497 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0); 498 499 // Use PC-relative addressing to access the GOT for this symbol, then load 500 // the address from the GOT. This generates the pattern (PseudoLA sym), 501 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))). 502 return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0); 503 } 504 505 switch (getTargetMachine().getCodeModel()) { 506 default: 507 report_fatal_error("Unsupported code model for lowering"); 508 case CodeModel::Small: { 509 // Generate a sequence for accessing addresses within the first 2 GiB of 510 // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)). 511 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI); 512 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO); 513 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0); 514 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0); 515 } 516 case CodeModel::Medium: { 517 // Generate a sequence for accessing addresses within any 2GiB range within 518 // the address space. This generates the pattern (PseudoLLA sym), which 519 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)). 520 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0); 521 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0); 522 } 523 } 524 } 525 526 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op, 527 SelectionDAG &DAG) const { 528 SDLoc DL(Op); 529 EVT Ty = Op.getValueType(); 530 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op); 531 int64_t Offset = N->getOffset(); 532 MVT XLenVT = Subtarget.getXLenVT(); 533 534 const GlobalValue *GV = N->getGlobal(); 535 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); 536 SDValue Addr = getAddr(N, DAG, IsLocal); 537 538 // In order to maximise the opportunity for common subexpression elimination, 539 // emit a separate ADD node for the global address offset instead of folding 540 // it in the global address node. Later peephole optimisations may choose to 541 // fold it back in when profitable. 542 if (Offset != 0) 543 return DAG.getNode(ISD::ADD, DL, Ty, Addr, 544 DAG.getConstant(Offset, DL, XLenVT)); 545 return Addr; 546 } 547 548 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op, 549 SelectionDAG &DAG) const { 550 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op); 551 552 return getAddr(N, DAG); 553 } 554 555 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op, 556 SelectionDAG &DAG) const { 557 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op); 558 559 return getAddr(N, DAG); 560 } 561 562 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N, 563 SelectionDAG &DAG, 564 bool UseGOT) const { 565 SDLoc DL(N); 566 EVT Ty = getPointerTy(DAG.getDataLayout()); 567 const GlobalValue *GV = N->getGlobal(); 568 MVT XLenVT = Subtarget.getXLenVT(); 569 570 if (UseGOT) { 571 // Use PC-relative addressing to access the GOT for this TLS symbol, then 572 // load the address from the GOT and add the thread pointer. This generates 573 // the pattern (PseudoLA_TLS_IE sym), which expands to 574 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)). 575 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0); 576 SDValue Load = 577 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0); 578 579 // Add the thread pointer. 580 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT); 581 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg); 582 } 583 584 // Generate a sequence for accessing the address relative to the thread 585 // pointer, with the appropriate adjustment for the thread pointer offset. 586 // This generates the pattern 587 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym)) 588 SDValue AddrHi = 589 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI); 590 SDValue AddrAdd = 591 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD); 592 SDValue AddrLo = 593 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO); 594 595 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0); 596 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT); 597 SDValue MNAdd = SDValue( 598 DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd), 599 0); 600 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0); 601 } 602 603 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N, 604 SelectionDAG &DAG) const { 605 SDLoc DL(N); 606 EVT Ty = getPointerTy(DAG.getDataLayout()); 607 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits()); 608 const GlobalValue *GV = N->getGlobal(); 609 610 // Use a PC-relative addressing mode to access the global dynamic GOT address. 611 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to 612 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)). 613 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0); 614 SDValue Load = 615 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0); 616 617 // Prepare argument list to generate call. 618 ArgListTy Args; 619 ArgListEntry Entry; 620 Entry.Node = Load; 621 Entry.Ty = CallTy; 622 Args.push_back(Entry); 623 624 // Setup call to __tls_get_addr. 625 TargetLowering::CallLoweringInfo CLI(DAG); 626 CLI.setDebugLoc(DL) 627 .setChain(DAG.getEntryNode()) 628 .setLibCallee(CallingConv::C, CallTy, 629 DAG.getExternalSymbol("__tls_get_addr", Ty), 630 std::move(Args)); 631 632 return LowerCallTo(CLI).first; 633 } 634 635 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op, 636 SelectionDAG &DAG) const { 637 SDLoc DL(Op); 638 EVT Ty = Op.getValueType(); 639 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op); 640 int64_t Offset = N->getOffset(); 641 MVT XLenVT = Subtarget.getXLenVT(); 642 643 TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal()); 644 645 SDValue Addr; 646 switch (Model) { 647 case TLSModel::LocalExec: 648 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false); 649 break; 650 case TLSModel::InitialExec: 651 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true); 652 break; 653 case TLSModel::LocalDynamic: 654 case TLSModel::GeneralDynamic: 655 Addr = getDynamicTLSAddr(N, DAG); 656 break; 657 } 658 659 // In order to maximise the opportunity for common subexpression elimination, 660 // emit a separate ADD node for the global address offset instead of folding 661 // it in the global address node. Later peephole optimisations may choose to 662 // fold it back in when profitable. 663 if (Offset != 0) 664 return DAG.getNode(ISD::ADD, DL, Ty, Addr, 665 DAG.getConstant(Offset, DL, XLenVT)); 666 return Addr; 667 } 668 669 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const { 670 SDValue CondV = Op.getOperand(0); 671 SDValue TrueV = Op.getOperand(1); 672 SDValue FalseV = Op.getOperand(2); 673 SDLoc DL(Op); 674 MVT XLenVT = Subtarget.getXLenVT(); 675 676 // If the result type is XLenVT and CondV is the output of a SETCC node 677 // which also operated on XLenVT inputs, then merge the SETCC node into the 678 // lowered RISCVISD::SELECT_CC to take advantage of the integer 679 // compare+branch instructions. i.e.: 680 // (select (setcc lhs, rhs, cc), truev, falsev) 681 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev) 682 if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC && 683 CondV.getOperand(0).getSimpleValueType() == XLenVT) { 684 SDValue LHS = CondV.getOperand(0); 685 SDValue RHS = CondV.getOperand(1); 686 auto CC = cast<CondCodeSDNode>(CondV.getOperand(2)); 687 ISD::CondCode CCVal = CC->get(); 688 689 normaliseSetCC(LHS, RHS, CCVal); 690 691 SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT); 692 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 693 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV}; 694 return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops); 695 } 696 697 // Otherwise: 698 // (select condv, truev, falsev) 699 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev) 700 SDValue Zero = DAG.getConstant(0, DL, XLenVT); 701 SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT); 702 703 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 704 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV}; 705 706 return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops); 707 } 708 709 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const { 710 MachineFunction &MF = DAG.getMachineFunction(); 711 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>(); 712 713 SDLoc DL(Op); 714 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 715 getPointerTy(MF.getDataLayout())); 716 717 // vastart just stores the address of the VarArgsFrameIndex slot into the 718 // memory location argument. 719 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 720 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1), 721 MachinePointerInfo(SV)); 722 } 723 724 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op, 725 SelectionDAG &DAG) const { 726 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); 727 MachineFunction &MF = DAG.getMachineFunction(); 728 MachineFrameInfo &MFI = MF.getFrameInfo(); 729 MFI.setFrameAddressIsTaken(true); 730 Register FrameReg = RI.getFrameRegister(MF); 731 int XLenInBytes = Subtarget.getXLen() / 8; 732 733 EVT VT = Op.getValueType(); 734 SDLoc DL(Op); 735 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT); 736 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 737 while (Depth--) { 738 int Offset = -(XLenInBytes * 2); 739 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr, 740 DAG.getIntPtrConstant(Offset, DL)); 741 FrameAddr = 742 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); 743 } 744 return FrameAddr; 745 } 746 747 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op, 748 SelectionDAG &DAG) const { 749 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); 750 MachineFunction &MF = DAG.getMachineFunction(); 751 MachineFrameInfo &MFI = MF.getFrameInfo(); 752 MFI.setReturnAddressIsTaken(true); 753 MVT XLenVT = Subtarget.getXLenVT(); 754 int XLenInBytes = Subtarget.getXLen() / 8; 755 756 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 757 return SDValue(); 758 759 EVT VT = Op.getValueType(); 760 SDLoc DL(Op); 761 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 762 if (Depth) { 763 int Off = -XLenInBytes; 764 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG); 765 SDValue Offset = DAG.getConstant(Off, DL, VT); 766 return DAG.getLoad(VT, DL, DAG.getEntryNode(), 767 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), 768 MachinePointerInfo()); 769 } 770 771 // Return the value of the return address register, marking it an implicit 772 // live-in. 773 Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT)); 774 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT); 775 } 776 777 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op, 778 SelectionDAG &DAG) const { 779 SDLoc DL(Op); 780 SDValue Lo = Op.getOperand(0); 781 SDValue Hi = Op.getOperand(1); 782 SDValue Shamt = Op.getOperand(2); 783 EVT VT = Lo.getValueType(); 784 785 // if Shamt-XLEN < 0: // Shamt < XLEN 786 // Lo = Lo << Shamt 787 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt)) 788 // else: 789 // Lo = 0 790 // Hi = Lo << (Shamt-XLEN) 791 792 SDValue Zero = DAG.getConstant(0, DL, VT); 793 SDValue One = DAG.getConstant(1, DL, VT); 794 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); 795 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); 796 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); 797 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); 798 799 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt); 800 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One); 801 SDValue ShiftRightLo = 802 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt); 803 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt); 804 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo); 805 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen); 806 807 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); 808 809 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero); 810 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); 811 812 SDValue Parts[2] = {Lo, Hi}; 813 return DAG.getMergeValues(Parts, DL); 814 } 815 816 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, 817 bool IsSRA) const { 818 SDLoc DL(Op); 819 SDValue Lo = Op.getOperand(0); 820 SDValue Hi = Op.getOperand(1); 821 SDValue Shamt = Op.getOperand(2); 822 EVT VT = Lo.getValueType(); 823 824 // SRA expansion: 825 // if Shamt-XLEN < 0: // Shamt < XLEN 826 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt)) 827 // Hi = Hi >>s Shamt 828 // else: 829 // Lo = Hi >>s (Shamt-XLEN); 830 // Hi = Hi >>s (XLEN-1) 831 // 832 // SRL expansion: 833 // if Shamt-XLEN < 0: // Shamt < XLEN 834 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt)) 835 // Hi = Hi >>u Shamt 836 // else: 837 // Lo = Hi >>u (Shamt-XLEN); 838 // Hi = 0; 839 840 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL; 841 842 SDValue Zero = DAG.getConstant(0, DL, VT); 843 SDValue One = DAG.getConstant(1, DL, VT); 844 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT); 845 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT); 846 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); 847 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); 848 849 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt); 850 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One); 851 SDValue ShiftLeftHi = 852 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt); 853 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi); 854 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt); 855 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen); 856 SDValue HiFalse = 857 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero; 858 859 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); 860 861 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse); 862 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); 863 864 SDValue Parts[2] = {Lo, Hi}; 865 return DAG.getMergeValues(Parts, DL); 866 } 867 868 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 869 SelectionDAG &DAG) const { 870 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 871 SDLoc DL(Op); 872 switch (IntNo) { 873 default: 874 return SDValue(); // Don't custom lower most intrinsics. 875 case Intrinsic::thread_pointer: { 876 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 877 return DAG.getRegister(RISCV::X4, PtrVT); 878 } 879 } 880 } 881 882 // Returns the opcode of the target-specific SDNode that implements the 32-bit 883 // form of the given Opcode. 884 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) { 885 switch (Opcode) { 886 default: 887 llvm_unreachable("Unexpected opcode"); 888 case ISD::SHL: 889 return RISCVISD::SLLW; 890 case ISD::SRA: 891 return RISCVISD::SRAW; 892 case ISD::SRL: 893 return RISCVISD::SRLW; 894 case ISD::SDIV: 895 return RISCVISD::DIVW; 896 case ISD::UDIV: 897 return RISCVISD::DIVUW; 898 case ISD::UREM: 899 return RISCVISD::REMUW; 900 } 901 } 902 903 // Converts the given 32-bit operation to a target-specific SelectionDAG node. 904 // Because i32 isn't a legal type for RV64, these operations would otherwise 905 // be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W 906 // later one because the fact the operation was originally of type i32 is 907 // lost. 908 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG) { 909 SDLoc DL(N); 910 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode()); 911 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); 912 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); 913 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1); 914 // ReplaceNodeResults requires we maintain the same type for the return value. 915 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes); 916 } 917 918 // Converts the given 32-bit operation to a i64 operation with signed extension 919 // semantic to reduce the signed extension instructions. 920 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) { 921 SDLoc DL(N); 922 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); 923 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); 924 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1); 925 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, 926 DAG.getValueType(MVT::i32)); 927 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes); 928 } 929 930 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, 931 SmallVectorImpl<SDValue> &Results, 932 SelectionDAG &DAG) const { 933 SDLoc DL(N); 934 switch (N->getOpcode()) { 935 default: 936 llvm_unreachable("Don't know how to custom type legalize this operation!"); 937 case ISD::STRICT_FP_TO_SINT: 938 case ISD::STRICT_FP_TO_UINT: 939 case ISD::FP_TO_SINT: 940 case ISD::FP_TO_UINT: { 941 bool IsStrict = N->isStrictFPOpcode(); 942 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && 943 "Unexpected custom legalisation"); 944 SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0); 945 RTLIB::Libcall LC; 946 if (N->getOpcode() == ISD::FP_TO_SINT || 947 N->getOpcode() == ISD::STRICT_FP_TO_SINT) 948 LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0)); 949 else 950 LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0)); 951 MakeLibCallOptions CallOptions; 952 EVT OpVT = Op0.getValueType(); 953 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 954 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 955 SDValue Result; 956 std::tie(Result, Chain) = 957 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain); 958 Results.push_back(Result); 959 if (IsStrict) 960 Results.push_back(Chain); 961 break; 962 } 963 case ISD::READCYCLECOUNTER: { 964 assert(!Subtarget.is64Bit() && 965 "READCYCLECOUNTER only has custom type legalization on riscv32"); 966 967 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 968 SDValue RCW = 969 DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0)); 970 971 Results.push_back( 972 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1))); 973 Results.push_back(RCW.getValue(2)); 974 break; 975 } 976 case ISD::ADD: 977 case ISD::SUB: 978 case ISD::MUL: 979 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && 980 "Unexpected custom legalisation"); 981 if (N->getOperand(1).getOpcode() == ISD::Constant) 982 return; 983 Results.push_back(customLegalizeToWOpWithSExt(N, DAG)); 984 break; 985 case ISD::SHL: 986 case ISD::SRA: 987 case ISD::SRL: 988 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && 989 "Unexpected custom legalisation"); 990 if (N->getOperand(1).getOpcode() == ISD::Constant) 991 return; 992 Results.push_back(customLegalizeToWOp(N, DAG)); 993 break; 994 case ISD::SDIV: 995 case ISD::UDIV: 996 case ISD::UREM: 997 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && 998 Subtarget.hasStdExtM() && "Unexpected custom legalisation"); 999 if (N->getOperand(0).getOpcode() == ISD::Constant || 1000 N->getOperand(1).getOpcode() == ISD::Constant) 1001 return; 1002 Results.push_back(customLegalizeToWOp(N, DAG)); 1003 break; 1004 case ISD::BITCAST: { 1005 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && 1006 Subtarget.hasStdExtF() && "Unexpected custom legalisation"); 1007 SDLoc DL(N); 1008 SDValue Op0 = N->getOperand(0); 1009 if (Op0.getValueType() != MVT::f32) 1010 return; 1011 SDValue FPConv = 1012 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0); 1013 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv)); 1014 break; 1015 } 1016 } 1017 } 1018 1019 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N, 1020 DAGCombinerInfo &DCI) const { 1021 SelectionDAG &DAG = DCI.DAG; 1022 1023 switch (N->getOpcode()) { 1024 default: 1025 break; 1026 case RISCVISD::SplitF64: { 1027 SDValue Op0 = N->getOperand(0); 1028 // If the input to SplitF64 is just BuildPairF64 then the operation is 1029 // redundant. Instead, use BuildPairF64's operands directly. 1030 if (Op0->getOpcode() == RISCVISD::BuildPairF64) 1031 return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1)); 1032 1033 SDLoc DL(N); 1034 1035 // It's cheaper to materialise two 32-bit integers than to load a double 1036 // from the constant pool and transfer it to integer registers through the 1037 // stack. 1038 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) { 1039 APInt V = C->getValueAPF().bitcastToAPInt(); 1040 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32); 1041 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32); 1042 return DCI.CombineTo(N, Lo, Hi); 1043 } 1044 1045 // This is a target-specific version of a DAGCombine performed in 1046 // DAGCombiner::visitBITCAST. It performs the equivalent of: 1047 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit) 1048 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit)) 1049 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) || 1050 !Op0.getNode()->hasOneUse()) 1051 break; 1052 SDValue NewSplitF64 = 1053 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), 1054 Op0.getOperand(0)); 1055 SDValue Lo = NewSplitF64.getValue(0); 1056 SDValue Hi = NewSplitF64.getValue(1); 1057 APInt SignBit = APInt::getSignMask(32); 1058 if (Op0.getOpcode() == ISD::FNEG) { 1059 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi, 1060 DAG.getConstant(SignBit, DL, MVT::i32)); 1061 return DCI.CombineTo(N, Lo, NewHi); 1062 } 1063 assert(Op0.getOpcode() == ISD::FABS); 1064 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi, 1065 DAG.getConstant(~SignBit, DL, MVT::i32)); 1066 return DCI.CombineTo(N, Lo, NewHi); 1067 } 1068 case RISCVISD::SLLW: 1069 case RISCVISD::SRAW: 1070 case RISCVISD::SRLW: { 1071 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read. 1072 SDValue LHS = N->getOperand(0); 1073 SDValue RHS = N->getOperand(1); 1074 APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32); 1075 APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5); 1076 if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) || 1077 (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI))) 1078 return SDValue(); 1079 break; 1080 } 1081 case RISCVISD::FMV_X_ANYEXTW_RV64: { 1082 SDLoc DL(N); 1083 SDValue Op0 = N->getOperand(0); 1084 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the 1085 // conversion is unnecessary and can be replaced with an ANY_EXTEND 1086 // of the FMV_W_X_RV64 operand. 1087 if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) { 1088 SDValue AExtOp = 1089 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0)); 1090 return DCI.CombineTo(N, AExtOp); 1091 } 1092 1093 // This is a target-specific version of a DAGCombine performed in 1094 // DAGCombiner::visitBITCAST. It performs the equivalent of: 1095 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit) 1096 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit)) 1097 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) || 1098 !Op0.getNode()->hasOneUse()) 1099 break; 1100 SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, 1101 Op0.getOperand(0)); 1102 APInt SignBit = APInt::getSignMask(32).sext(64); 1103 if (Op0.getOpcode() == ISD::FNEG) { 1104 return DCI.CombineTo(N, 1105 DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV, 1106 DAG.getConstant(SignBit, DL, MVT::i64))); 1107 } 1108 assert(Op0.getOpcode() == ISD::FABS); 1109 return DCI.CombineTo(N, 1110 DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV, 1111 DAG.getConstant(~SignBit, DL, MVT::i64))); 1112 } 1113 } 1114 1115 return SDValue(); 1116 } 1117 1118 bool RISCVTargetLowering::isDesirableToCommuteWithShift( 1119 const SDNode *N, CombineLevel Level) const { 1120 // The following folds are only desirable if `(OP _, c1 << c2)` can be 1121 // materialised in fewer instructions than `(OP _, c1)`: 1122 // 1123 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2) 1124 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2) 1125 SDValue N0 = N->getOperand(0); 1126 EVT Ty = N0.getValueType(); 1127 if (Ty.isScalarInteger() && 1128 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) { 1129 auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 1130 auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1131 if (C1 && C2) { 1132 APInt C1Int = C1->getAPIntValue(); 1133 APInt ShiftedC1Int = C1Int << C2->getAPIntValue(); 1134 1135 // We can materialise `c1 << c2` into an add immediate, so it's "free", 1136 // and the combine should happen, to potentially allow further combines 1137 // later. 1138 if (ShiftedC1Int.getMinSignedBits() <= 64 && 1139 isLegalAddImmediate(ShiftedC1Int.getSExtValue())) 1140 return true; 1141 1142 // We can materialise `c1` in an add immediate, so it's "free", and the 1143 // combine should be prevented. 1144 if (C1Int.getMinSignedBits() <= 64 && 1145 isLegalAddImmediate(C1Int.getSExtValue())) 1146 return false; 1147 1148 // Neither constant will fit into an immediate, so find materialisation 1149 // costs. 1150 int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(), 1151 Subtarget.is64Bit()); 1152 int ShiftedC1Cost = RISCVMatInt::getIntMatCost( 1153 ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit()); 1154 1155 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the 1156 // combine should be prevented. 1157 if (C1Cost < ShiftedC1Cost) 1158 return false; 1159 } 1160 } 1161 return true; 1162 } 1163 1164 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode( 1165 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, 1166 unsigned Depth) const { 1167 switch (Op.getOpcode()) { 1168 default: 1169 break; 1170 case RISCVISD::SLLW: 1171 case RISCVISD::SRAW: 1172 case RISCVISD::SRLW: 1173 case RISCVISD::DIVW: 1174 case RISCVISD::DIVUW: 1175 case RISCVISD::REMUW: 1176 // TODO: As the result is sign-extended, this is conservatively correct. A 1177 // more precise answer could be calculated for SRAW depending on known 1178 // bits in the shift amount. 1179 return 33; 1180 } 1181 1182 return 1; 1183 } 1184 1185 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI, 1186 MachineBasicBlock *BB) { 1187 assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction"); 1188 1189 // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves. 1190 // Should the count have wrapped while it was being read, we need to try 1191 // again. 1192 // ... 1193 // read: 1194 // rdcycleh x3 # load high word of cycle 1195 // rdcycle x2 # load low word of cycle 1196 // rdcycleh x4 # load high word of cycle 1197 // bne x3, x4, read # check if high word reads match, otherwise try again 1198 // ... 1199 1200 MachineFunction &MF = *BB->getParent(); 1201 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 1202 MachineFunction::iterator It = ++BB->getIterator(); 1203 1204 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB); 1205 MF.insert(It, LoopMBB); 1206 1207 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB); 1208 MF.insert(It, DoneMBB); 1209 1210 // Transfer the remainder of BB and its successor edges to DoneMBB. 1211 DoneMBB->splice(DoneMBB->begin(), BB, 1212 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 1213 DoneMBB->transferSuccessorsAndUpdatePHIs(BB); 1214 1215 BB->addSuccessor(LoopMBB); 1216 1217 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 1218 Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass); 1219 Register LoReg = MI.getOperand(0).getReg(); 1220 Register HiReg = MI.getOperand(1).getReg(); 1221 DebugLoc DL = MI.getDebugLoc(); 1222 1223 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 1224 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg) 1225 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding) 1226 .addReg(RISCV::X0); 1227 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg) 1228 .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding) 1229 .addReg(RISCV::X0); 1230 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg) 1231 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding) 1232 .addReg(RISCV::X0); 1233 1234 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE)) 1235 .addReg(HiReg) 1236 .addReg(ReadAgainReg) 1237 .addMBB(LoopMBB); 1238 1239 LoopMBB->addSuccessor(LoopMBB); 1240 LoopMBB->addSuccessor(DoneMBB); 1241 1242 MI.eraseFromParent(); 1243 1244 return DoneMBB; 1245 } 1246 1247 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI, 1248 MachineBasicBlock *BB) { 1249 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction"); 1250 1251 MachineFunction &MF = *BB->getParent(); 1252 DebugLoc DL = MI.getDebugLoc(); 1253 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); 1254 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo(); 1255 Register LoReg = MI.getOperand(0).getReg(); 1256 Register HiReg = MI.getOperand(1).getReg(); 1257 Register SrcReg = MI.getOperand(2).getReg(); 1258 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass; 1259 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF); 1260 1261 TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC, 1262 RI); 1263 MachineMemOperand *MMO = 1264 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), 1265 MachineMemOperand::MOLoad, 8, Align(8)); 1266 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg) 1267 .addFrameIndex(FI) 1268 .addImm(0) 1269 .addMemOperand(MMO); 1270 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg) 1271 .addFrameIndex(FI) 1272 .addImm(4) 1273 .addMemOperand(MMO); 1274 MI.eraseFromParent(); // The pseudo instruction is gone now. 1275 return BB; 1276 } 1277 1278 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI, 1279 MachineBasicBlock *BB) { 1280 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo && 1281 "Unexpected instruction"); 1282 1283 MachineFunction &MF = *BB->getParent(); 1284 DebugLoc DL = MI.getDebugLoc(); 1285 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); 1286 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo(); 1287 Register DstReg = MI.getOperand(0).getReg(); 1288 Register LoReg = MI.getOperand(1).getReg(); 1289 Register HiReg = MI.getOperand(2).getReg(); 1290 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass; 1291 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF); 1292 1293 MachineMemOperand *MMO = 1294 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), 1295 MachineMemOperand::MOStore, 8, Align(8)); 1296 BuildMI(*BB, MI, DL, TII.get(RISCV::SW)) 1297 .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill())) 1298 .addFrameIndex(FI) 1299 .addImm(0) 1300 .addMemOperand(MMO); 1301 BuildMI(*BB, MI, DL, TII.get(RISCV::SW)) 1302 .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill())) 1303 .addFrameIndex(FI) 1304 .addImm(4) 1305 .addMemOperand(MMO); 1306 TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI); 1307 MI.eraseFromParent(); // The pseudo instruction is gone now. 1308 return BB; 1309 } 1310 1311 static bool isSelectPseudo(MachineInstr &MI) { 1312 switch (MI.getOpcode()) { 1313 default: 1314 return false; 1315 case RISCV::Select_GPR_Using_CC_GPR: 1316 case RISCV::Select_FPR32_Using_CC_GPR: 1317 case RISCV::Select_FPR64_Using_CC_GPR: 1318 return true; 1319 } 1320 } 1321 1322 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI, 1323 MachineBasicBlock *BB) { 1324 // To "insert" Select_* instructions, we actually have to insert the triangle 1325 // control-flow pattern. The incoming instructions know the destination vreg 1326 // to set, the condition code register to branch on, the true/false values to 1327 // select between, and the condcode to use to select the appropriate branch. 1328 // 1329 // We produce the following control flow: 1330 // HeadMBB 1331 // | \ 1332 // | IfFalseMBB 1333 // | / 1334 // TailMBB 1335 // 1336 // When we find a sequence of selects we attempt to optimize their emission 1337 // by sharing the control flow. Currently we only handle cases where we have 1338 // multiple selects with the exact same condition (same LHS, RHS and CC). 1339 // The selects may be interleaved with other instructions if the other 1340 // instructions meet some requirements we deem safe: 1341 // - They are debug instructions. Otherwise, 1342 // - They do not have side-effects, do not access memory and their inputs do 1343 // not depend on the results of the select pseudo-instructions. 1344 // The TrueV/FalseV operands of the selects cannot depend on the result of 1345 // previous selects in the sequence. 1346 // These conditions could be further relaxed. See the X86 target for a 1347 // related approach and more information. 1348 Register LHS = MI.getOperand(1).getReg(); 1349 Register RHS = MI.getOperand(2).getReg(); 1350 auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm()); 1351 1352 SmallVector<MachineInstr *, 4> SelectDebugValues; 1353 SmallSet<Register, 4> SelectDests; 1354 SelectDests.insert(MI.getOperand(0).getReg()); 1355 1356 MachineInstr *LastSelectPseudo = &MI; 1357 1358 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI); 1359 SequenceMBBI != E; ++SequenceMBBI) { 1360 if (SequenceMBBI->isDebugInstr()) 1361 continue; 1362 else if (isSelectPseudo(*SequenceMBBI)) { 1363 if (SequenceMBBI->getOperand(1).getReg() != LHS || 1364 SequenceMBBI->getOperand(2).getReg() != RHS || 1365 SequenceMBBI->getOperand(3).getImm() != CC || 1366 SelectDests.count(SequenceMBBI->getOperand(4).getReg()) || 1367 SelectDests.count(SequenceMBBI->getOperand(5).getReg())) 1368 break; 1369 LastSelectPseudo = &*SequenceMBBI; 1370 SequenceMBBI->collectDebugValues(SelectDebugValues); 1371 SelectDests.insert(SequenceMBBI->getOperand(0).getReg()); 1372 } else { 1373 if (SequenceMBBI->hasUnmodeledSideEffects() || 1374 SequenceMBBI->mayLoadOrStore()) 1375 break; 1376 if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) { 1377 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg()); 1378 })) 1379 break; 1380 } 1381 } 1382 1383 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo(); 1384 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 1385 DebugLoc DL = MI.getDebugLoc(); 1386 MachineFunction::iterator I = ++BB->getIterator(); 1387 1388 MachineBasicBlock *HeadMBB = BB; 1389 MachineFunction *F = BB->getParent(); 1390 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB); 1391 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB); 1392 1393 F->insert(I, IfFalseMBB); 1394 F->insert(I, TailMBB); 1395 1396 // Transfer debug instructions associated with the selects to TailMBB. 1397 for (MachineInstr *DebugInstr : SelectDebugValues) { 1398 TailMBB->push_back(DebugInstr->removeFromParent()); 1399 } 1400 1401 // Move all instructions after the sequence to TailMBB. 1402 TailMBB->splice(TailMBB->end(), HeadMBB, 1403 std::next(LastSelectPseudo->getIterator()), HeadMBB->end()); 1404 // Update machine-CFG edges by transferring all successors of the current 1405 // block to the new block which will contain the Phi nodes for the selects. 1406 TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB); 1407 // Set the successors for HeadMBB. 1408 HeadMBB->addSuccessor(IfFalseMBB); 1409 HeadMBB->addSuccessor(TailMBB); 1410 1411 // Insert appropriate branch. 1412 unsigned Opcode = getBranchOpcodeForIntCondCode(CC); 1413 1414 BuildMI(HeadMBB, DL, TII.get(Opcode)) 1415 .addReg(LHS) 1416 .addReg(RHS) 1417 .addMBB(TailMBB); 1418 1419 // IfFalseMBB just falls through to TailMBB. 1420 IfFalseMBB->addSuccessor(TailMBB); 1421 1422 // Create PHIs for all of the select pseudo-instructions. 1423 auto SelectMBBI = MI.getIterator(); 1424 auto SelectEnd = std::next(LastSelectPseudo->getIterator()); 1425 auto InsertionPoint = TailMBB->begin(); 1426 while (SelectMBBI != SelectEnd) { 1427 auto Next = std::next(SelectMBBI); 1428 if (isSelectPseudo(*SelectMBBI)) { 1429 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ] 1430 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(), 1431 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg()) 1432 .addReg(SelectMBBI->getOperand(4).getReg()) 1433 .addMBB(HeadMBB) 1434 .addReg(SelectMBBI->getOperand(5).getReg()) 1435 .addMBB(IfFalseMBB); 1436 SelectMBBI->eraseFromParent(); 1437 } 1438 SelectMBBI = Next; 1439 } 1440 1441 F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs); 1442 return TailMBB; 1443 } 1444 1445 MachineBasicBlock * 1446 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 1447 MachineBasicBlock *BB) const { 1448 switch (MI.getOpcode()) { 1449 default: 1450 llvm_unreachable("Unexpected instr type to insert"); 1451 case RISCV::ReadCycleWide: 1452 assert(!Subtarget.is64Bit() && 1453 "ReadCycleWrite is only to be used on riscv32"); 1454 return emitReadCycleWidePseudo(MI, BB); 1455 case RISCV::Select_GPR_Using_CC_GPR: 1456 case RISCV::Select_FPR32_Using_CC_GPR: 1457 case RISCV::Select_FPR64_Using_CC_GPR: 1458 return emitSelectPseudo(MI, BB); 1459 case RISCV::BuildPairF64Pseudo: 1460 return emitBuildPairF64Pseudo(MI, BB); 1461 case RISCV::SplitF64Pseudo: 1462 return emitSplitF64Pseudo(MI, BB); 1463 } 1464 } 1465 1466 // Calling Convention Implementation. 1467 // The expectations for frontend ABI lowering vary from target to target. 1468 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI 1469 // details, but this is a longer term goal. For now, we simply try to keep the 1470 // role of the frontend as simple and well-defined as possible. The rules can 1471 // be summarised as: 1472 // * Never split up large scalar arguments. We handle them here. 1473 // * If a hardfloat calling convention is being used, and the struct may be 1474 // passed in a pair of registers (fp+fp, int+fp), and both registers are 1475 // available, then pass as two separate arguments. If either the GPRs or FPRs 1476 // are exhausted, then pass according to the rule below. 1477 // * If a struct could never be passed in registers or directly in a stack 1478 // slot (as it is larger than 2*XLEN and the floating point rules don't 1479 // apply), then pass it using a pointer with the byval attribute. 1480 // * If a struct is less than 2*XLEN, then coerce to either a two-element 1481 // word-sized array or a 2*XLEN scalar (depending on alignment). 1482 // * The frontend can determine whether a struct is returned by reference or 1483 // not based on its size and fields. If it will be returned by reference, the 1484 // frontend must modify the prototype so a pointer with the sret annotation is 1485 // passed as the first argument. This is not necessary for large scalar 1486 // returns. 1487 // * Struct return values and varargs should be coerced to structs containing 1488 // register-size fields in the same situations they would be for fixed 1489 // arguments. 1490 1491 static const MCPhysReg ArgGPRs[] = { 1492 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, 1493 RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17 1494 }; 1495 static const MCPhysReg ArgFPR32s[] = { 1496 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, 1497 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F 1498 }; 1499 static const MCPhysReg ArgFPR64s[] = { 1500 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, 1501 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D 1502 }; 1503 1504 // Pass a 2*XLEN argument that has been split into two XLEN values through 1505 // registers or the stack as necessary. 1506 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, 1507 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, 1508 MVT ValVT2, MVT LocVT2, 1509 ISD::ArgFlagsTy ArgFlags2) { 1510 unsigned XLenInBytes = XLen / 8; 1511 if (Register Reg = State.AllocateReg(ArgGPRs)) { 1512 // At least one half can be passed via register. 1513 State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg, 1514 VA1.getLocVT(), CCValAssign::Full)); 1515 } else { 1516 // Both halves must be passed on the stack, with proper alignment. 1517 Align StackAlign = 1518 std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign()); 1519 State.addLoc( 1520 CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(), 1521 State.AllocateStack(XLenInBytes, StackAlign), 1522 VA1.getLocVT(), CCValAssign::Full)); 1523 State.addLoc(CCValAssign::getMem( 1524 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)), 1525 LocVT2, CCValAssign::Full)); 1526 return false; 1527 } 1528 1529 if (Register Reg = State.AllocateReg(ArgGPRs)) { 1530 // The second half can also be passed via register. 1531 State.addLoc( 1532 CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full)); 1533 } else { 1534 // The second half is passed via the stack, without additional alignment. 1535 State.addLoc(CCValAssign::getMem( 1536 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)), 1537 LocVT2, CCValAssign::Full)); 1538 } 1539 1540 return false; 1541 } 1542 1543 // Implements the RISC-V calling convention. Returns true upon failure. 1544 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, 1545 MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, 1546 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, 1547 bool IsRet, Type *OrigTy) { 1548 unsigned XLen = DL.getLargestLegalIntTypeSizeInBits(); 1549 assert(XLen == 32 || XLen == 64); 1550 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64; 1551 1552 // Any return value split in to more than two values can't be returned 1553 // directly. 1554 if (IsRet && ValNo > 1) 1555 return true; 1556 1557 // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a 1558 // variadic argument, or if no F32 argument registers are available. 1559 bool UseGPRForF32 = true; 1560 // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a 1561 // variadic argument, or if no F64 argument registers are available. 1562 bool UseGPRForF64 = true; 1563 1564 switch (ABI) { 1565 default: 1566 llvm_unreachable("Unexpected ABI"); 1567 case RISCVABI::ABI_ILP32: 1568 case RISCVABI::ABI_LP64: 1569 break; 1570 case RISCVABI::ABI_ILP32F: 1571 case RISCVABI::ABI_LP64F: 1572 UseGPRForF32 = !IsFixed; 1573 break; 1574 case RISCVABI::ABI_ILP32D: 1575 case RISCVABI::ABI_LP64D: 1576 UseGPRForF32 = !IsFixed; 1577 UseGPRForF64 = !IsFixed; 1578 break; 1579 } 1580 1581 if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) 1582 UseGPRForF32 = true; 1583 if (State.getFirstUnallocated(ArgFPR64s) == array_lengthof(ArgFPR64s)) 1584 UseGPRForF64 = true; 1585 1586 // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local 1587 // variables rather than directly checking against the target ABI. 1588 1589 if (UseGPRForF32 && ValVT == MVT::f32) { 1590 LocVT = XLenVT; 1591 LocInfo = CCValAssign::BCvt; 1592 } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) { 1593 LocVT = MVT::i64; 1594 LocInfo = CCValAssign::BCvt; 1595 } 1596 1597 // If this is a variadic argument, the RISC-V calling convention requires 1598 // that it is assigned an 'even' or 'aligned' register if it has 8-byte 1599 // alignment (RV32) or 16-byte alignment (RV64). An aligned register should 1600 // be used regardless of whether the original argument was split during 1601 // legalisation or not. The argument will not be passed by registers if the 1602 // original type is larger than 2*XLEN, so the register alignment rule does 1603 // not apply. 1604 unsigned TwoXLenInBytes = (2 * XLen) / 8; 1605 if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes && 1606 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) { 1607 unsigned RegIdx = State.getFirstUnallocated(ArgGPRs); 1608 // Skip 'odd' register if necessary. 1609 if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1) 1610 State.AllocateReg(ArgGPRs); 1611 } 1612 1613 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs(); 1614 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags = 1615 State.getPendingArgFlags(); 1616 1617 assert(PendingLocs.size() == PendingArgFlags.size() && 1618 "PendingLocs and PendingArgFlags out of sync"); 1619 1620 // Handle passing f64 on RV32D with a soft float ABI or when floating point 1621 // registers are exhausted. 1622 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) { 1623 assert(!ArgFlags.isSplit() && PendingLocs.empty() && 1624 "Can't lower f64 if it is split"); 1625 // Depending on available argument GPRS, f64 may be passed in a pair of 1626 // GPRs, split between a GPR and the stack, or passed completely on the 1627 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these 1628 // cases. 1629 Register Reg = State.AllocateReg(ArgGPRs); 1630 LocVT = MVT::i32; 1631 if (!Reg) { 1632 unsigned StackOffset = State.AllocateStack(8, Align(8)); 1633 State.addLoc( 1634 CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo)); 1635 return false; 1636 } 1637 if (!State.AllocateReg(ArgGPRs)) 1638 State.AllocateStack(4, Align(4)); 1639 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); 1640 return false; 1641 } 1642 1643 // Split arguments might be passed indirectly, so keep track of the pending 1644 // values. 1645 if (ArgFlags.isSplit() || !PendingLocs.empty()) { 1646 LocVT = XLenVT; 1647 LocInfo = CCValAssign::Indirect; 1648 PendingLocs.push_back( 1649 CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); 1650 PendingArgFlags.push_back(ArgFlags); 1651 if (!ArgFlags.isSplitEnd()) { 1652 return false; 1653 } 1654 } 1655 1656 // If the split argument only had two elements, it should be passed directly 1657 // in registers or on the stack. 1658 if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) { 1659 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()"); 1660 // Apply the normal calling convention rules to the first half of the 1661 // split argument. 1662 CCValAssign VA = PendingLocs[0]; 1663 ISD::ArgFlagsTy AF = PendingArgFlags[0]; 1664 PendingLocs.clear(); 1665 PendingArgFlags.clear(); 1666 return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT, 1667 ArgFlags); 1668 } 1669 1670 // Allocate to a register if possible, or else a stack slot. 1671 Register Reg; 1672 if (ValVT == MVT::f32 && !UseGPRForF32) 1673 Reg = State.AllocateReg(ArgFPR32s, ArgFPR64s); 1674 else if (ValVT == MVT::f64 && !UseGPRForF64) 1675 Reg = State.AllocateReg(ArgFPR64s, ArgFPR32s); 1676 else 1677 Reg = State.AllocateReg(ArgGPRs); 1678 unsigned StackOffset = 1679 Reg ? 0 : State.AllocateStack(XLen / 8, Align(XLen / 8)); 1680 1681 // If we reach this point and PendingLocs is non-empty, we must be at the 1682 // end of a split argument that must be passed indirectly. 1683 if (!PendingLocs.empty()) { 1684 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()"); 1685 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()"); 1686 1687 for (auto &It : PendingLocs) { 1688 if (Reg) 1689 It.convertToReg(Reg); 1690 else 1691 It.convertToMem(StackOffset); 1692 State.addLoc(It); 1693 } 1694 PendingLocs.clear(); 1695 PendingArgFlags.clear(); 1696 return false; 1697 } 1698 1699 assert((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) && 1700 "Expected an XLenVT at this stage"); 1701 1702 if (Reg) { 1703 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); 1704 return false; 1705 } 1706 1707 // When an f32 or f64 is passed on the stack, no bit-conversion is needed. 1708 if (ValVT == MVT::f32 || ValVT == MVT::f64) { 1709 LocVT = ValVT; 1710 LocInfo = CCValAssign::Full; 1711 } 1712 State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo)); 1713 return false; 1714 } 1715 1716 void RISCVTargetLowering::analyzeInputArgs( 1717 MachineFunction &MF, CCState &CCInfo, 1718 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const { 1719 unsigned NumArgs = Ins.size(); 1720 FunctionType *FType = MF.getFunction().getFunctionType(); 1721 1722 for (unsigned i = 0; i != NumArgs; ++i) { 1723 MVT ArgVT = Ins[i].VT; 1724 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags; 1725 1726 Type *ArgTy = nullptr; 1727 if (IsRet) 1728 ArgTy = FType->getReturnType(); 1729 else if (Ins[i].isOrigArg()) 1730 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex()); 1731 1732 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI(); 1733 if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full, 1734 ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy)) { 1735 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " 1736 << EVT(ArgVT).getEVTString() << '\n'); 1737 llvm_unreachable(nullptr); 1738 } 1739 } 1740 } 1741 1742 void RISCVTargetLowering::analyzeOutputArgs( 1743 MachineFunction &MF, CCState &CCInfo, 1744 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet, 1745 CallLoweringInfo *CLI) const { 1746 unsigned NumArgs = Outs.size(); 1747 1748 for (unsigned i = 0; i != NumArgs; i++) { 1749 MVT ArgVT = Outs[i].VT; 1750 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 1751 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr; 1752 1753 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI(); 1754 if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full, 1755 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) { 1756 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " 1757 << EVT(ArgVT).getEVTString() << "\n"); 1758 llvm_unreachable(nullptr); 1759 } 1760 } 1761 } 1762 1763 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect 1764 // values. 1765 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, 1766 const CCValAssign &VA, const SDLoc &DL) { 1767 switch (VA.getLocInfo()) { 1768 default: 1769 llvm_unreachable("Unexpected CCValAssign::LocInfo"); 1770 case CCValAssign::Full: 1771 break; 1772 case CCValAssign::BCvt: 1773 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) { 1774 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val); 1775 break; 1776 } 1777 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); 1778 break; 1779 } 1780 return Val; 1781 } 1782 1783 // The caller is responsible for loading the full value if the argument is 1784 // passed with CCValAssign::Indirect. 1785 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain, 1786 const CCValAssign &VA, const SDLoc &DL) { 1787 MachineFunction &MF = DAG.getMachineFunction(); 1788 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 1789 EVT LocVT = VA.getLocVT(); 1790 SDValue Val; 1791 const TargetRegisterClass *RC; 1792 1793 switch (LocVT.getSimpleVT().SimpleTy) { 1794 default: 1795 llvm_unreachable("Unexpected register type"); 1796 case MVT::i32: 1797 case MVT::i64: 1798 RC = &RISCV::GPRRegClass; 1799 break; 1800 case MVT::f32: 1801 RC = &RISCV::FPR32RegClass; 1802 break; 1803 case MVT::f64: 1804 RC = &RISCV::FPR64RegClass; 1805 break; 1806 } 1807 1808 Register VReg = RegInfo.createVirtualRegister(RC); 1809 RegInfo.addLiveIn(VA.getLocReg(), VReg); 1810 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 1811 1812 if (VA.getLocInfo() == CCValAssign::Indirect) 1813 return Val; 1814 1815 return convertLocVTToValVT(DAG, Val, VA, DL); 1816 } 1817 1818 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, 1819 const CCValAssign &VA, const SDLoc &DL) { 1820 EVT LocVT = VA.getLocVT(); 1821 1822 switch (VA.getLocInfo()) { 1823 default: 1824 llvm_unreachable("Unexpected CCValAssign::LocInfo"); 1825 case CCValAssign::Full: 1826 break; 1827 case CCValAssign::BCvt: 1828 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) { 1829 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val); 1830 break; 1831 } 1832 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); 1833 break; 1834 } 1835 return Val; 1836 } 1837 1838 // The caller is responsible for loading the full value if the argument is 1839 // passed with CCValAssign::Indirect. 1840 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, 1841 const CCValAssign &VA, const SDLoc &DL) { 1842 MachineFunction &MF = DAG.getMachineFunction(); 1843 MachineFrameInfo &MFI = MF.getFrameInfo(); 1844 EVT LocVT = VA.getLocVT(); 1845 EVT ValVT = VA.getValVT(); 1846 EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0)); 1847 int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8, 1848 VA.getLocMemOffset(), /*Immutable=*/true); 1849 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 1850 SDValue Val; 1851 1852 ISD::LoadExtType ExtType; 1853 switch (VA.getLocInfo()) { 1854 default: 1855 llvm_unreachable("Unexpected CCValAssign::LocInfo"); 1856 case CCValAssign::Full: 1857 case CCValAssign::Indirect: 1858 case CCValAssign::BCvt: 1859 ExtType = ISD::NON_EXTLOAD; 1860 break; 1861 } 1862 Val = DAG.getExtLoad( 1863 ExtType, DL, LocVT, Chain, FIN, 1864 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT); 1865 return Val; 1866 } 1867 1868 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, 1869 const CCValAssign &VA, const SDLoc &DL) { 1870 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 && 1871 "Unexpected VA"); 1872 MachineFunction &MF = DAG.getMachineFunction(); 1873 MachineFrameInfo &MFI = MF.getFrameInfo(); 1874 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 1875 1876 if (VA.isMemLoc()) { 1877 // f64 is passed on the stack. 1878 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true); 1879 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); 1880 return DAG.getLoad(MVT::f64, DL, Chain, FIN, 1881 MachinePointerInfo::getFixedStack(MF, FI)); 1882 } 1883 1884 assert(VA.isRegLoc() && "Expected register VA assignment"); 1885 1886 Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass); 1887 RegInfo.addLiveIn(VA.getLocReg(), LoVReg); 1888 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32); 1889 SDValue Hi; 1890 if (VA.getLocReg() == RISCV::X17) { 1891 // Second half of f64 is passed on the stack. 1892 int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true); 1893 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); 1894 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN, 1895 MachinePointerInfo::getFixedStack(MF, FI)); 1896 } else { 1897 // Second half of f64 is passed in another GPR. 1898 Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass); 1899 RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg); 1900 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32); 1901 } 1902 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi); 1903 } 1904 1905 // FastCC has less than 1% performance improvement for some particular 1906 // benchmark. But theoretically, it may has benenfit for some cases. 1907 static bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT, 1908 CCValAssign::LocInfo LocInfo, 1909 ISD::ArgFlagsTy ArgFlags, CCState &State) { 1910 1911 if (LocVT == MVT::i32 || LocVT == MVT::i64) { 1912 // X5 and X6 might be used for save-restore libcall. 1913 static const MCPhysReg GPRList[] = { 1914 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14, 1915 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28, 1916 RISCV::X29, RISCV::X30, RISCV::X31}; 1917 if (unsigned Reg = State.AllocateReg(GPRList)) { 1918 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); 1919 return false; 1920 } 1921 } 1922 1923 if (LocVT == MVT::f32) { 1924 static const MCPhysReg FPR32List[] = { 1925 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F, 1926 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F, 1927 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F, 1928 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F}; 1929 if (unsigned Reg = State.AllocateReg(FPR32List)) { 1930 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); 1931 return false; 1932 } 1933 } 1934 1935 if (LocVT == MVT::f64) { 1936 static const MCPhysReg FPR64List[] = { 1937 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D, 1938 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D, 1939 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D, 1940 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D}; 1941 if (unsigned Reg = State.AllocateReg(FPR64List)) { 1942 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); 1943 return false; 1944 } 1945 } 1946 1947 if (LocVT == MVT::i32 || LocVT == MVT::f32) { 1948 unsigned Offset4 = State.AllocateStack(4, Align(4)); 1949 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo)); 1950 return false; 1951 } 1952 1953 if (LocVT == MVT::i64 || LocVT == MVT::f64) { 1954 unsigned Offset5 = State.AllocateStack(8, Align(8)); 1955 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo)); 1956 return false; 1957 } 1958 1959 return true; // CC didn't match. 1960 } 1961 1962 // Transform physical registers into virtual registers. 1963 SDValue RISCVTargetLowering::LowerFormalArguments( 1964 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 1965 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 1966 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 1967 1968 switch (CallConv) { 1969 default: 1970 report_fatal_error("Unsupported calling convention"); 1971 case CallingConv::C: 1972 case CallingConv::Fast: 1973 break; 1974 } 1975 1976 MachineFunction &MF = DAG.getMachineFunction(); 1977 1978 const Function &Func = MF.getFunction(); 1979 if (Func.hasFnAttribute("interrupt")) { 1980 if (!Func.arg_empty()) 1981 report_fatal_error( 1982 "Functions with the interrupt attribute cannot have arguments!"); 1983 1984 StringRef Kind = 1985 MF.getFunction().getFnAttribute("interrupt").getValueAsString(); 1986 1987 if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine")) 1988 report_fatal_error( 1989 "Function interrupt attribute argument not supported!"); 1990 } 1991 1992 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 1993 MVT XLenVT = Subtarget.getXLenVT(); 1994 unsigned XLenInBytes = Subtarget.getXLen() / 8; 1995 // Used with vargs to acumulate store chains. 1996 std::vector<SDValue> OutChains; 1997 1998 // Assign locations to all of the incoming arguments. 1999 SmallVector<CCValAssign, 16> ArgLocs; 2000 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 2001 2002 if (CallConv == CallingConv::Fast) 2003 CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_FastCC); 2004 else 2005 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false); 2006 2007 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 2008 CCValAssign &VA = ArgLocs[i]; 2009 SDValue ArgValue; 2010 // Passing f64 on RV32D with a soft float ABI must be handled as a special 2011 // case. 2012 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) 2013 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL); 2014 else if (VA.isRegLoc()) 2015 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL); 2016 else 2017 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL); 2018 2019 if (VA.getLocInfo() == CCValAssign::Indirect) { 2020 // If the original argument was split and passed by reference (e.g. i128 2021 // on RV32), we need to load all parts of it here (using the same 2022 // address). 2023 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, 2024 MachinePointerInfo())); 2025 unsigned ArgIndex = Ins[i].OrigArgIndex; 2026 assert(Ins[i].PartOffset == 0); 2027 while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) { 2028 CCValAssign &PartVA = ArgLocs[i + 1]; 2029 unsigned PartOffset = Ins[i + 1].PartOffset; 2030 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, 2031 DAG.getIntPtrConstant(PartOffset, DL)); 2032 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, 2033 MachinePointerInfo())); 2034 ++i; 2035 } 2036 continue; 2037 } 2038 InVals.push_back(ArgValue); 2039 } 2040 2041 if (IsVarArg) { 2042 ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs); 2043 unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs); 2044 const TargetRegisterClass *RC = &RISCV::GPRRegClass; 2045 MachineFrameInfo &MFI = MF.getFrameInfo(); 2046 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 2047 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); 2048 2049 // Offset of the first variable argument from stack pointer, and size of 2050 // the vararg save area. For now, the varargs save area is either zero or 2051 // large enough to hold a0-a7. 2052 int VaArgOffset, VarArgsSaveSize; 2053 2054 // If all registers are allocated, then all varargs must be passed on the 2055 // stack and we don't need to save any argregs. 2056 if (ArgRegs.size() == Idx) { 2057 VaArgOffset = CCInfo.getNextStackOffset(); 2058 VarArgsSaveSize = 0; 2059 } else { 2060 VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx); 2061 VaArgOffset = -VarArgsSaveSize; 2062 } 2063 2064 // Record the frame index of the first variable argument 2065 // which is a value necessary to VASTART. 2066 int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true); 2067 RVFI->setVarArgsFrameIndex(FI); 2068 2069 // If saving an odd number of registers then create an extra stack slot to 2070 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures 2071 // offsets to even-numbered registered remain 2*XLEN-aligned. 2072 if (Idx % 2) { 2073 MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true); 2074 VarArgsSaveSize += XLenInBytes; 2075 } 2076 2077 // Copy the integer registers that may have been used for passing varargs 2078 // to the vararg save area. 2079 for (unsigned I = Idx; I < ArgRegs.size(); 2080 ++I, VaArgOffset += XLenInBytes) { 2081 const Register Reg = RegInfo.createVirtualRegister(RC); 2082 RegInfo.addLiveIn(ArgRegs[I], Reg); 2083 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT); 2084 FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true); 2085 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 2086 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff, 2087 MachinePointerInfo::getFixedStack(MF, FI)); 2088 cast<StoreSDNode>(Store.getNode()) 2089 ->getMemOperand() 2090 ->setValue((Value *)nullptr); 2091 OutChains.push_back(Store); 2092 } 2093 RVFI->setVarArgsSaveSize(VarArgsSaveSize); 2094 } 2095 2096 // All stores are grouped in one node to allow the matching between 2097 // the size of Ins and InVals. This only happens for vararg functions. 2098 if (!OutChains.empty()) { 2099 OutChains.push_back(Chain); 2100 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains); 2101 } 2102 2103 return Chain; 2104 } 2105 2106 /// isEligibleForTailCallOptimization - Check whether the call is eligible 2107 /// for tail call optimization. 2108 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization. 2109 bool RISCVTargetLowering::isEligibleForTailCallOptimization( 2110 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF, 2111 const SmallVector<CCValAssign, 16> &ArgLocs) const { 2112 2113 auto &Callee = CLI.Callee; 2114 auto CalleeCC = CLI.CallConv; 2115 auto &Outs = CLI.Outs; 2116 auto &Caller = MF.getFunction(); 2117 auto CallerCC = Caller.getCallingConv(); 2118 2119 // Exception-handling functions need a special set of instructions to 2120 // indicate a return to the hardware. Tail-calling another function would 2121 // probably break this. 2122 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This 2123 // should be expanded as new function attributes are introduced. 2124 if (Caller.hasFnAttribute("interrupt")) 2125 return false; 2126 2127 // Do not tail call opt if the stack is used to pass parameters. 2128 if (CCInfo.getNextStackOffset() != 0) 2129 return false; 2130 2131 // Do not tail call opt if any parameters need to be passed indirectly. 2132 // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are 2133 // passed indirectly. So the address of the value will be passed in a 2134 // register, or if not available, then the address is put on the stack. In 2135 // order to pass indirectly, space on the stack often needs to be allocated 2136 // in order to store the value. In this case the CCInfo.getNextStackOffset() 2137 // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs 2138 // are passed CCValAssign::Indirect. 2139 for (auto &VA : ArgLocs) 2140 if (VA.getLocInfo() == CCValAssign::Indirect) 2141 return false; 2142 2143 // Do not tail call opt if either caller or callee uses struct return 2144 // semantics. 2145 auto IsCallerStructRet = Caller.hasStructRetAttr(); 2146 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet(); 2147 if (IsCallerStructRet || IsCalleeStructRet) 2148 return false; 2149 2150 // Externally-defined functions with weak linkage should not be 2151 // tail-called. The behaviour of branch instructions in this situation (as 2152 // used for tail calls) is implementation-defined, so we cannot rely on the 2153 // linker replacing the tail call with a return. 2154 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 2155 const GlobalValue *GV = G->getGlobal(); 2156 if (GV->hasExternalWeakLinkage()) 2157 return false; 2158 } 2159 2160 // The callee has to preserve all registers the caller needs to preserve. 2161 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo(); 2162 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); 2163 if (CalleeCC != CallerCC) { 2164 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); 2165 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) 2166 return false; 2167 } 2168 2169 // Byval parameters hand the function a pointer directly into the stack area 2170 // we want to reuse during a tail call. Working around this *is* possible 2171 // but less efficient and uglier in LowerCall. 2172 for (auto &Arg : Outs) 2173 if (Arg.Flags.isByVal()) 2174 return false; 2175 2176 return true; 2177 } 2178 2179 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input 2180 // and output parameter nodes. 2181 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, 2182 SmallVectorImpl<SDValue> &InVals) const { 2183 SelectionDAG &DAG = CLI.DAG; 2184 SDLoc &DL = CLI.DL; 2185 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 2186 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 2187 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 2188 SDValue Chain = CLI.Chain; 2189 SDValue Callee = CLI.Callee; 2190 bool &IsTailCall = CLI.IsTailCall; 2191 CallingConv::ID CallConv = CLI.CallConv; 2192 bool IsVarArg = CLI.IsVarArg; 2193 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2194 MVT XLenVT = Subtarget.getXLenVT(); 2195 2196 MachineFunction &MF = DAG.getMachineFunction(); 2197 2198 // Analyze the operands of the call, assigning locations to each operand. 2199 SmallVector<CCValAssign, 16> ArgLocs; 2200 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 2201 2202 if (CallConv == CallingConv::Fast) 2203 ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_FastCC); 2204 else 2205 analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI); 2206 2207 // Check if it's really possible to do a tail call. 2208 if (IsTailCall) 2209 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs); 2210 2211 if (IsTailCall) 2212 ++NumTailCalls; 2213 else if (CLI.CB && CLI.CB->isMustTailCall()) 2214 report_fatal_error("failed to perform tail call elimination on a call " 2215 "site marked musttail"); 2216 2217 // Get a count of how many bytes are to be pushed on the stack. 2218 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 2219 2220 // Create local copies for byval args 2221 SmallVector<SDValue, 8> ByValArgs; 2222 for (unsigned i = 0, e = Outs.size(); i != e; ++i) { 2223 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2224 if (!Flags.isByVal()) 2225 continue; 2226 2227 SDValue Arg = OutVals[i]; 2228 unsigned Size = Flags.getByValSize(); 2229 Align Alignment = Flags.getNonZeroByValAlign(); 2230 2231 int FI = 2232 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false); 2233 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 2234 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT); 2235 2236 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment, 2237 /*IsVolatile=*/false, 2238 /*AlwaysInline=*/false, IsTailCall, 2239 MachinePointerInfo(), MachinePointerInfo()); 2240 ByValArgs.push_back(FIPtr); 2241 } 2242 2243 if (!IsTailCall) 2244 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL); 2245 2246 // Copy argument values to their designated locations. 2247 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass; 2248 SmallVector<SDValue, 8> MemOpChains; 2249 SDValue StackPtr; 2250 for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) { 2251 CCValAssign &VA = ArgLocs[i]; 2252 SDValue ArgValue = OutVals[i]; 2253 ISD::ArgFlagsTy Flags = Outs[i].Flags; 2254 2255 // Handle passing f64 on RV32D with a soft float ABI as a special case. 2256 bool IsF64OnRV32DSoftABI = 2257 VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64; 2258 if (IsF64OnRV32DSoftABI && VA.isRegLoc()) { 2259 SDValue SplitF64 = DAG.getNode( 2260 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue); 2261 SDValue Lo = SplitF64.getValue(0); 2262 SDValue Hi = SplitF64.getValue(1); 2263 2264 Register RegLo = VA.getLocReg(); 2265 RegsToPass.push_back(std::make_pair(RegLo, Lo)); 2266 2267 if (RegLo == RISCV::X17) { 2268 // Second half of f64 is passed on the stack. 2269 // Work out the address of the stack slot. 2270 if (!StackPtr.getNode()) 2271 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT); 2272 // Emit the store. 2273 MemOpChains.push_back( 2274 DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo())); 2275 } else { 2276 // Second half of f64 is passed in another GPR. 2277 assert(RegLo < RISCV::X31 && "Invalid register pair"); 2278 Register RegHigh = RegLo + 1; 2279 RegsToPass.push_back(std::make_pair(RegHigh, Hi)); 2280 } 2281 continue; 2282 } 2283 2284 // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way 2285 // as any other MemLoc. 2286 2287 // Promote the value if needed. 2288 // For now, only handle fully promoted and indirect arguments. 2289 if (VA.getLocInfo() == CCValAssign::Indirect) { 2290 // Store the argument in a stack slot and pass its address. 2291 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT); 2292 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 2293 MemOpChains.push_back( 2294 DAG.getStore(Chain, DL, ArgValue, SpillSlot, 2295 MachinePointerInfo::getFixedStack(MF, FI))); 2296 // If the original argument was split (e.g. i128), we need 2297 // to store all parts of it here (and pass just one address). 2298 unsigned ArgIndex = Outs[i].OrigArgIndex; 2299 assert(Outs[i].PartOffset == 0); 2300 while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) { 2301 SDValue PartValue = OutVals[i + 1]; 2302 unsigned PartOffset = Outs[i + 1].PartOffset; 2303 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, 2304 DAG.getIntPtrConstant(PartOffset, DL)); 2305 MemOpChains.push_back( 2306 DAG.getStore(Chain, DL, PartValue, Address, 2307 MachinePointerInfo::getFixedStack(MF, FI))); 2308 ++i; 2309 } 2310 ArgValue = SpillSlot; 2311 } else { 2312 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL); 2313 } 2314 2315 // Use local copy if it is a byval arg. 2316 if (Flags.isByVal()) 2317 ArgValue = ByValArgs[j++]; 2318 2319 if (VA.isRegLoc()) { 2320 // Queue up the argument copies and emit them at the end. 2321 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 2322 } else { 2323 assert(VA.isMemLoc() && "Argument not register or memory"); 2324 assert(!IsTailCall && "Tail call not allowed if stack is used " 2325 "for passing parameters"); 2326 2327 // Work out the address of the stack slot. 2328 if (!StackPtr.getNode()) 2329 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT); 2330 SDValue Address = 2331 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 2332 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL)); 2333 2334 // Emit the store. 2335 MemOpChains.push_back( 2336 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo())); 2337 } 2338 } 2339 2340 // Join the stores, which are independent of one another. 2341 if (!MemOpChains.empty()) 2342 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 2343 2344 SDValue Glue; 2345 2346 // Build a sequence of copy-to-reg nodes, chained and glued together. 2347 for (auto &Reg : RegsToPass) { 2348 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue); 2349 Glue = Chain.getValue(1); 2350 } 2351 2352 // Validate that none of the argument registers have been marked as 2353 // reserved, if so report an error. Do the same for the return address if this 2354 // is not a tailcall. 2355 validateCCReservedRegs(RegsToPass, MF); 2356 if (!IsTailCall && 2357 MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1)) 2358 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ 2359 MF.getFunction(), 2360 "Return address register required, but has been reserved."}); 2361 2362 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a 2363 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't 2364 // split it and then direct call can be matched by PseudoCALL. 2365 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) { 2366 const GlobalValue *GV = S->getGlobal(); 2367 2368 unsigned OpFlags = RISCVII::MO_CALL; 2369 if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) 2370 OpFlags = RISCVII::MO_PLT; 2371 2372 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags); 2373 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 2374 unsigned OpFlags = RISCVII::MO_CALL; 2375 2376 if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(), 2377 nullptr)) 2378 OpFlags = RISCVII::MO_PLT; 2379 2380 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags); 2381 } 2382 2383 // The first call operand is the chain and the second is the target address. 2384 SmallVector<SDValue, 8> Ops; 2385 Ops.push_back(Chain); 2386 Ops.push_back(Callee); 2387 2388 // Add argument registers to the end of the list so that they are 2389 // known live into the call. 2390 for (auto &Reg : RegsToPass) 2391 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType())); 2392 2393 if (!IsTailCall) { 2394 // Add a register mask operand representing the call-preserved registers. 2395 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 2396 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); 2397 assert(Mask && "Missing call preserved mask for calling convention"); 2398 Ops.push_back(DAG.getRegisterMask(Mask)); 2399 } 2400 2401 // Glue the call to the argument copies, if any. 2402 if (Glue.getNode()) 2403 Ops.push_back(Glue); 2404 2405 // Emit the call. 2406 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2407 2408 if (IsTailCall) { 2409 MF.getFrameInfo().setHasTailCall(); 2410 return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops); 2411 } 2412 2413 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops); 2414 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); 2415 Glue = Chain.getValue(1); 2416 2417 // Mark the end of the call, which is glued to the call itself. 2418 Chain = DAG.getCALLSEQ_END(Chain, 2419 DAG.getConstant(NumBytes, DL, PtrVT, true), 2420 DAG.getConstant(0, DL, PtrVT, true), 2421 Glue, DL); 2422 Glue = Chain.getValue(1); 2423 2424 // Assign locations to each value returned by this call. 2425 SmallVector<CCValAssign, 16> RVLocs; 2426 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext()); 2427 analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true); 2428 2429 // Copy all of the result registers out of their specified physreg. 2430 for (auto &VA : RVLocs) { 2431 // Copy the value out 2432 SDValue RetValue = 2433 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue); 2434 // Glue the RetValue to the end of the call sequence 2435 Chain = RetValue.getValue(1); 2436 Glue = RetValue.getValue(2); 2437 2438 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) { 2439 assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment"); 2440 SDValue RetValue2 = 2441 DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue); 2442 Chain = RetValue2.getValue(1); 2443 Glue = RetValue2.getValue(2); 2444 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue, 2445 RetValue2); 2446 } 2447 2448 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL); 2449 2450 InVals.push_back(RetValue); 2451 } 2452 2453 return Chain; 2454 } 2455 2456 bool RISCVTargetLowering::CanLowerReturn( 2457 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, 2458 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { 2459 SmallVector<CCValAssign, 16> RVLocs; 2460 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); 2461 for (unsigned i = 0, e = Outs.size(); i != e; ++i) { 2462 MVT VT = Outs[i].VT; 2463 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 2464 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI(); 2465 if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full, 2466 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr)) 2467 return false; 2468 } 2469 return true; 2470 } 2471 2472 SDValue 2473 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 2474 bool IsVarArg, 2475 const SmallVectorImpl<ISD::OutputArg> &Outs, 2476 const SmallVectorImpl<SDValue> &OutVals, 2477 const SDLoc &DL, SelectionDAG &DAG) const { 2478 const MachineFunction &MF = DAG.getMachineFunction(); 2479 const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>(); 2480 2481 // Stores the assignment of the return value to a location. 2482 SmallVector<CCValAssign, 16> RVLocs; 2483 2484 // Info about the registers and stack slot. 2485 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, 2486 *DAG.getContext()); 2487 2488 analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true, 2489 nullptr); 2490 2491 SDValue Glue; 2492 SmallVector<SDValue, 4> RetOps(1, Chain); 2493 2494 // Copy the result values into the output registers. 2495 for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) { 2496 SDValue Val = OutVals[i]; 2497 CCValAssign &VA = RVLocs[i]; 2498 assert(VA.isRegLoc() && "Can only return in registers!"); 2499 2500 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) { 2501 // Handle returning f64 on RV32D with a soft float ABI. 2502 assert(VA.isRegLoc() && "Expected return via registers"); 2503 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL, 2504 DAG.getVTList(MVT::i32, MVT::i32), Val); 2505 SDValue Lo = SplitF64.getValue(0); 2506 SDValue Hi = SplitF64.getValue(1); 2507 Register RegLo = VA.getLocReg(); 2508 assert(RegLo < RISCV::X31 && "Invalid register pair"); 2509 Register RegHi = RegLo + 1; 2510 2511 if (STI.isRegisterReservedByUser(RegLo) || 2512 STI.isRegisterReservedByUser(RegHi)) 2513 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ 2514 MF.getFunction(), 2515 "Return value register required, but has been reserved."}); 2516 2517 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue); 2518 Glue = Chain.getValue(1); 2519 RetOps.push_back(DAG.getRegister(RegLo, MVT::i32)); 2520 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue); 2521 Glue = Chain.getValue(1); 2522 RetOps.push_back(DAG.getRegister(RegHi, MVT::i32)); 2523 } else { 2524 // Handle a 'normal' return. 2525 Val = convertValVTToLocVT(DAG, Val, VA, DL); 2526 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue); 2527 2528 if (STI.isRegisterReservedByUser(VA.getLocReg())) 2529 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ 2530 MF.getFunction(), 2531 "Return value register required, but has been reserved."}); 2532 2533 // Guarantee that all emitted copies are stuck together. 2534 Glue = Chain.getValue(1); 2535 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 2536 } 2537 } 2538 2539 RetOps[0] = Chain; // Update chain. 2540 2541 // Add the glue node if we have it. 2542 if (Glue.getNode()) { 2543 RetOps.push_back(Glue); 2544 } 2545 2546 // Interrupt service routines use different return instructions. 2547 const Function &Func = DAG.getMachineFunction().getFunction(); 2548 if (Func.hasFnAttribute("interrupt")) { 2549 if (!Func.getReturnType()->isVoidTy()) 2550 report_fatal_error( 2551 "Functions with the interrupt attribute must have void return type!"); 2552 2553 MachineFunction &MF = DAG.getMachineFunction(); 2554 StringRef Kind = 2555 MF.getFunction().getFnAttribute("interrupt").getValueAsString(); 2556 2557 unsigned RetOpc; 2558 if (Kind == "user") 2559 RetOpc = RISCVISD::URET_FLAG; 2560 else if (Kind == "supervisor") 2561 RetOpc = RISCVISD::SRET_FLAG; 2562 else 2563 RetOpc = RISCVISD::MRET_FLAG; 2564 2565 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps); 2566 } 2567 2568 return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps); 2569 } 2570 2571 void RISCVTargetLowering::validateCCReservedRegs( 2572 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs, 2573 MachineFunction &MF) const { 2574 const Function &F = MF.getFunction(); 2575 const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>(); 2576 2577 if (std::any_of(std::begin(Regs), std::end(Regs), [&STI](auto Reg) { 2578 return STI.isRegisterReservedByUser(Reg.first); 2579 })) 2580 F.getContext().diagnose(DiagnosticInfoUnsupported{ 2581 F, "Argument register required, but has been reserved."}); 2582 } 2583 2584 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 2585 return CI->isTailCall(); 2586 } 2587 2588 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const { 2589 switch ((RISCVISD::NodeType)Opcode) { 2590 case RISCVISD::FIRST_NUMBER: 2591 break; 2592 case RISCVISD::RET_FLAG: 2593 return "RISCVISD::RET_FLAG"; 2594 case RISCVISD::URET_FLAG: 2595 return "RISCVISD::URET_FLAG"; 2596 case RISCVISD::SRET_FLAG: 2597 return "RISCVISD::SRET_FLAG"; 2598 case RISCVISD::MRET_FLAG: 2599 return "RISCVISD::MRET_FLAG"; 2600 case RISCVISD::CALL: 2601 return "RISCVISD::CALL"; 2602 case RISCVISD::SELECT_CC: 2603 return "RISCVISD::SELECT_CC"; 2604 case RISCVISD::BuildPairF64: 2605 return "RISCVISD::BuildPairF64"; 2606 case RISCVISD::SplitF64: 2607 return "RISCVISD::SplitF64"; 2608 case RISCVISD::TAIL: 2609 return "RISCVISD::TAIL"; 2610 case RISCVISD::SLLW: 2611 return "RISCVISD::SLLW"; 2612 case RISCVISD::SRAW: 2613 return "RISCVISD::SRAW"; 2614 case RISCVISD::SRLW: 2615 return "RISCVISD::SRLW"; 2616 case RISCVISD::DIVW: 2617 return "RISCVISD::DIVW"; 2618 case RISCVISD::DIVUW: 2619 return "RISCVISD::DIVUW"; 2620 case RISCVISD::REMUW: 2621 return "RISCVISD::REMUW"; 2622 case RISCVISD::FMV_W_X_RV64: 2623 return "RISCVISD::FMV_W_X_RV64"; 2624 case RISCVISD::FMV_X_ANYEXTW_RV64: 2625 return "RISCVISD::FMV_X_ANYEXTW_RV64"; 2626 case RISCVISD::READ_CYCLE_WIDE: 2627 return "RISCVISD::READ_CYCLE_WIDE"; 2628 } 2629 return nullptr; 2630 } 2631 2632 /// getConstraintType - Given a constraint letter, return the type of 2633 /// constraint it is for this target. 2634 RISCVTargetLowering::ConstraintType 2635 RISCVTargetLowering::getConstraintType(StringRef Constraint) const { 2636 if (Constraint.size() == 1) { 2637 switch (Constraint[0]) { 2638 default: 2639 break; 2640 case 'f': 2641 return C_RegisterClass; 2642 case 'I': 2643 case 'J': 2644 case 'K': 2645 return C_Immediate; 2646 case 'A': 2647 return C_Memory; 2648 } 2649 } 2650 return TargetLowering::getConstraintType(Constraint); 2651 } 2652 2653 std::pair<unsigned, const TargetRegisterClass *> 2654 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 2655 StringRef Constraint, 2656 MVT VT) const { 2657 // First, see if this is a constraint that directly corresponds to a 2658 // RISCV register class. 2659 if (Constraint.size() == 1) { 2660 switch (Constraint[0]) { 2661 case 'r': 2662 return std::make_pair(0U, &RISCV::GPRRegClass); 2663 case 'f': 2664 if (Subtarget.hasStdExtF() && VT == MVT::f32) 2665 return std::make_pair(0U, &RISCV::FPR32RegClass); 2666 if (Subtarget.hasStdExtD() && VT == MVT::f64) 2667 return std::make_pair(0U, &RISCV::FPR64RegClass); 2668 break; 2669 default: 2670 break; 2671 } 2672 } 2673 2674 // Clang will correctly decode the usage of register name aliases into their 2675 // official names. However, other frontends like `rustc` do not. This allows 2676 // users of these frontends to use the ABI names for registers in LLVM-style 2677 // register constraints. 2678 Register XRegFromAlias = StringSwitch<Register>(Constraint.lower()) 2679 .Case("{zero}", RISCV::X0) 2680 .Case("{ra}", RISCV::X1) 2681 .Case("{sp}", RISCV::X2) 2682 .Case("{gp}", RISCV::X3) 2683 .Case("{tp}", RISCV::X4) 2684 .Case("{t0}", RISCV::X5) 2685 .Case("{t1}", RISCV::X6) 2686 .Case("{t2}", RISCV::X7) 2687 .Cases("{s0}", "{fp}", RISCV::X8) 2688 .Case("{s1}", RISCV::X9) 2689 .Case("{a0}", RISCV::X10) 2690 .Case("{a1}", RISCV::X11) 2691 .Case("{a2}", RISCV::X12) 2692 .Case("{a3}", RISCV::X13) 2693 .Case("{a4}", RISCV::X14) 2694 .Case("{a5}", RISCV::X15) 2695 .Case("{a6}", RISCV::X16) 2696 .Case("{a7}", RISCV::X17) 2697 .Case("{s2}", RISCV::X18) 2698 .Case("{s3}", RISCV::X19) 2699 .Case("{s4}", RISCV::X20) 2700 .Case("{s5}", RISCV::X21) 2701 .Case("{s6}", RISCV::X22) 2702 .Case("{s7}", RISCV::X23) 2703 .Case("{s8}", RISCV::X24) 2704 .Case("{s9}", RISCV::X25) 2705 .Case("{s10}", RISCV::X26) 2706 .Case("{s11}", RISCV::X27) 2707 .Case("{t3}", RISCV::X28) 2708 .Case("{t4}", RISCV::X29) 2709 .Case("{t5}", RISCV::X30) 2710 .Case("{t6}", RISCV::X31) 2711 .Default(RISCV::NoRegister); 2712 if (XRegFromAlias != RISCV::NoRegister) 2713 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass); 2714 2715 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the 2716 // TableGen record rather than the AsmName to choose registers for InlineAsm 2717 // constraints, plus we want to match those names to the widest floating point 2718 // register type available, manually select floating point registers here. 2719 // 2720 // The second case is the ABI name of the register, so that frontends can also 2721 // use the ABI names in register constraint lists. 2722 if (Subtarget.hasStdExtF() || Subtarget.hasStdExtD()) { 2723 std::pair<Register, Register> FReg = 2724 StringSwitch<std::pair<Register, Register>>(Constraint.lower()) 2725 .Cases("{f0}", "{ft0}", {RISCV::F0_F, RISCV::F0_D}) 2726 .Cases("{f1}", "{ft1}", {RISCV::F1_F, RISCV::F1_D}) 2727 .Cases("{f2}", "{ft2}", {RISCV::F2_F, RISCV::F2_D}) 2728 .Cases("{f3}", "{ft3}", {RISCV::F3_F, RISCV::F3_D}) 2729 .Cases("{f4}", "{ft4}", {RISCV::F4_F, RISCV::F4_D}) 2730 .Cases("{f5}", "{ft5}", {RISCV::F5_F, RISCV::F5_D}) 2731 .Cases("{f6}", "{ft6}", {RISCV::F6_F, RISCV::F6_D}) 2732 .Cases("{f7}", "{ft7}", {RISCV::F7_F, RISCV::F7_D}) 2733 .Cases("{f8}", "{fs0}", {RISCV::F8_F, RISCV::F8_D}) 2734 .Cases("{f9}", "{fs1}", {RISCV::F9_F, RISCV::F9_D}) 2735 .Cases("{f10}", "{fa0}", {RISCV::F10_F, RISCV::F10_D}) 2736 .Cases("{f11}", "{fa1}", {RISCV::F11_F, RISCV::F11_D}) 2737 .Cases("{f12}", "{fa2}", {RISCV::F12_F, RISCV::F12_D}) 2738 .Cases("{f13}", "{fa3}", {RISCV::F13_F, RISCV::F13_D}) 2739 .Cases("{f14}", "{fa4}", {RISCV::F14_F, RISCV::F14_D}) 2740 .Cases("{f15}", "{fa5}", {RISCV::F15_F, RISCV::F15_D}) 2741 .Cases("{f16}", "{fa6}", {RISCV::F16_F, RISCV::F16_D}) 2742 .Cases("{f17}", "{fa7}", {RISCV::F17_F, RISCV::F17_D}) 2743 .Cases("{f18}", "{fs2}", {RISCV::F18_F, RISCV::F18_D}) 2744 .Cases("{f19}", "{fs3}", {RISCV::F19_F, RISCV::F19_D}) 2745 .Cases("{f20}", "{fs4}", {RISCV::F20_F, RISCV::F20_D}) 2746 .Cases("{f21}", "{fs5}", {RISCV::F21_F, RISCV::F21_D}) 2747 .Cases("{f22}", "{fs6}", {RISCV::F22_F, RISCV::F22_D}) 2748 .Cases("{f23}", "{fs7}", {RISCV::F23_F, RISCV::F23_D}) 2749 .Cases("{f24}", "{fs8}", {RISCV::F24_F, RISCV::F24_D}) 2750 .Cases("{f25}", "{fs9}", {RISCV::F25_F, RISCV::F25_D}) 2751 .Cases("{f26}", "{fs10}", {RISCV::F26_F, RISCV::F26_D}) 2752 .Cases("{f27}", "{fs11}", {RISCV::F27_F, RISCV::F27_D}) 2753 .Cases("{f28}", "{ft8}", {RISCV::F28_F, RISCV::F28_D}) 2754 .Cases("{f29}", "{ft9}", {RISCV::F29_F, RISCV::F29_D}) 2755 .Cases("{f30}", "{ft10}", {RISCV::F30_F, RISCV::F30_D}) 2756 .Cases("{f31}", "{ft11}", {RISCV::F31_F, RISCV::F31_D}) 2757 .Default({RISCV::NoRegister, RISCV::NoRegister}); 2758 if (FReg.first != RISCV::NoRegister) 2759 return Subtarget.hasStdExtD() 2760 ? std::make_pair(FReg.second, &RISCV::FPR64RegClass) 2761 : std::make_pair(FReg.first, &RISCV::FPR32RegClass); 2762 } 2763 2764 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 2765 } 2766 2767 unsigned 2768 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const { 2769 // Currently only support length 1 constraints. 2770 if (ConstraintCode.size() == 1) { 2771 switch (ConstraintCode[0]) { 2772 case 'A': 2773 return InlineAsm::Constraint_A; 2774 default: 2775 break; 2776 } 2777 } 2778 2779 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); 2780 } 2781 2782 void RISCVTargetLowering::LowerAsmOperandForConstraint( 2783 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops, 2784 SelectionDAG &DAG) const { 2785 // Currently only support length 1 constraints. 2786 if (Constraint.length() == 1) { 2787 switch (Constraint[0]) { 2788 case 'I': 2789 // Validate & create a 12-bit signed immediate operand. 2790 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 2791 uint64_t CVal = C->getSExtValue(); 2792 if (isInt<12>(CVal)) 2793 Ops.push_back( 2794 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT())); 2795 } 2796 return; 2797 case 'J': 2798 // Validate & create an integer zero operand. 2799 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 2800 if (C->getZExtValue() == 0) 2801 Ops.push_back( 2802 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT())); 2803 return; 2804 case 'K': 2805 // Validate & create a 5-bit unsigned immediate operand. 2806 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 2807 uint64_t CVal = C->getZExtValue(); 2808 if (isUInt<5>(CVal)) 2809 Ops.push_back( 2810 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT())); 2811 } 2812 return; 2813 default: 2814 break; 2815 } 2816 } 2817 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 2818 } 2819 2820 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilder<> &Builder, 2821 Instruction *Inst, 2822 AtomicOrdering Ord) const { 2823 if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent) 2824 return Builder.CreateFence(Ord); 2825 if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord)) 2826 return Builder.CreateFence(AtomicOrdering::Release); 2827 return nullptr; 2828 } 2829 2830 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder, 2831 Instruction *Inst, 2832 AtomicOrdering Ord) const { 2833 if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord)) 2834 return Builder.CreateFence(AtomicOrdering::Acquire); 2835 return nullptr; 2836 } 2837 2838 TargetLowering::AtomicExpansionKind 2839 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { 2840 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating 2841 // point operations can't be used in an lr/sc sequence without breaking the 2842 // forward-progress guarantee. 2843 if (AI->isFloatingPointOperation()) 2844 return AtomicExpansionKind::CmpXChg; 2845 2846 unsigned Size = AI->getType()->getPrimitiveSizeInBits(); 2847 if (Size == 8 || Size == 16) 2848 return AtomicExpansionKind::MaskedIntrinsic; 2849 return AtomicExpansionKind::None; 2850 } 2851 2852 static Intrinsic::ID 2853 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) { 2854 if (XLen == 32) { 2855 switch (BinOp) { 2856 default: 2857 llvm_unreachable("Unexpected AtomicRMW BinOp"); 2858 case AtomicRMWInst::Xchg: 2859 return Intrinsic::riscv_masked_atomicrmw_xchg_i32; 2860 case AtomicRMWInst::Add: 2861 return Intrinsic::riscv_masked_atomicrmw_add_i32; 2862 case AtomicRMWInst::Sub: 2863 return Intrinsic::riscv_masked_atomicrmw_sub_i32; 2864 case AtomicRMWInst::Nand: 2865 return Intrinsic::riscv_masked_atomicrmw_nand_i32; 2866 case AtomicRMWInst::Max: 2867 return Intrinsic::riscv_masked_atomicrmw_max_i32; 2868 case AtomicRMWInst::Min: 2869 return Intrinsic::riscv_masked_atomicrmw_min_i32; 2870 case AtomicRMWInst::UMax: 2871 return Intrinsic::riscv_masked_atomicrmw_umax_i32; 2872 case AtomicRMWInst::UMin: 2873 return Intrinsic::riscv_masked_atomicrmw_umin_i32; 2874 } 2875 } 2876 2877 if (XLen == 64) { 2878 switch (BinOp) { 2879 default: 2880 llvm_unreachable("Unexpected AtomicRMW BinOp"); 2881 case AtomicRMWInst::Xchg: 2882 return Intrinsic::riscv_masked_atomicrmw_xchg_i64; 2883 case AtomicRMWInst::Add: 2884 return Intrinsic::riscv_masked_atomicrmw_add_i64; 2885 case AtomicRMWInst::Sub: 2886 return Intrinsic::riscv_masked_atomicrmw_sub_i64; 2887 case AtomicRMWInst::Nand: 2888 return Intrinsic::riscv_masked_atomicrmw_nand_i64; 2889 case AtomicRMWInst::Max: 2890 return Intrinsic::riscv_masked_atomicrmw_max_i64; 2891 case AtomicRMWInst::Min: 2892 return Intrinsic::riscv_masked_atomicrmw_min_i64; 2893 case AtomicRMWInst::UMax: 2894 return Intrinsic::riscv_masked_atomicrmw_umax_i64; 2895 case AtomicRMWInst::UMin: 2896 return Intrinsic::riscv_masked_atomicrmw_umin_i64; 2897 } 2898 } 2899 2900 llvm_unreachable("Unexpected XLen\n"); 2901 } 2902 2903 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic( 2904 IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, 2905 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const { 2906 unsigned XLen = Subtarget.getXLen(); 2907 Value *Ordering = 2908 Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering())); 2909 Type *Tys[] = {AlignedAddr->getType()}; 2910 Function *LrwOpScwLoop = Intrinsic::getDeclaration( 2911 AI->getModule(), 2912 getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys); 2913 2914 if (XLen == 64) { 2915 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty()); 2916 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty()); 2917 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty()); 2918 } 2919 2920 Value *Result; 2921 2922 // Must pass the shift amount needed to sign extend the loaded value prior 2923 // to performing a signed comparison for min/max. ShiftAmt is the number of 2924 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which 2925 // is the number of bits to left+right shift the value in order to 2926 // sign-extend. 2927 if (AI->getOperation() == AtomicRMWInst::Min || 2928 AI->getOperation() == AtomicRMWInst::Max) { 2929 const DataLayout &DL = AI->getModule()->getDataLayout(); 2930 unsigned ValWidth = 2931 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType()); 2932 Value *SextShamt = 2933 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt); 2934 Result = Builder.CreateCall(LrwOpScwLoop, 2935 {AlignedAddr, Incr, Mask, SextShamt, Ordering}); 2936 } else { 2937 Result = 2938 Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering}); 2939 } 2940 2941 if (XLen == 64) 2942 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty()); 2943 return Result; 2944 } 2945 2946 TargetLowering::AtomicExpansionKind 2947 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR( 2948 AtomicCmpXchgInst *CI) const { 2949 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits(); 2950 if (Size == 8 || Size == 16) 2951 return AtomicExpansionKind::MaskedIntrinsic; 2952 return AtomicExpansionKind::None; 2953 } 2954 2955 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( 2956 IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, 2957 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { 2958 unsigned XLen = Subtarget.getXLen(); 2959 Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord)); 2960 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32; 2961 if (XLen == 64) { 2962 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty()); 2963 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty()); 2964 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty()); 2965 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64; 2966 } 2967 Type *Tys[] = {AlignedAddr->getType()}; 2968 Function *MaskedCmpXchg = 2969 Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys); 2970 Value *Result = Builder.CreateCall( 2971 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering}); 2972 if (XLen == 64) 2973 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty()); 2974 return Result; 2975 } 2976 2977 Register RISCVTargetLowering::getExceptionPointerRegister( 2978 const Constant *PersonalityFn) const { 2979 return RISCV::X10; 2980 } 2981 2982 Register RISCVTargetLowering::getExceptionSelectorRegister( 2983 const Constant *PersonalityFn) const { 2984 return RISCV::X11; 2985 } 2986 2987 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const { 2988 // Return false to suppress the unnecessary extensions if the LibCall 2989 // arguments or return value is f32 type for LP64 ABI. 2990 RISCVABI::ABI ABI = Subtarget.getTargetABI(); 2991 if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32)) 2992 return false; 2993 2994 return true; 2995 } 2996 2997 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, 2998 SDValue C) const { 2999 // Check integral scalar types. 3000 if (VT.isScalarInteger()) { 3001 // Do not perform the transformation on riscv32 with the M extension. 3002 if (!Subtarget.is64Bit() && Subtarget.hasStdExtM()) 3003 return false; 3004 if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) { 3005 if (ConstNode->getAPIntValue().getBitWidth() > 8 * sizeof(int64_t)) 3006 return false; 3007 int64_t Imm = ConstNode->getSExtValue(); 3008 if (isPowerOf2_64(Imm + 1) || isPowerOf2_64(Imm - 1) || 3009 isPowerOf2_64(1 - Imm) || isPowerOf2_64(-1 - Imm)) 3010 return true; 3011 } 3012 } 3013 3014 return false; 3015 } 3016 3017 #define GET_REGISTER_MATCHER 3018 #include "RISCVGenAsmMatcher.inc" 3019 3020 Register 3021 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT, 3022 const MachineFunction &MF) const { 3023 Register Reg = MatchRegisterAltName(RegName); 3024 if (Reg == RISCV::NoRegister) 3025 Reg = MatchRegisterName(RegName); 3026 if (Reg == RISCV::NoRegister) 3027 report_fatal_error( 3028 Twine("Invalid register name \"" + StringRef(RegName) + "\".")); 3029 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF); 3030 if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg)) 3031 report_fatal_error(Twine("Trying to obtain non-reserved register \"" + 3032 StringRef(RegName) + "\".")); 3033 return Reg; 3034 } 3035