1 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the RISCV implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVInstrInfo.h" 14 #include "MCTargetDesc/RISCVMatInt.h" 15 #include "RISCV.h" 16 #include "RISCVMachineFunctionInfo.h" 17 #include "RISCVSubtarget.h" 18 #include "RISCVTargetMachine.h" 19 #include "llvm/ADT/STLExtras.h" 20 #include "llvm/ADT/SmallVector.h" 21 #include "llvm/Analysis/MemoryLocation.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/LiveVariables.h" 24 #include "llvm/CodeGen/MachineFunctionPass.h" 25 #include "llvm/CodeGen/MachineInstrBuilder.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/RegisterScavenging.h" 28 #include "llvm/MC/MCInstBuilder.h" 29 #include "llvm/MC/TargetRegistry.h" 30 #include "llvm/Support/ErrorHandling.h" 31 32 using namespace llvm; 33 34 #define GEN_CHECK_COMPRESS_INSTR 35 #include "RISCVGenCompressInstEmitter.inc" 36 37 #define GET_INSTRINFO_CTOR_DTOR 38 #define GET_INSTRINFO_NAMED_OPS 39 #include "RISCVGenInstrInfo.inc" 40 41 static cl::opt<bool> PreferWholeRegisterMove( 42 "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, 43 cl::desc("Prefer whole register move for vector registers.")); 44 45 namespace llvm { 46 namespace RISCVVPseudosTable { 47 48 using namespace RISCV; 49 50 #define GET_RISCVVPseudosTable_IMPL 51 #include "RISCVGenSearchableTables.inc" 52 53 } // namespace RISCVVPseudosTable 54 } // namespace llvm 55 56 RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI) 57 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP), 58 STI(STI) {} 59 60 MCInst RISCVInstrInfo::getNop() const { 61 if (STI.getFeatureBits()[RISCV::FeatureStdExtC]) 62 return MCInstBuilder(RISCV::C_NOP); 63 return MCInstBuilder(RISCV::ADDI) 64 .addReg(RISCV::X0) 65 .addReg(RISCV::X0) 66 .addImm(0); 67 } 68 69 unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 70 int &FrameIndex) const { 71 switch (MI.getOpcode()) { 72 default: 73 return 0; 74 case RISCV::LB: 75 case RISCV::LBU: 76 case RISCV::LH: 77 case RISCV::LHU: 78 case RISCV::FLH: 79 case RISCV::LW: 80 case RISCV::FLW: 81 case RISCV::LWU: 82 case RISCV::LD: 83 case RISCV::FLD: 84 break; 85 } 86 87 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 88 MI.getOperand(2).getImm() == 0) { 89 FrameIndex = MI.getOperand(1).getIndex(); 90 return MI.getOperand(0).getReg(); 91 } 92 93 return 0; 94 } 95 96 unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 97 int &FrameIndex) const { 98 switch (MI.getOpcode()) { 99 default: 100 return 0; 101 case RISCV::SB: 102 case RISCV::SH: 103 case RISCV::SW: 104 case RISCV::FSH: 105 case RISCV::FSW: 106 case RISCV::SD: 107 case RISCV::FSD: 108 break; 109 } 110 111 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 112 MI.getOperand(2).getImm() == 0) { 113 FrameIndex = MI.getOperand(1).getIndex(); 114 return MI.getOperand(0).getReg(); 115 } 116 117 return 0; 118 } 119 120 static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg, 121 unsigned NumRegs) { 122 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs; 123 } 124 125 static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, 126 const MachineBasicBlock &MBB, 127 MachineBasicBlock::const_iterator MBBI, 128 MachineBasicBlock::const_iterator &DefMBBI, 129 RISCVII::VLMUL LMul) { 130 if (PreferWholeRegisterMove) 131 return false; 132 133 assert(MBBI->getOpcode() == TargetOpcode::COPY && 134 "Unexpected COPY instruction."); 135 Register SrcReg = MBBI->getOperand(1).getReg(); 136 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 137 138 bool FoundDef = false; 139 bool FirstVSetVLI = false; 140 unsigned FirstSEW = 0; 141 while (MBBI != MBB.begin()) { 142 --MBBI; 143 if (MBBI->isMetaInstruction()) 144 continue; 145 146 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI || 147 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 || 148 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) { 149 // There is a vsetvli between COPY and source define instruction. 150 // vy = def_vop ... (producing instruction) 151 // ... 152 // vsetvli 153 // ... 154 // vx = COPY vy 155 if (!FoundDef) { 156 if (!FirstVSetVLI) { 157 FirstVSetVLI = true; 158 unsigned FirstVType = MBBI->getOperand(2).getImm(); 159 RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType); 160 FirstSEW = RISCVVType::getSEW(FirstVType); 161 // The first encountered vsetvli must have the same lmul as the 162 // register class of COPY. 163 if (FirstLMul != LMul) 164 return false; 165 } 166 // Only permit `vsetvli x0, x0, vtype` between COPY and the source 167 // define instruction. 168 if (MBBI->getOperand(0).getReg() != RISCV::X0) 169 return false; 170 if (MBBI->getOperand(1).isImm()) 171 return false; 172 if (MBBI->getOperand(1).getReg() != RISCV::X0) 173 return false; 174 continue; 175 } 176 177 // MBBI is the first vsetvli before the producing instruction. 178 unsigned VType = MBBI->getOperand(2).getImm(); 179 // If there is a vsetvli between COPY and the producing instruction. 180 if (FirstVSetVLI) { 181 // If SEW is different, return false. 182 if (RISCVVType::getSEW(VType) != FirstSEW) 183 return false; 184 } 185 186 // If the vsetvli is tail undisturbed, keep the whole register move. 187 if (!RISCVVType::isTailAgnostic(VType)) 188 return false; 189 190 // The checking is conservative. We only have register classes for 191 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v 192 // for fractional LMUL operations. However, we could not use the vsetvli 193 // lmul for widening operations. The result of widening operation is 194 // 2 x LMUL. 195 return LMul == RISCVVType::getVLMUL(VType); 196 } else if (MBBI->isInlineAsm() || MBBI->isCall()) { 197 return false; 198 } else if (MBBI->getNumDefs()) { 199 // Check all the instructions which will change VL. 200 // For example, vleff has implicit def VL. 201 if (MBBI->modifiesRegister(RISCV::VL)) 202 return false; 203 204 // Only converting whole register copies to vmv.v.v when the defining 205 // value appears in the explicit operands. 206 for (const MachineOperand &MO : MBBI->explicit_operands()) { 207 if (!MO.isReg() || !MO.isDef()) 208 continue; 209 if (!FoundDef && TRI->isSubRegisterEq(MO.getReg(), SrcReg)) { 210 // We only permit the source of COPY has the same LMUL as the defined 211 // operand. 212 // There are cases we need to keep the whole register copy if the LMUL 213 // is different. 214 // For example, 215 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m 216 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2 217 // # The COPY may be created by vlmul_trunc intrinsic. 218 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4 219 // 220 // After widening, the valid value will be 4 x e32 elements. If we 221 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements. 222 // FIXME: The COPY of subregister of Zvlsseg register will not be able 223 // to convert to vmv.v.[v|i] under the constraint. 224 if (MO.getReg() != SrcReg) 225 return false; 226 227 // In widening reduction instructions with LMUL_1 input vector case, 228 // only checking the LMUL is insufficient due to reduction result is 229 // always LMUL_1. 230 // For example, 231 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu 232 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27 233 // $v26 = COPY killed renamable $v8 234 // After widening, The valid value will be 1 x e16 elements. If we 235 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements. 236 uint64_t TSFlags = MBBI->getDesc().TSFlags; 237 if (RISCVII::isRVVWideningReduction(TSFlags)) 238 return false; 239 240 // Found the definition. 241 FoundDef = true; 242 DefMBBI = MBBI; 243 // If the producing instruction does not depend on vsetvli, do not 244 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD. 245 if (!RISCVII::hasSEWOp(TSFlags)) 246 return false; 247 break; 248 } 249 } 250 } 251 } 252 253 return false; 254 } 255 256 void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 257 MachineBasicBlock::iterator MBBI, 258 const DebugLoc &DL, MCRegister DstReg, 259 MCRegister SrcReg, bool KillSrc) const { 260 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) { 261 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg) 262 .addReg(SrcReg, getKillRegState(KillSrc)) 263 .addImm(0); 264 return; 265 } 266 267 // Handle copy from csr 268 if (RISCV::VCSRRegClass.contains(SrcReg) && 269 RISCV::GPRRegClass.contains(DstReg)) { 270 const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); 271 BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg) 272 .addImm(RISCVSysReg::lookupSysRegByName(TRI.getName(SrcReg))->Encoding) 273 .addReg(RISCV::X0); 274 return; 275 } 276 277 // FPR->FPR copies and VR->VR copies. 278 unsigned Opc; 279 bool IsScalableVector = true; 280 unsigned NF = 1; 281 RISCVII::VLMUL LMul = RISCVII::LMUL_1; 282 unsigned SubRegIdx = RISCV::sub_vrm1_0; 283 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) { 284 Opc = RISCV::FSGNJ_H; 285 IsScalableVector = false; 286 } else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) { 287 Opc = RISCV::FSGNJ_S; 288 IsScalableVector = false; 289 } else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) { 290 Opc = RISCV::FSGNJ_D; 291 IsScalableVector = false; 292 } else if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { 293 Opc = RISCV::PseudoVMV1R_V; 294 LMul = RISCVII::LMUL_1; 295 } else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { 296 Opc = RISCV::PseudoVMV2R_V; 297 LMul = RISCVII::LMUL_2; 298 } else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { 299 Opc = RISCV::PseudoVMV4R_V; 300 LMul = RISCVII::LMUL_4; 301 } else if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { 302 Opc = RISCV::PseudoVMV8R_V; 303 LMul = RISCVII::LMUL_8; 304 } else if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { 305 Opc = RISCV::PseudoVMV1R_V; 306 SubRegIdx = RISCV::sub_vrm1_0; 307 NF = 2; 308 LMul = RISCVII::LMUL_1; 309 } else if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { 310 Opc = RISCV::PseudoVMV2R_V; 311 SubRegIdx = RISCV::sub_vrm2_0; 312 NF = 2; 313 LMul = RISCVII::LMUL_2; 314 } else if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { 315 Opc = RISCV::PseudoVMV4R_V; 316 SubRegIdx = RISCV::sub_vrm4_0; 317 NF = 2; 318 LMul = RISCVII::LMUL_4; 319 } else if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { 320 Opc = RISCV::PseudoVMV1R_V; 321 SubRegIdx = RISCV::sub_vrm1_0; 322 NF = 3; 323 LMul = RISCVII::LMUL_1; 324 } else if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { 325 Opc = RISCV::PseudoVMV2R_V; 326 SubRegIdx = RISCV::sub_vrm2_0; 327 NF = 3; 328 LMul = RISCVII::LMUL_2; 329 } else if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { 330 Opc = RISCV::PseudoVMV1R_V; 331 SubRegIdx = RISCV::sub_vrm1_0; 332 NF = 4; 333 LMul = RISCVII::LMUL_1; 334 } else if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { 335 Opc = RISCV::PseudoVMV2R_V; 336 SubRegIdx = RISCV::sub_vrm2_0; 337 NF = 4; 338 LMul = RISCVII::LMUL_2; 339 } else if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { 340 Opc = RISCV::PseudoVMV1R_V; 341 SubRegIdx = RISCV::sub_vrm1_0; 342 NF = 5; 343 LMul = RISCVII::LMUL_1; 344 } else if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { 345 Opc = RISCV::PseudoVMV1R_V; 346 SubRegIdx = RISCV::sub_vrm1_0; 347 NF = 6; 348 LMul = RISCVII::LMUL_1; 349 } else if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { 350 Opc = RISCV::PseudoVMV1R_V; 351 SubRegIdx = RISCV::sub_vrm1_0; 352 NF = 7; 353 LMul = RISCVII::LMUL_1; 354 } else if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { 355 Opc = RISCV::PseudoVMV1R_V; 356 SubRegIdx = RISCV::sub_vrm1_0; 357 NF = 8; 358 LMul = RISCVII::LMUL_1; 359 } else { 360 llvm_unreachable("Impossible reg-to-reg copy"); 361 } 362 363 if (IsScalableVector) { 364 bool UseVMV_V_V = false; 365 MachineBasicBlock::const_iterator DefMBBI; 366 unsigned DefExplicitOpNum; 367 unsigned VIOpc; 368 if (isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) { 369 UseVMV_V_V = true; 370 DefExplicitOpNum = DefMBBI->getNumExplicitOperands(); 371 // We only need to handle LMUL = 1/2/4/8 here because we only define 372 // vector register classes for LMUL = 1/2/4/8. 373 switch (LMul) { 374 default: 375 llvm_unreachable("Impossible LMUL for vector register copy."); 376 case RISCVII::LMUL_1: 377 Opc = RISCV::PseudoVMV_V_V_M1; 378 VIOpc = RISCV::PseudoVMV_V_I_M1; 379 break; 380 case RISCVII::LMUL_2: 381 Opc = RISCV::PseudoVMV_V_V_M2; 382 VIOpc = RISCV::PseudoVMV_V_I_M2; 383 break; 384 case RISCVII::LMUL_4: 385 Opc = RISCV::PseudoVMV_V_V_M4; 386 VIOpc = RISCV::PseudoVMV_V_I_M4; 387 break; 388 case RISCVII::LMUL_8: 389 Opc = RISCV::PseudoVMV_V_V_M8; 390 VIOpc = RISCV::PseudoVMV_V_I_M8; 391 break; 392 } 393 } 394 395 bool UseVMV_V_I = false; 396 if (UseVMV_V_V && (DefMBBI->getOpcode() == VIOpc)) { 397 UseVMV_V_I = true; 398 Opc = VIOpc; 399 } 400 401 if (NF == 1) { 402 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), DstReg); 403 if (UseVMV_V_I) 404 MIB = MIB.add(DefMBBI->getOperand(1)); 405 else 406 MIB = MIB.addReg(SrcReg, getKillRegState(KillSrc)); 407 if (UseVMV_V_V) { 408 // The last two arguments of vector instructions are 409 // AVL, SEW. We also need to append the implicit-use vl and vtype. 410 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 2)); // AVL 411 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 1)); // SEW 412 MIB.addReg(RISCV::VL, RegState::Implicit); 413 MIB.addReg(RISCV::VTYPE, RegState::Implicit); 414 } 415 } else { 416 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 417 418 int I = 0, End = NF, Incr = 1; 419 unsigned SrcEncoding = TRI->getEncodingValue(SrcReg); 420 unsigned DstEncoding = TRI->getEncodingValue(DstReg); 421 unsigned LMulVal; 422 bool Fractional; 423 std::tie(LMulVal, Fractional) = RISCVVType::decodeVLMUL(LMul); 424 assert(!Fractional && "It is impossible be fractional lmul here."); 425 if (forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NF * LMulVal)) { 426 I = NF - 1; 427 End = -1; 428 Incr = -1; 429 } 430 431 for (; I != End; I += Incr) { 432 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), 433 TRI->getSubReg(DstReg, SubRegIdx + I)); 434 if (UseVMV_V_I) 435 MIB = MIB.add(DefMBBI->getOperand(1)); 436 else 437 MIB = MIB.addReg(TRI->getSubReg(SrcReg, SubRegIdx + I), 438 getKillRegState(KillSrc)); 439 if (UseVMV_V_V) { 440 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 2)); // AVL 441 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 1)); // SEW 442 MIB.addReg(RISCV::VL, RegState::Implicit); 443 MIB.addReg(RISCV::VTYPE, RegState::Implicit); 444 } 445 } 446 } 447 } else { 448 BuildMI(MBB, MBBI, DL, get(Opc), DstReg) 449 .addReg(SrcReg, getKillRegState(KillSrc)) 450 .addReg(SrcReg, getKillRegState(KillSrc)); 451 } 452 } 453 454 void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 455 MachineBasicBlock::iterator I, 456 Register SrcReg, bool IsKill, int FI, 457 const TargetRegisterClass *RC, 458 const TargetRegisterInfo *TRI) const { 459 DebugLoc DL; 460 if (I != MBB.end()) 461 DL = I->getDebugLoc(); 462 463 MachineFunction *MF = MBB.getParent(); 464 MachineFrameInfo &MFI = MF->getFrameInfo(); 465 466 unsigned Opcode; 467 bool IsScalableVector = true; 468 bool IsZvlsseg = true; 469 if (RISCV::GPRRegClass.hasSubClassEq(RC)) { 470 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ? 471 RISCV::SW : RISCV::SD; 472 IsScalableVector = false; 473 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) { 474 Opcode = RISCV::FSH; 475 IsScalableVector = false; 476 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) { 477 Opcode = RISCV::FSW; 478 IsScalableVector = false; 479 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) { 480 Opcode = RISCV::FSD; 481 IsScalableVector = false; 482 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) { 483 Opcode = RISCV::PseudoVSPILL_M1; 484 IsZvlsseg = false; 485 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) { 486 Opcode = RISCV::PseudoVSPILL_M2; 487 IsZvlsseg = false; 488 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) { 489 Opcode = RISCV::PseudoVSPILL_M4; 490 IsZvlsseg = false; 491 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) { 492 Opcode = RISCV::PseudoVSPILL_M8; 493 IsZvlsseg = false; 494 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) 495 Opcode = RISCV::PseudoVSPILL2_M1; 496 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC)) 497 Opcode = RISCV::PseudoVSPILL2_M2; 498 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC)) 499 Opcode = RISCV::PseudoVSPILL2_M4; 500 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) 501 Opcode = RISCV::PseudoVSPILL3_M1; 502 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC)) 503 Opcode = RISCV::PseudoVSPILL3_M2; 504 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) 505 Opcode = RISCV::PseudoVSPILL4_M1; 506 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC)) 507 Opcode = RISCV::PseudoVSPILL4_M2; 508 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) 509 Opcode = RISCV::PseudoVSPILL5_M1; 510 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) 511 Opcode = RISCV::PseudoVSPILL6_M1; 512 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) 513 Opcode = RISCV::PseudoVSPILL7_M1; 514 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) 515 Opcode = RISCV::PseudoVSPILL8_M1; 516 else 517 llvm_unreachable("Can't store this register to stack slot"); 518 519 if (IsScalableVector) { 520 MachineMemOperand *MMO = MF->getMachineMemOperand( 521 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 522 MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); 523 524 MFI.setStackID(FI, TargetStackID::ScalableVector); 525 auto MIB = BuildMI(MBB, I, DL, get(Opcode)) 526 .addReg(SrcReg, getKillRegState(IsKill)) 527 .addFrameIndex(FI) 528 .addMemOperand(MMO); 529 if (IsZvlsseg) { 530 // For spilling/reloading Zvlsseg registers, append the dummy field for 531 // the scaled vector length. The argument will be used when expanding 532 // these pseudo instructions. 533 MIB.addReg(RISCV::X0); 534 } 535 } else { 536 MachineMemOperand *MMO = MF->getMachineMemOperand( 537 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 538 MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); 539 540 BuildMI(MBB, I, DL, get(Opcode)) 541 .addReg(SrcReg, getKillRegState(IsKill)) 542 .addFrameIndex(FI) 543 .addImm(0) 544 .addMemOperand(MMO); 545 } 546 } 547 548 void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 549 MachineBasicBlock::iterator I, 550 Register DstReg, int FI, 551 const TargetRegisterClass *RC, 552 const TargetRegisterInfo *TRI) const { 553 DebugLoc DL; 554 if (I != MBB.end()) 555 DL = I->getDebugLoc(); 556 557 MachineFunction *MF = MBB.getParent(); 558 MachineFrameInfo &MFI = MF->getFrameInfo(); 559 560 unsigned Opcode; 561 bool IsScalableVector = true; 562 bool IsZvlsseg = true; 563 if (RISCV::GPRRegClass.hasSubClassEq(RC)) { 564 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ? 565 RISCV::LW : RISCV::LD; 566 IsScalableVector = false; 567 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) { 568 Opcode = RISCV::FLH; 569 IsScalableVector = false; 570 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) { 571 Opcode = RISCV::FLW; 572 IsScalableVector = false; 573 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) { 574 Opcode = RISCV::FLD; 575 IsScalableVector = false; 576 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) { 577 Opcode = RISCV::PseudoVRELOAD_M1; 578 IsZvlsseg = false; 579 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) { 580 Opcode = RISCV::PseudoVRELOAD_M2; 581 IsZvlsseg = false; 582 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) { 583 Opcode = RISCV::PseudoVRELOAD_M4; 584 IsZvlsseg = false; 585 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) { 586 Opcode = RISCV::PseudoVRELOAD_M8; 587 IsZvlsseg = false; 588 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) 589 Opcode = RISCV::PseudoVRELOAD2_M1; 590 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC)) 591 Opcode = RISCV::PseudoVRELOAD2_M2; 592 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC)) 593 Opcode = RISCV::PseudoVRELOAD2_M4; 594 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) 595 Opcode = RISCV::PseudoVRELOAD3_M1; 596 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC)) 597 Opcode = RISCV::PseudoVRELOAD3_M2; 598 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) 599 Opcode = RISCV::PseudoVRELOAD4_M1; 600 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC)) 601 Opcode = RISCV::PseudoVRELOAD4_M2; 602 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) 603 Opcode = RISCV::PseudoVRELOAD5_M1; 604 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) 605 Opcode = RISCV::PseudoVRELOAD6_M1; 606 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) 607 Opcode = RISCV::PseudoVRELOAD7_M1; 608 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) 609 Opcode = RISCV::PseudoVRELOAD8_M1; 610 else 611 llvm_unreachable("Can't load this register from stack slot"); 612 613 if (IsScalableVector) { 614 MachineMemOperand *MMO = MF->getMachineMemOperand( 615 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 616 MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); 617 618 MFI.setStackID(FI, TargetStackID::ScalableVector); 619 auto MIB = BuildMI(MBB, I, DL, get(Opcode), DstReg) 620 .addFrameIndex(FI) 621 .addMemOperand(MMO); 622 if (IsZvlsseg) { 623 // For spilling/reloading Zvlsseg registers, append the dummy field for 624 // the scaled vector length. The argument will be used when expanding 625 // these pseudo instructions. 626 MIB.addReg(RISCV::X0); 627 } 628 } else { 629 MachineMemOperand *MMO = MF->getMachineMemOperand( 630 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 631 MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); 632 633 BuildMI(MBB, I, DL, get(Opcode), DstReg) 634 .addFrameIndex(FI) 635 .addImm(0) 636 .addMemOperand(MMO); 637 } 638 } 639 640 void RISCVInstrInfo::movImm(MachineBasicBlock &MBB, 641 MachineBasicBlock::iterator MBBI, 642 const DebugLoc &DL, Register DstReg, uint64_t Val, 643 MachineInstr::MIFlag Flag) const { 644 Register SrcReg = RISCV::X0; 645 646 if (!STI.is64Bit() && !isInt<32>(Val)) 647 report_fatal_error("Should only materialize 32-bit constants for RV32"); 648 649 RISCVMatInt::InstSeq Seq = 650 RISCVMatInt::generateInstSeq(Val, STI.getFeatureBits()); 651 assert(!Seq.empty()); 652 653 for (RISCVMatInt::Inst &Inst : Seq) { 654 switch (Inst.getOpndKind()) { 655 case RISCVMatInt::Imm: 656 BuildMI(MBB, MBBI, DL, get(Inst.Opc), DstReg) 657 .addImm(Inst.Imm) 658 .setMIFlag(Flag); 659 break; 660 case RISCVMatInt::RegX0: 661 BuildMI(MBB, MBBI, DL, get(Inst.Opc), DstReg) 662 .addReg(SrcReg, RegState::Kill) 663 .addReg(RISCV::X0) 664 .setMIFlag(Flag); 665 break; 666 case RISCVMatInt::RegReg: 667 BuildMI(MBB, MBBI, DL, get(Inst.Opc), DstReg) 668 .addReg(SrcReg, RegState::Kill) 669 .addReg(SrcReg, RegState::Kill) 670 .setMIFlag(Flag); 671 break; 672 case RISCVMatInt::RegImm: 673 BuildMI(MBB, MBBI, DL, get(Inst.Opc), DstReg) 674 .addReg(SrcReg, RegState::Kill) 675 .addImm(Inst.Imm) 676 .setMIFlag(Flag); 677 break; 678 } 679 680 // Only the first instruction has X0 as its source. 681 SrcReg = DstReg; 682 } 683 } 684 685 static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) { 686 switch (Opc) { 687 default: 688 return RISCVCC::COND_INVALID; 689 case RISCV::BEQ: 690 return RISCVCC::COND_EQ; 691 case RISCV::BNE: 692 return RISCVCC::COND_NE; 693 case RISCV::BLT: 694 return RISCVCC::COND_LT; 695 case RISCV::BGE: 696 return RISCVCC::COND_GE; 697 case RISCV::BLTU: 698 return RISCVCC::COND_LTU; 699 case RISCV::BGEU: 700 return RISCVCC::COND_GEU; 701 } 702 } 703 704 // The contents of values added to Cond are not examined outside of 705 // RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we 706 // push BranchOpcode, Reg1, Reg2. 707 static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target, 708 SmallVectorImpl<MachineOperand> &Cond) { 709 // Block ends with fall-through condbranch. 710 assert(LastInst.getDesc().isConditionalBranch() && 711 "Unknown conditional branch"); 712 Target = LastInst.getOperand(2).getMBB(); 713 unsigned CC = getCondFromBranchOpc(LastInst.getOpcode()); 714 Cond.push_back(MachineOperand::CreateImm(CC)); 715 Cond.push_back(LastInst.getOperand(0)); 716 Cond.push_back(LastInst.getOperand(1)); 717 } 718 719 const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const { 720 switch (CC) { 721 default: 722 llvm_unreachable("Unknown condition code!"); 723 case RISCVCC::COND_EQ: 724 return get(RISCV::BEQ); 725 case RISCVCC::COND_NE: 726 return get(RISCV::BNE); 727 case RISCVCC::COND_LT: 728 return get(RISCV::BLT); 729 case RISCVCC::COND_GE: 730 return get(RISCV::BGE); 731 case RISCVCC::COND_LTU: 732 return get(RISCV::BLTU); 733 case RISCVCC::COND_GEU: 734 return get(RISCV::BGEU); 735 } 736 } 737 738 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) { 739 switch (CC) { 740 default: 741 llvm_unreachable("Unrecognized conditional branch"); 742 case RISCVCC::COND_EQ: 743 return RISCVCC::COND_NE; 744 case RISCVCC::COND_NE: 745 return RISCVCC::COND_EQ; 746 case RISCVCC::COND_LT: 747 return RISCVCC::COND_GE; 748 case RISCVCC::COND_GE: 749 return RISCVCC::COND_LT; 750 case RISCVCC::COND_LTU: 751 return RISCVCC::COND_GEU; 752 case RISCVCC::COND_GEU: 753 return RISCVCC::COND_LTU; 754 } 755 } 756 757 bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 758 MachineBasicBlock *&TBB, 759 MachineBasicBlock *&FBB, 760 SmallVectorImpl<MachineOperand> &Cond, 761 bool AllowModify) const { 762 TBB = FBB = nullptr; 763 Cond.clear(); 764 765 // If the block has no terminators, it just falls into the block after it. 766 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 767 if (I == MBB.end() || !isUnpredicatedTerminator(*I)) 768 return false; 769 770 // Count the number of terminators and find the first unconditional or 771 // indirect branch. 772 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end(); 773 int NumTerminators = 0; 774 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J); 775 J++) { 776 NumTerminators++; 777 if (J->getDesc().isUnconditionalBranch() || 778 J->getDesc().isIndirectBranch()) { 779 FirstUncondOrIndirectBr = J.getReverse(); 780 } 781 } 782 783 // If AllowModify is true, we can erase any terminators after 784 // FirstUncondOrIndirectBR. 785 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) { 786 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) { 787 std::next(FirstUncondOrIndirectBr)->eraseFromParent(); 788 NumTerminators--; 789 } 790 I = FirstUncondOrIndirectBr; 791 } 792 793 // We can't handle blocks that end in an indirect branch. 794 if (I->getDesc().isIndirectBranch()) 795 return true; 796 797 // We can't handle blocks with more than 2 terminators. 798 if (NumTerminators > 2) 799 return true; 800 801 // Handle a single unconditional branch. 802 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) { 803 TBB = getBranchDestBlock(*I); 804 return false; 805 } 806 807 // Handle a single conditional branch. 808 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) { 809 parseCondBranch(*I, TBB, Cond); 810 return false; 811 } 812 813 // Handle a conditional branch followed by an unconditional branch. 814 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() && 815 I->getDesc().isUnconditionalBranch()) { 816 parseCondBranch(*std::prev(I), TBB, Cond); 817 FBB = getBranchDestBlock(*I); 818 return false; 819 } 820 821 // Otherwise, we can't handle this. 822 return true; 823 } 824 825 unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB, 826 int *BytesRemoved) const { 827 if (BytesRemoved) 828 *BytesRemoved = 0; 829 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 830 if (I == MBB.end()) 831 return 0; 832 833 if (!I->getDesc().isUnconditionalBranch() && 834 !I->getDesc().isConditionalBranch()) 835 return 0; 836 837 // Remove the branch. 838 if (BytesRemoved) 839 *BytesRemoved += getInstSizeInBytes(*I); 840 I->eraseFromParent(); 841 842 I = MBB.end(); 843 844 if (I == MBB.begin()) 845 return 1; 846 --I; 847 if (!I->getDesc().isConditionalBranch()) 848 return 1; 849 850 // Remove the branch. 851 if (BytesRemoved) 852 *BytesRemoved += getInstSizeInBytes(*I); 853 I->eraseFromParent(); 854 return 2; 855 } 856 857 // Inserts a branch into the end of the specific MachineBasicBlock, returning 858 // the number of instructions inserted. 859 unsigned RISCVInstrInfo::insertBranch( 860 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, 861 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const { 862 if (BytesAdded) 863 *BytesAdded = 0; 864 865 // Shouldn't be a fall through. 866 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 867 assert((Cond.size() == 3 || Cond.size() == 0) && 868 "RISCV branch conditions have two components!"); 869 870 // Unconditional branch. 871 if (Cond.empty()) { 872 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB); 873 if (BytesAdded) 874 *BytesAdded += getInstSizeInBytes(MI); 875 return 1; 876 } 877 878 // Either a one or two-way conditional branch. 879 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm()); 880 MachineInstr &CondMI = 881 *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB); 882 if (BytesAdded) 883 *BytesAdded += getInstSizeInBytes(CondMI); 884 885 // One-way conditional branch. 886 if (!FBB) 887 return 1; 888 889 // Two-way conditional branch. 890 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB); 891 if (BytesAdded) 892 *BytesAdded += getInstSizeInBytes(MI); 893 return 2; 894 } 895 896 void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 897 MachineBasicBlock &DestBB, 898 MachineBasicBlock &RestoreBB, 899 const DebugLoc &DL, int64_t BrOffset, 900 RegScavenger *RS) const { 901 assert(RS && "RegScavenger required for long branching"); 902 assert(MBB.empty() && 903 "new block should be inserted for expanding unconditional branch"); 904 assert(MBB.pred_size() == 1); 905 906 MachineFunction *MF = MBB.getParent(); 907 MachineRegisterInfo &MRI = MF->getRegInfo(); 908 909 if (!isInt<32>(BrOffset)) 910 report_fatal_error( 911 "Branch offsets outside of the signed 32-bit range not supported"); 912 913 // FIXME: A virtual register must be used initially, as the register 914 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch 915 // uses the same workaround). 916 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass); 917 auto II = MBB.end(); 918 919 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump)) 920 .addReg(ScratchReg, RegState::Define | RegState::Dead) 921 .addMBB(&DestBB, RISCVII::MO_CALL); 922 923 RS->enterBasicBlockEnd(MBB); 924 Register Scav = RS->scavengeRegisterBackwards(RISCV::GPRRegClass, 925 MI.getIterator(), false, 0); 926 // TODO: The case when there is no scavenged register needs special handling. 927 assert(Scav != RISCV::NoRegister && "No register is scavenged!"); 928 MRI.replaceRegWith(ScratchReg, Scav); 929 MRI.clearVirtRegs(); 930 RS->setRegUsed(Scav); 931 } 932 933 bool RISCVInstrInfo::reverseBranchCondition( 934 SmallVectorImpl<MachineOperand> &Cond) const { 935 assert((Cond.size() == 3) && "Invalid branch condition!"); 936 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm()); 937 Cond[0].setImm(getOppositeBranchCondition(CC)); 938 return false; 939 } 940 941 MachineBasicBlock * 942 RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { 943 assert(MI.getDesc().isBranch() && "Unexpected opcode!"); 944 // The branch target is always the last operand. 945 int NumOp = MI.getNumExplicitOperands(); 946 return MI.getOperand(NumOp - 1).getMBB(); 947 } 948 949 bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 950 int64_t BrOffset) const { 951 unsigned XLen = STI.getXLen(); 952 // Ideally we could determine the supported branch offset from the 953 // RISCVII::FormMask, but this can't be used for Pseudo instructions like 954 // PseudoBR. 955 switch (BranchOp) { 956 default: 957 llvm_unreachable("Unexpected opcode!"); 958 case RISCV::BEQ: 959 case RISCV::BNE: 960 case RISCV::BLT: 961 case RISCV::BGE: 962 case RISCV::BLTU: 963 case RISCV::BGEU: 964 return isIntN(13, BrOffset); 965 case RISCV::JAL: 966 case RISCV::PseudoBR: 967 return isIntN(21, BrOffset); 968 case RISCV::PseudoJump: 969 return isIntN(32, SignExtend64(BrOffset + 0x800, XLen)); 970 } 971 } 972 973 unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 974 if (MI.isMetaInstruction()) 975 return 0; 976 977 unsigned Opcode = MI.getOpcode(); 978 979 if (Opcode == TargetOpcode::INLINEASM || 980 Opcode == TargetOpcode::INLINEASM_BR) { 981 const MachineFunction &MF = *MI.getParent()->getParent(); 982 const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget()); 983 return getInlineAsmLength(MI.getOperand(0).getSymbolName(), 984 *TM.getMCAsmInfo()); 985 } 986 987 if (MI.getParent() && MI.getParent()->getParent()) { 988 const auto MF = MI.getMF(); 989 const auto &TM = static_cast<const RISCVTargetMachine &>(MF->getTarget()); 990 const MCRegisterInfo &MRI = *TM.getMCRegisterInfo(); 991 const MCSubtargetInfo &STI = *TM.getMCSubtargetInfo(); 992 const RISCVSubtarget &ST = MF->getSubtarget<RISCVSubtarget>(); 993 if (isCompressibleInst(MI, &ST, MRI, STI)) 994 return 2; 995 } 996 return get(Opcode).getSize(); 997 } 998 999 bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const { 1000 const unsigned Opcode = MI.getOpcode(); 1001 switch (Opcode) { 1002 default: 1003 break; 1004 case RISCV::FSGNJ_D: 1005 case RISCV::FSGNJ_S: 1006 case RISCV::FSGNJ_H: 1007 // The canonical floating-point move is fsgnj rd, rs, rs. 1008 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && 1009 MI.getOperand(1).getReg() == MI.getOperand(2).getReg(); 1010 case RISCV::ADDI: 1011 case RISCV::ORI: 1012 case RISCV::XORI: 1013 return (MI.getOperand(1).isReg() && 1014 MI.getOperand(1).getReg() == RISCV::X0) || 1015 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0); 1016 } 1017 return MI.isAsCheapAsAMove(); 1018 } 1019 1020 Optional<DestSourcePair> 1021 RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 1022 if (MI.isMoveReg()) 1023 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1024 switch (MI.getOpcode()) { 1025 default: 1026 break; 1027 case RISCV::ADDI: 1028 // Operand 1 can be a frameindex but callers expect registers 1029 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() && 1030 MI.getOperand(2).getImm() == 0) 1031 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1032 break; 1033 case RISCV::FSGNJ_D: 1034 case RISCV::FSGNJ_S: 1035 case RISCV::FSGNJ_H: 1036 // The canonical floating-point move is fsgnj rd, rs, rs. 1037 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && 1038 MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 1039 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1040 break; 1041 } 1042 return None; 1043 } 1044 1045 bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI, 1046 StringRef &ErrInfo) const { 1047 const MCInstrInfo *MCII = STI.getInstrInfo(); 1048 MCInstrDesc const &Desc = MCII->get(MI.getOpcode()); 1049 1050 for (auto &OI : enumerate(Desc.operands())) { 1051 unsigned OpType = OI.value().OperandType; 1052 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM && 1053 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) { 1054 const MachineOperand &MO = MI.getOperand(OI.index()); 1055 if (MO.isImm()) { 1056 int64_t Imm = MO.getImm(); 1057 bool Ok; 1058 switch (OpType) { 1059 default: 1060 llvm_unreachable("Unexpected operand type"); 1061 1062 // clang-format off 1063 #define CASE_OPERAND_UIMM(NUM) \ 1064 case RISCVOp::OPERAND_UIMM##NUM: \ 1065 Ok = isUInt<NUM>(Imm); \ 1066 break; 1067 CASE_OPERAND_UIMM(2) 1068 CASE_OPERAND_UIMM(3) 1069 CASE_OPERAND_UIMM(4) 1070 CASE_OPERAND_UIMM(5) 1071 CASE_OPERAND_UIMM(7) 1072 CASE_OPERAND_UIMM(12) 1073 CASE_OPERAND_UIMM(20) 1074 // clang-format on 1075 case RISCVOp::OPERAND_SIMM12: 1076 Ok = isInt<12>(Imm); 1077 break; 1078 case RISCVOp::OPERAND_SIMM12_LSB00000: 1079 Ok = isShiftedInt<7, 5>(Imm); 1080 break; 1081 case RISCVOp::OPERAND_UIMMLOG2XLEN: 1082 if (STI.getTargetTriple().isArch64Bit()) 1083 Ok = isUInt<6>(Imm); 1084 else 1085 Ok = isUInt<5>(Imm); 1086 break; 1087 case RISCVOp::OPERAND_RVKRNUM: 1088 Ok = Imm >= 0 && Imm <= 10; 1089 break; 1090 } 1091 if (!Ok) { 1092 ErrInfo = "Invalid immediate"; 1093 return false; 1094 } 1095 } 1096 } 1097 } 1098 1099 return true; 1100 } 1101 1102 // Return true if get the base operand, byte offset of an instruction and the 1103 // memory width. Width is the size of memory that is being loaded/stored. 1104 bool RISCVInstrInfo::getMemOperandWithOffsetWidth( 1105 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, 1106 unsigned &Width, const TargetRegisterInfo *TRI) const { 1107 if (!LdSt.mayLoadOrStore()) 1108 return false; 1109 1110 // Here we assume the standard RISC-V ISA, which uses a base+offset 1111 // addressing mode. You'll need to relax these conditions to support custom 1112 // load/stores instructions. 1113 if (LdSt.getNumExplicitOperands() != 3) 1114 return false; 1115 if (!LdSt.getOperand(1).isReg() || !LdSt.getOperand(2).isImm()) 1116 return false; 1117 1118 if (!LdSt.hasOneMemOperand()) 1119 return false; 1120 1121 Width = (*LdSt.memoperands_begin())->getSize(); 1122 BaseReg = &LdSt.getOperand(1); 1123 Offset = LdSt.getOperand(2).getImm(); 1124 return true; 1125 } 1126 1127 bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint( 1128 const MachineInstr &MIa, const MachineInstr &MIb) const { 1129 assert(MIa.mayLoadOrStore() && "MIa must be a load or store."); 1130 assert(MIb.mayLoadOrStore() && "MIb must be a load or store."); 1131 1132 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() || 1133 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 1134 return false; 1135 1136 // Retrieve the base register, offset from the base register and width. Width 1137 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If 1138 // base registers are identical, and the offset of a lower memory access + 1139 // the width doesn't overlap the offset of a higher memory access, 1140 // then the memory accesses are different. 1141 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 1142 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; 1143 int64_t OffsetA = 0, OffsetB = 0; 1144 unsigned int WidthA = 0, WidthB = 0; 1145 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && 1146 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { 1147 if (BaseOpA->isIdenticalTo(*BaseOpB)) { 1148 int LowOffset = std::min(OffsetA, OffsetB); 1149 int HighOffset = std::max(OffsetA, OffsetB); 1150 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 1151 if (LowOffset + LowWidth <= HighOffset) 1152 return true; 1153 } 1154 } 1155 return false; 1156 } 1157 1158 std::pair<unsigned, unsigned> 1159 RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 1160 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK; 1161 return std::make_pair(TF & Mask, TF & ~Mask); 1162 } 1163 1164 ArrayRef<std::pair<unsigned, const char *>> 1165 RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 1166 using namespace RISCVII; 1167 static const std::pair<unsigned, const char *> TargetFlags[] = { 1168 {MO_CALL, "riscv-call"}, 1169 {MO_PLT, "riscv-plt"}, 1170 {MO_LO, "riscv-lo"}, 1171 {MO_HI, "riscv-hi"}, 1172 {MO_PCREL_LO, "riscv-pcrel-lo"}, 1173 {MO_PCREL_HI, "riscv-pcrel-hi"}, 1174 {MO_GOT_HI, "riscv-got-hi"}, 1175 {MO_TPREL_LO, "riscv-tprel-lo"}, 1176 {MO_TPREL_HI, "riscv-tprel-hi"}, 1177 {MO_TPREL_ADD, "riscv-tprel-add"}, 1178 {MO_TLS_GOT_HI, "riscv-tls-got-hi"}, 1179 {MO_TLS_GD_HI, "riscv-tls-gd-hi"}}; 1180 return makeArrayRef(TargetFlags); 1181 } 1182 bool RISCVInstrInfo::isFunctionSafeToOutlineFrom( 1183 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const { 1184 const Function &F = MF.getFunction(); 1185 1186 // Can F be deduplicated by the linker? If it can, don't outline from it. 1187 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 1188 return false; 1189 1190 // Don't outline from functions with section markings; the program could 1191 // expect that all the code is in the named section. 1192 if (F.hasSection()) 1193 return false; 1194 1195 // It's safe to outline from MF. 1196 return true; 1197 } 1198 1199 bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, 1200 unsigned &Flags) const { 1201 // More accurate safety checking is done in getOutliningCandidateInfo. 1202 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags); 1203 } 1204 1205 // Enum values indicating how an outlined call should be constructed. 1206 enum MachineOutlinerConstructionID { 1207 MachineOutlinerDefault 1208 }; 1209 1210 bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault( 1211 MachineFunction &MF) const { 1212 return MF.getFunction().hasMinSize(); 1213 } 1214 1215 outliner::OutlinedFunction RISCVInstrInfo::getOutliningCandidateInfo( 1216 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 1217 1218 // First we need to filter out candidates where the X5 register (IE t0) can't 1219 // be used to setup the function call. 1220 auto CannotInsertCall = [](outliner::Candidate &C) { 1221 const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo(); 1222 return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI); 1223 }; 1224 1225 llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall); 1226 1227 // If the sequence doesn't have enough candidates left, then we're done. 1228 if (RepeatedSequenceLocs.size() < 2) 1229 return outliner::OutlinedFunction(); 1230 1231 unsigned SequenceSize = 0; 1232 1233 auto I = RepeatedSequenceLocs[0].front(); 1234 auto E = std::next(RepeatedSequenceLocs[0].back()); 1235 for (; I != E; ++I) 1236 SequenceSize += getInstSizeInBytes(*I); 1237 1238 // call t0, function = 8 bytes. 1239 unsigned CallOverhead = 8; 1240 for (auto &C : RepeatedSequenceLocs) 1241 C.setCallInfo(MachineOutlinerDefault, CallOverhead); 1242 1243 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled. 1244 unsigned FrameOverhead = 4; 1245 if (RepeatedSequenceLocs[0].getMF()->getSubtarget() 1246 .getFeatureBits()[RISCV::FeatureStdExtC]) 1247 FrameOverhead = 2; 1248 1249 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1250 FrameOverhead, MachineOutlinerDefault); 1251 } 1252 1253 outliner::InstrType 1254 RISCVInstrInfo::getOutliningType(MachineBasicBlock::iterator &MBBI, 1255 unsigned Flags) const { 1256 MachineInstr &MI = *MBBI; 1257 MachineBasicBlock *MBB = MI.getParent(); 1258 const TargetRegisterInfo *TRI = 1259 MBB->getParent()->getSubtarget().getRegisterInfo(); 1260 1261 // Positions generally can't safely be outlined. 1262 if (MI.isPosition()) { 1263 // We can manually strip out CFI instructions later. 1264 if (MI.isCFIInstruction()) 1265 // If current function has exception handling code, we can't outline & 1266 // strip these CFI instructions since it may break .eh_frame section 1267 // needed in unwinding. 1268 return MI.getMF()->getFunction().needsUnwindTableEntry() 1269 ? outliner::InstrType::Illegal 1270 : outliner::InstrType::Invisible; 1271 1272 return outliner::InstrType::Illegal; 1273 } 1274 1275 // Don't trust the user to write safe inline assembly. 1276 if (MI.isInlineAsm()) 1277 return outliner::InstrType::Illegal; 1278 1279 // We can't outline branches to other basic blocks. 1280 if (MI.isTerminator() && !MBB->succ_empty()) 1281 return outliner::InstrType::Illegal; 1282 1283 // We need support for tail calls to outlined functions before return 1284 // statements can be allowed. 1285 if (MI.isReturn()) 1286 return outliner::InstrType::Illegal; 1287 1288 // Don't allow modifying the X5 register which we use for return addresses for 1289 // these outlined functions. 1290 if (MI.modifiesRegister(RISCV::X5, TRI) || 1291 MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5)) 1292 return outliner::InstrType::Illegal; 1293 1294 // Make sure the operands don't reference something unsafe. 1295 for (const auto &MO : MI.operands()) 1296 if (MO.isMBB() || MO.isBlockAddress() || MO.isCPI() || MO.isJTI()) 1297 return outliner::InstrType::Illegal; 1298 1299 // Don't allow instructions which won't be materialized to impact outlining 1300 // analysis. 1301 if (MI.isMetaInstruction()) 1302 return outliner::InstrType::Invisible; 1303 1304 return outliner::InstrType::Legal; 1305 } 1306 1307 void RISCVInstrInfo::buildOutlinedFrame( 1308 MachineBasicBlock &MBB, MachineFunction &MF, 1309 const outliner::OutlinedFunction &OF) const { 1310 1311 // Strip out any CFI instructions 1312 bool Changed = true; 1313 while (Changed) { 1314 Changed = false; 1315 auto I = MBB.begin(); 1316 auto E = MBB.end(); 1317 for (; I != E; ++I) { 1318 if (I->isCFIInstruction()) { 1319 I->removeFromParent(); 1320 Changed = true; 1321 break; 1322 } 1323 } 1324 } 1325 1326 MBB.addLiveIn(RISCV::X5); 1327 1328 // Add in a return instruction to the end of the outlined frame. 1329 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR)) 1330 .addReg(RISCV::X0, RegState::Define) 1331 .addReg(RISCV::X5) 1332 .addImm(0)); 1333 } 1334 1335 MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall( 1336 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, 1337 MachineFunction &MF, outliner::Candidate &C) const { 1338 1339 // Add in a call instruction to the outlined function at the given location. 1340 It = MBB.insert(It, 1341 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5) 1342 .addGlobalAddress(M.getNamedValue(MF.getName()), 0, 1343 RISCVII::MO_CALL)); 1344 return It; 1345 } 1346 1347 // MIR printer helper function to annotate Operands with a comment. 1348 std::string RISCVInstrInfo::createMIROperandComment( 1349 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, 1350 const TargetRegisterInfo *TRI) const { 1351 // Print a generic comment for this operand if there is one. 1352 std::string GenericComment = 1353 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI); 1354 if (!GenericComment.empty()) 1355 return GenericComment; 1356 1357 // If not, we must have an immediate operand. 1358 if (!Op.isImm()) 1359 return std::string(); 1360 1361 std::string Comment; 1362 raw_string_ostream OS(Comment); 1363 1364 uint64_t TSFlags = MI.getDesc().TSFlags; 1365 1366 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW 1367 // operand of vector codegen pseudos. 1368 if ((MI.getOpcode() == RISCV::VSETVLI || MI.getOpcode() == RISCV::VSETIVLI || 1369 MI.getOpcode() == RISCV::PseudoVSETVLI || 1370 MI.getOpcode() == RISCV::PseudoVSETIVLI || 1371 MI.getOpcode() == RISCV::PseudoVSETVLIX0) && 1372 OpIdx == 2) { 1373 unsigned Imm = MI.getOperand(OpIdx).getImm(); 1374 RISCVVType::printVType(Imm, OS); 1375 } else if (RISCVII::hasSEWOp(TSFlags)) { 1376 unsigned NumOperands = MI.getNumExplicitOperands(); 1377 bool HasPolicy = RISCVII::hasVecPolicyOp(TSFlags); 1378 1379 // The SEW operand is before any policy operand. 1380 if (OpIdx != NumOperands - HasPolicy - 1) 1381 return std::string(); 1382 1383 unsigned Log2SEW = MI.getOperand(OpIdx).getImm(); 1384 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8; 1385 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW"); 1386 1387 OS << "e" << SEW; 1388 } 1389 1390 OS.flush(); 1391 return Comment; 1392 } 1393 1394 // clang-format off 1395 #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL) \ 1396 RISCV::PseudoV##OP##_##TYPE##_##LMUL 1397 1398 #define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE) \ 1399 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1): \ 1400 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2): \ 1401 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4): \ 1402 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8) 1403 1404 #define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE) \ 1405 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2): \ 1406 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE) 1407 1408 #define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE) \ 1409 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4): \ 1410 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE) 1411 1412 #define CASE_VFMA_OPCODE_LMULS(OP, TYPE) \ 1413 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8): \ 1414 case CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE) 1415 1416 #define CASE_VFMA_SPLATS(OP) \ 1417 CASE_VFMA_OPCODE_LMULS_MF4(OP, VF16): \ 1418 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VF32): \ 1419 case CASE_VFMA_OPCODE_LMULS_M1(OP, VF64) 1420 // clang-format on 1421 1422 bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 1423 unsigned &SrcOpIdx1, 1424 unsigned &SrcOpIdx2) const { 1425 const MCInstrDesc &Desc = MI.getDesc(); 1426 if (!Desc.isCommutable()) 1427 return false; 1428 1429 switch (MI.getOpcode()) { 1430 case CASE_VFMA_SPLATS(FMADD): 1431 case CASE_VFMA_SPLATS(FMSUB): 1432 case CASE_VFMA_SPLATS(FMACC): 1433 case CASE_VFMA_SPLATS(FMSAC): 1434 case CASE_VFMA_SPLATS(FNMADD): 1435 case CASE_VFMA_SPLATS(FNMSUB): 1436 case CASE_VFMA_SPLATS(FNMACC): 1437 case CASE_VFMA_SPLATS(FNMSAC): 1438 case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV): 1439 case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV): 1440 case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV): 1441 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV): 1442 case CASE_VFMA_OPCODE_LMULS(MADD, VX): 1443 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX): 1444 case CASE_VFMA_OPCODE_LMULS(MACC, VX): 1445 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): 1446 case CASE_VFMA_OPCODE_LMULS(MACC, VV): 1447 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { 1448 // If the tail policy is undisturbed we can't commute. 1449 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); 1450 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) 1451 return false; 1452 1453 // For these instructions we can only swap operand 1 and operand 3 by 1454 // changing the opcode. 1455 unsigned CommutableOpIdx1 = 1; 1456 unsigned CommutableOpIdx2 = 3; 1457 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 1458 CommutableOpIdx2)) 1459 return false; 1460 return true; 1461 } 1462 case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV): 1463 case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV): 1464 case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV): 1465 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV): 1466 case CASE_VFMA_OPCODE_LMULS(MADD, VV): 1467 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { 1468 // If the tail policy is undisturbed we can't commute. 1469 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); 1470 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) 1471 return false; 1472 1473 // For these instructions we have more freedom. We can commute with the 1474 // other multiplicand or with the addend/subtrahend/minuend. 1475 1476 // Any fixed operand must be from source 1, 2 or 3. 1477 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3) 1478 return false; 1479 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3) 1480 return false; 1481 1482 // It both ops are fixed one must be the tied source. 1483 if (SrcOpIdx1 != CommuteAnyOperandIndex && 1484 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1) 1485 return false; 1486 1487 // Look for two different register operands assumed to be commutable 1488 // regardless of the FMA opcode. The FMA opcode is adjusted later if 1489 // needed. 1490 if (SrcOpIdx1 == CommuteAnyOperandIndex || 1491 SrcOpIdx2 == CommuteAnyOperandIndex) { 1492 // At least one of operands to be commuted is not specified and 1493 // this method is free to choose appropriate commutable operands. 1494 unsigned CommutableOpIdx1 = SrcOpIdx1; 1495 if (SrcOpIdx1 == SrcOpIdx2) { 1496 // Both of operands are not fixed. Set one of commutable 1497 // operands to the tied source. 1498 CommutableOpIdx1 = 1; 1499 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) { 1500 // Only one of the operands is not fixed. 1501 CommutableOpIdx1 = SrcOpIdx2; 1502 } 1503 1504 // CommutableOpIdx1 is well defined now. Let's choose another commutable 1505 // operand and assign its index to CommutableOpIdx2. 1506 unsigned CommutableOpIdx2; 1507 if (CommutableOpIdx1 != 1) { 1508 // If we haven't already used the tied source, we must use it now. 1509 CommutableOpIdx2 = 1; 1510 } else { 1511 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg(); 1512 1513 // The commuted operands should have different registers. 1514 // Otherwise, the commute transformation does not change anything and 1515 // is useless. We use this as a hint to make our decision. 1516 if (Op1Reg != MI.getOperand(2).getReg()) 1517 CommutableOpIdx2 = 2; 1518 else 1519 CommutableOpIdx2 = 3; 1520 } 1521 1522 // Assign the found pair of commutable indices to SrcOpIdx1 and 1523 // SrcOpIdx2 to return those values. 1524 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 1525 CommutableOpIdx2)) 1526 return false; 1527 } 1528 1529 return true; 1530 } 1531 } 1532 1533 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 1534 } 1535 1536 #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \ 1537 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \ 1538 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \ 1539 break; 1540 1541 #define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \ 1542 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \ 1543 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \ 1544 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \ 1545 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8) 1546 1547 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \ 1548 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \ 1549 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) 1550 1551 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \ 1552 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \ 1553 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) 1554 1555 #define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \ 1556 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \ 1557 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) 1558 1559 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \ 1560 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VF16) \ 1561 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VF32) \ 1562 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VF64) 1563 1564 MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI, 1565 bool NewMI, 1566 unsigned OpIdx1, 1567 unsigned OpIdx2) const { 1568 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 1569 if (NewMI) 1570 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 1571 return MI; 1572 }; 1573 1574 switch (MI.getOpcode()) { 1575 case CASE_VFMA_SPLATS(FMACC): 1576 case CASE_VFMA_SPLATS(FMADD): 1577 case CASE_VFMA_SPLATS(FMSAC): 1578 case CASE_VFMA_SPLATS(FMSUB): 1579 case CASE_VFMA_SPLATS(FNMACC): 1580 case CASE_VFMA_SPLATS(FNMADD): 1581 case CASE_VFMA_SPLATS(FNMSAC): 1582 case CASE_VFMA_SPLATS(FNMSUB): 1583 case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV): 1584 case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV): 1585 case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV): 1586 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV): 1587 case CASE_VFMA_OPCODE_LMULS(MADD, VX): 1588 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX): 1589 case CASE_VFMA_OPCODE_LMULS(MACC, VX): 1590 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): 1591 case CASE_VFMA_OPCODE_LMULS(MACC, VV): 1592 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { 1593 // It only make sense to toggle these between clobbering the 1594 // addend/subtrahend/minuend one of the multiplicands. 1595 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index"); 1596 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index"); 1597 unsigned Opc; 1598 switch (MI.getOpcode()) { 1599 default: 1600 llvm_unreachable("Unexpected opcode"); 1601 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD) 1602 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC) 1603 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB) 1604 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC) 1605 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD) 1606 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC) 1607 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB) 1608 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC) 1609 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMACC, FMADD, VV) 1610 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSAC, FMSUB, VV) 1611 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMACC, FNMADD, VV) 1612 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSAC, FNMSUB, VV) 1613 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX) 1614 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX) 1615 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX) 1616 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX) 1617 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV) 1618 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV) 1619 } 1620 1621 auto &WorkingMI = cloneIfNew(MI); 1622 WorkingMI.setDesc(get(Opc)); 1623 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1624 OpIdx1, OpIdx2); 1625 } 1626 case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV): 1627 case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV): 1628 case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV): 1629 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV): 1630 case CASE_VFMA_OPCODE_LMULS(MADD, VV): 1631 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { 1632 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index"); 1633 // If one of the operands, is the addend we need to change opcode. 1634 // Otherwise we're just swapping 2 of the multiplicands. 1635 if (OpIdx1 == 3 || OpIdx2 == 3) { 1636 unsigned Opc; 1637 switch (MI.getOpcode()) { 1638 default: 1639 llvm_unreachable("Unexpected opcode"); 1640 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMADD, FMACC, VV) 1641 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSUB, FMSAC, VV) 1642 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMADD, FNMACC, VV) 1643 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSUB, FNMSAC, VV) 1644 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV) 1645 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV) 1646 } 1647 1648 auto &WorkingMI = cloneIfNew(MI); 1649 WorkingMI.setDesc(get(Opc)); 1650 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1651 OpIdx1, OpIdx2); 1652 } 1653 // Let the default code handle it. 1654 break; 1655 } 1656 } 1657 1658 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 1659 } 1660 1661 #undef CASE_VFMA_CHANGE_OPCODE_SPLATS 1662 #undef CASE_VFMA_CHANGE_OPCODE_LMULS 1663 #undef CASE_VFMA_CHANGE_OPCODE_COMMON 1664 #undef CASE_VFMA_SPLATS 1665 #undef CASE_VFMA_OPCODE_LMULS 1666 #undef CASE_VFMA_OPCODE_COMMON 1667 1668 // clang-format off 1669 #define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \ 1670 RISCV::PseudoV##OP##_##LMUL##_TIED 1671 1672 #define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \ 1673 CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \ 1674 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \ 1675 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \ 1676 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \ 1677 case CASE_WIDEOP_OPCODE_COMMON(OP, M4) 1678 1679 #define CASE_WIDEOP_OPCODE_LMULS(OP) \ 1680 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \ 1681 case CASE_WIDEOP_OPCODE_LMULS_MF4(OP) 1682 // clang-format on 1683 1684 #define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \ 1685 case RISCV::PseudoV##OP##_##LMUL##_TIED: \ 1686 NewOpc = RISCV::PseudoV##OP##_##LMUL; \ 1687 break; 1688 1689 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \ 1690 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \ 1691 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \ 1692 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \ 1693 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \ 1694 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4) 1695 1696 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \ 1697 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \ 1698 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) 1699 1700 MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI, 1701 LiveVariables *LV, 1702 LiveIntervals *LIS) const { 1703 switch (MI.getOpcode()) { 1704 default: 1705 break; 1706 case CASE_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV): 1707 case CASE_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): 1708 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV): 1709 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV): 1710 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV): 1711 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): { 1712 // If the tail policy is undisturbed we can't convert. 1713 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) && 1714 MI.getNumExplicitOperands() == 6); 1715 if ((MI.getOperand(5).getImm() & 1) == 0) 1716 return nullptr; 1717 1718 // clang-format off 1719 unsigned NewOpc; 1720 switch (MI.getOpcode()) { 1721 default: 1722 llvm_unreachable("Unexpected opcode"); 1723 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV) 1724 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV) 1725 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV) 1726 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV) 1727 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV) 1728 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV) 1729 } 1730 // clang-format on 1731 1732 MachineBasicBlock &MBB = *MI.getParent(); 1733 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 1734 .add(MI.getOperand(0)) 1735 .add(MI.getOperand(1)) 1736 .add(MI.getOperand(2)) 1737 .add(MI.getOperand(3)) 1738 .add(MI.getOperand(4)); 1739 MIB.copyImplicitOps(MI); 1740 1741 if (LV) { 1742 unsigned NumOps = MI.getNumOperands(); 1743 for (unsigned I = 1; I < NumOps; ++I) { 1744 MachineOperand &Op = MI.getOperand(I); 1745 if (Op.isReg() && Op.isKill()) 1746 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 1747 } 1748 } 1749 1750 if (LIS) { 1751 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB); 1752 1753 if (MI.getOperand(0).isEarlyClobber()) { 1754 // Use operand 1 was tied to early-clobber def operand 0, so its live 1755 // interval could have ended at an early-clobber slot. Now they are not 1756 // tied we need to update it to the normal register slot. 1757 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg()); 1758 LiveRange::Segment *S = LI.getSegmentContaining(Idx); 1759 if (S->end == Idx.getRegSlot(true)) 1760 S->end = Idx.getRegSlot(); 1761 } 1762 } 1763 1764 return MIB; 1765 } 1766 } 1767 1768 return nullptr; 1769 } 1770 1771 #undef CASE_WIDEOP_CHANGE_OPCODE_LMULS 1772 #undef CASE_WIDEOP_CHANGE_OPCODE_COMMON 1773 #undef CASE_WIDEOP_OPCODE_LMULS 1774 #undef CASE_WIDEOP_OPCODE_COMMON 1775 1776 Register RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, 1777 MachineBasicBlock &MBB, 1778 MachineBasicBlock::iterator II, 1779 const DebugLoc &DL, 1780 int64_t Amount, 1781 MachineInstr::MIFlag Flag) const { 1782 assert(Amount > 0 && "There is no need to get VLEN scaled value."); 1783 assert(Amount % 8 == 0 && 1784 "Reserve the stack by the multiple of one vector size."); 1785 1786 MachineRegisterInfo &MRI = MF.getRegInfo(); 1787 int64_t NumOfVReg = Amount / 8; 1788 1789 Register VL = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1790 BuildMI(MBB, II, DL, get(RISCV::PseudoReadVLENB), VL) 1791 .setMIFlag(Flag); 1792 assert(isInt<32>(NumOfVReg) && 1793 "Expect the number of vector registers within 32-bits."); 1794 if (isPowerOf2_32(NumOfVReg)) { 1795 uint32_t ShiftAmount = Log2_32(NumOfVReg); 1796 if (ShiftAmount == 0) 1797 return VL; 1798 BuildMI(MBB, II, DL, get(RISCV::SLLI), VL) 1799 .addReg(VL, RegState::Kill) 1800 .addImm(ShiftAmount) 1801 .setMIFlag(Flag); 1802 } else if ((NumOfVReg == 3 || NumOfVReg == 5 || NumOfVReg == 9) && 1803 STI.hasStdExtZba()) { 1804 // We can use Zba SHXADD instructions for multiply in some cases. 1805 // TODO: Generalize to SHXADD+SLLI. 1806 unsigned Opc; 1807 switch (NumOfVReg) { 1808 default: llvm_unreachable("Unexpected number of vregs"); 1809 case 3: Opc = RISCV::SH1ADD; break; 1810 case 5: Opc = RISCV::SH2ADD; break; 1811 case 9: Opc = RISCV::SH3ADD; break; 1812 } 1813 BuildMI(MBB, II, DL, get(Opc), VL) 1814 .addReg(VL, RegState::Kill) 1815 .addReg(VL) 1816 .setMIFlag(Flag); 1817 } else if (isPowerOf2_32(NumOfVReg - 1)) { 1818 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1819 uint32_t ShiftAmount = Log2_32(NumOfVReg - 1); 1820 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister) 1821 .addReg(VL) 1822 .addImm(ShiftAmount) 1823 .setMIFlag(Flag); 1824 BuildMI(MBB, II, DL, get(RISCV::ADD), VL) 1825 .addReg(ScaledRegister, RegState::Kill) 1826 .addReg(VL, RegState::Kill) 1827 .setMIFlag(Flag); 1828 } else if (isPowerOf2_32(NumOfVReg + 1)) { 1829 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1830 uint32_t ShiftAmount = Log2_32(NumOfVReg + 1); 1831 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister) 1832 .addReg(VL) 1833 .addImm(ShiftAmount) 1834 .setMIFlag(Flag); 1835 BuildMI(MBB, II, DL, get(RISCV::SUB), VL) 1836 .addReg(ScaledRegister, RegState::Kill) 1837 .addReg(VL, RegState::Kill) 1838 .setMIFlag(Flag); 1839 } else { 1840 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1841 movImm(MBB, II, DL, N, NumOfVReg, Flag); 1842 if (!STI.hasStdExtM()) 1843 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ 1844 MF.getFunction(), 1845 "M-extension must be enabled to calculate the vscaled size/offset."}); 1846 BuildMI(MBB, II, DL, get(RISCV::MUL), VL) 1847 .addReg(VL, RegState::Kill) 1848 .addReg(N, RegState::Kill) 1849 .setMIFlag(Flag); 1850 } 1851 1852 return VL; 1853 } 1854 1855 static bool isRVVWholeLoadStore(unsigned Opcode) { 1856 switch (Opcode) { 1857 default: 1858 return false; 1859 case RISCV::VS1R_V: 1860 case RISCV::VS2R_V: 1861 case RISCV::VS4R_V: 1862 case RISCV::VS8R_V: 1863 case RISCV::VL1RE8_V: 1864 case RISCV::VL2RE8_V: 1865 case RISCV::VL4RE8_V: 1866 case RISCV::VL8RE8_V: 1867 case RISCV::VL1RE16_V: 1868 case RISCV::VL2RE16_V: 1869 case RISCV::VL4RE16_V: 1870 case RISCV::VL8RE16_V: 1871 case RISCV::VL1RE32_V: 1872 case RISCV::VL2RE32_V: 1873 case RISCV::VL4RE32_V: 1874 case RISCV::VL8RE32_V: 1875 case RISCV::VL1RE64_V: 1876 case RISCV::VL2RE64_V: 1877 case RISCV::VL4RE64_V: 1878 case RISCV::VL8RE64_V: 1879 return true; 1880 } 1881 } 1882 1883 bool RISCV::isRVVSpill(const MachineInstr &MI) { 1884 // RVV lacks any support for immediate addressing for stack addresses, so be 1885 // conservative. 1886 unsigned Opcode = MI.getOpcode(); 1887 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) && 1888 !isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode)) 1889 return false; 1890 return true; 1891 } 1892 1893 Optional<std::pair<unsigned, unsigned>> 1894 RISCV::isRVVSpillForZvlsseg(unsigned Opcode) { 1895 switch (Opcode) { 1896 default: 1897 return None; 1898 case RISCV::PseudoVSPILL2_M1: 1899 case RISCV::PseudoVRELOAD2_M1: 1900 return std::make_pair(2u, 1u); 1901 case RISCV::PseudoVSPILL2_M2: 1902 case RISCV::PseudoVRELOAD2_M2: 1903 return std::make_pair(2u, 2u); 1904 case RISCV::PseudoVSPILL2_M4: 1905 case RISCV::PseudoVRELOAD2_M4: 1906 return std::make_pair(2u, 4u); 1907 case RISCV::PseudoVSPILL3_M1: 1908 case RISCV::PseudoVRELOAD3_M1: 1909 return std::make_pair(3u, 1u); 1910 case RISCV::PseudoVSPILL3_M2: 1911 case RISCV::PseudoVRELOAD3_M2: 1912 return std::make_pair(3u, 2u); 1913 case RISCV::PseudoVSPILL4_M1: 1914 case RISCV::PseudoVRELOAD4_M1: 1915 return std::make_pair(4u, 1u); 1916 case RISCV::PseudoVSPILL4_M2: 1917 case RISCV::PseudoVRELOAD4_M2: 1918 return std::make_pair(4u, 2u); 1919 case RISCV::PseudoVSPILL5_M1: 1920 case RISCV::PseudoVRELOAD5_M1: 1921 return std::make_pair(5u, 1u); 1922 case RISCV::PseudoVSPILL6_M1: 1923 case RISCV::PseudoVRELOAD6_M1: 1924 return std::make_pair(6u, 1u); 1925 case RISCV::PseudoVSPILL7_M1: 1926 case RISCV::PseudoVRELOAD7_M1: 1927 return std::make_pair(7u, 1u); 1928 case RISCV::PseudoVSPILL8_M1: 1929 case RISCV::PseudoVRELOAD8_M1: 1930 return std::make_pair(8u, 1u); 1931 } 1932 } 1933 1934 bool RISCV::isFaultFirstLoad(const MachineInstr &MI) { 1935 return MI.getNumExplicitDefs() == 2 && MI.modifiesRegister(RISCV::VL) && 1936 !MI.isInlineAsm(); 1937 } 1938