1 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the RISCV implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "RISCVInstrInfo.h" 14 #include "MCTargetDesc/RISCVMatInt.h" 15 #include "RISCV.h" 16 #include "RISCVMachineFunctionInfo.h" 17 #include "RISCVSubtarget.h" 18 #include "RISCVTargetMachine.h" 19 #include "llvm/ADT/STLExtras.h" 20 #include "llvm/ADT/SmallVector.h" 21 #include "llvm/Analysis/MemoryLocation.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/LiveVariables.h" 24 #include "llvm/CodeGen/MachineFunctionPass.h" 25 #include "llvm/CodeGen/MachineInstrBuilder.h" 26 #include "llvm/CodeGen/MachineRegisterInfo.h" 27 #include "llvm/CodeGen/RegisterScavenging.h" 28 #include "llvm/MC/MCInstBuilder.h" 29 #include "llvm/MC/TargetRegistry.h" 30 #include "llvm/Support/ErrorHandling.h" 31 32 using namespace llvm; 33 34 #define GEN_CHECK_COMPRESS_INSTR 35 #include "RISCVGenCompressInstEmitter.inc" 36 37 #define GET_INSTRINFO_CTOR_DTOR 38 #define GET_INSTRINFO_NAMED_OPS 39 #include "RISCVGenInstrInfo.inc" 40 41 static cl::opt<bool> PreferWholeRegisterMove( 42 "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, 43 cl::desc("Prefer whole register move for vector registers.")); 44 45 namespace llvm { 46 namespace RISCVVPseudosTable { 47 48 using namespace RISCV; 49 50 #define GET_RISCVVPseudosTable_IMPL 51 #include "RISCVGenSearchableTables.inc" 52 53 } // namespace RISCVVPseudosTable 54 } // namespace llvm 55 56 RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI) 57 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP), 58 STI(STI) {} 59 60 MCInst RISCVInstrInfo::getNop() const { 61 if (STI.getFeatureBits()[RISCV::FeatureStdExtC]) 62 return MCInstBuilder(RISCV::C_NOP); 63 return MCInstBuilder(RISCV::ADDI) 64 .addReg(RISCV::X0) 65 .addReg(RISCV::X0) 66 .addImm(0); 67 } 68 69 unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 70 int &FrameIndex) const { 71 switch (MI.getOpcode()) { 72 default: 73 return 0; 74 case RISCV::LB: 75 case RISCV::LBU: 76 case RISCV::LH: 77 case RISCV::LHU: 78 case RISCV::FLH: 79 case RISCV::LW: 80 case RISCV::FLW: 81 case RISCV::LWU: 82 case RISCV::LD: 83 case RISCV::FLD: 84 break; 85 } 86 87 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 88 MI.getOperand(2).getImm() == 0) { 89 FrameIndex = MI.getOperand(1).getIndex(); 90 return MI.getOperand(0).getReg(); 91 } 92 93 return 0; 94 } 95 96 unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 97 int &FrameIndex) const { 98 switch (MI.getOpcode()) { 99 default: 100 return 0; 101 case RISCV::SB: 102 case RISCV::SH: 103 case RISCV::SW: 104 case RISCV::FSH: 105 case RISCV::FSW: 106 case RISCV::SD: 107 case RISCV::FSD: 108 break; 109 } 110 111 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 112 MI.getOperand(2).getImm() == 0) { 113 FrameIndex = MI.getOperand(1).getIndex(); 114 return MI.getOperand(0).getReg(); 115 } 116 117 return 0; 118 } 119 120 static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg, 121 unsigned NumRegs) { 122 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs; 123 } 124 125 static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, 126 const MachineBasicBlock &MBB, 127 MachineBasicBlock::const_iterator MBBI, 128 MachineBasicBlock::const_iterator &DefMBBI, 129 RISCVII::VLMUL LMul) { 130 if (PreferWholeRegisterMove) 131 return false; 132 133 assert(MBBI->getOpcode() == TargetOpcode::COPY && 134 "Unexpected COPY instruction."); 135 Register SrcReg = MBBI->getOperand(1).getReg(); 136 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 137 138 bool FoundDef = false; 139 bool FirstVSetVLI = false; 140 unsigned FirstSEW = 0; 141 while (MBBI != MBB.begin()) { 142 --MBBI; 143 if (MBBI->isMetaInstruction()) 144 continue; 145 146 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI || 147 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 || 148 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) { 149 // There is a vsetvli between COPY and source define instruction. 150 // vy = def_vop ... (producing instruction) 151 // ... 152 // vsetvli 153 // ... 154 // vx = COPY vy 155 if (!FoundDef) { 156 if (!FirstVSetVLI) { 157 FirstVSetVLI = true; 158 unsigned FirstVType = MBBI->getOperand(2).getImm(); 159 RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType); 160 FirstSEW = RISCVVType::getSEW(FirstVType); 161 // The first encountered vsetvli must have the same lmul as the 162 // register class of COPY. 163 if (FirstLMul != LMul) 164 return false; 165 } 166 // Only permit `vsetvli x0, x0, vtype` between COPY and the source 167 // define instruction. 168 if (MBBI->getOperand(0).getReg() != RISCV::X0) 169 return false; 170 if (MBBI->getOperand(1).isImm()) 171 return false; 172 if (MBBI->getOperand(1).getReg() != RISCV::X0) 173 return false; 174 continue; 175 } 176 177 // MBBI is the first vsetvli before the producing instruction. 178 unsigned VType = MBBI->getOperand(2).getImm(); 179 // If there is a vsetvli between COPY and the producing instruction. 180 if (FirstVSetVLI) { 181 // If SEW is different, return false. 182 if (RISCVVType::getSEW(VType) != FirstSEW) 183 return false; 184 } 185 186 // If the vsetvli is tail undisturbed, keep the whole register move. 187 if (!RISCVVType::isTailAgnostic(VType)) 188 return false; 189 190 // The checking is conservative. We only have register classes for 191 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v 192 // for fractional LMUL operations. However, we could not use the vsetvli 193 // lmul for widening operations. The result of widening operation is 194 // 2 x LMUL. 195 return LMul == RISCVVType::getVLMUL(VType); 196 } else if (MBBI->isInlineAsm() || MBBI->isCall()) { 197 return false; 198 } else if (MBBI->getNumDefs()) { 199 // Check all the instructions which will change VL. 200 // For example, vleff has implicit def VL. 201 if (MBBI->modifiesRegister(RISCV::VL)) 202 return false; 203 204 // Only converting whole register copies to vmv.v.v when the defining 205 // value appears in the explicit operands. 206 for (const MachineOperand &MO : MBBI->explicit_operands()) { 207 if (!MO.isReg() || !MO.isDef()) 208 continue; 209 if (!FoundDef && TRI->isSubRegisterEq(MO.getReg(), SrcReg)) { 210 // We only permit the source of COPY has the same LMUL as the defined 211 // operand. 212 // There are cases we need to keep the whole register copy if the LMUL 213 // is different. 214 // For example, 215 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m 216 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2 217 // # The COPY may be created by vlmul_trunc intrinsic. 218 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4 219 // 220 // After widening, the valid value will be 4 x e32 elements. If we 221 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements. 222 // FIXME: The COPY of subregister of Zvlsseg register will not be able 223 // to convert to vmv.v.[v|i] under the constraint. 224 if (MO.getReg() != SrcReg) 225 return false; 226 227 // In widening reduction instructions with LMUL_1 input vector case, 228 // only checking the LMUL is insufficient due to reduction result is 229 // always LMUL_1. 230 // For example, 231 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu 232 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27 233 // $v26 = COPY killed renamable $v8 234 // After widening, The valid value will be 1 x e16 elements. If we 235 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements. 236 uint64_t TSFlags = MBBI->getDesc().TSFlags; 237 if (RISCVII::isRVVWideningReduction(TSFlags)) 238 return false; 239 240 // Found the definition. 241 FoundDef = true; 242 DefMBBI = MBBI; 243 // If the producing instruction does not depend on vsetvli, do not 244 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD. 245 if (!RISCVII::hasSEWOp(TSFlags)) 246 return false; 247 break; 248 } 249 } 250 } 251 } 252 253 return false; 254 } 255 256 void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 257 MachineBasicBlock::iterator MBBI, 258 const DebugLoc &DL, MCRegister DstReg, 259 MCRegister SrcReg, bool KillSrc) const { 260 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) { 261 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg) 262 .addReg(SrcReg, getKillRegState(KillSrc)) 263 .addImm(0); 264 return; 265 } 266 267 // FPR->FPR copies and VR->VR copies. 268 unsigned Opc; 269 bool IsScalableVector = true; 270 unsigned NF = 1; 271 RISCVII::VLMUL LMul = RISCVII::LMUL_1; 272 unsigned SubRegIdx = RISCV::sub_vrm1_0; 273 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) { 274 Opc = RISCV::FSGNJ_H; 275 IsScalableVector = false; 276 } else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) { 277 Opc = RISCV::FSGNJ_S; 278 IsScalableVector = false; 279 } else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) { 280 Opc = RISCV::FSGNJ_D; 281 IsScalableVector = false; 282 } else if (RISCV::VRRegClass.contains(DstReg, SrcReg)) { 283 Opc = RISCV::PseudoVMV1R_V; 284 LMul = RISCVII::LMUL_1; 285 } else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) { 286 Opc = RISCV::PseudoVMV2R_V; 287 LMul = RISCVII::LMUL_2; 288 } else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) { 289 Opc = RISCV::PseudoVMV4R_V; 290 LMul = RISCVII::LMUL_4; 291 } else if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) { 292 Opc = RISCV::PseudoVMV8R_V; 293 LMul = RISCVII::LMUL_8; 294 } else if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) { 295 Opc = RISCV::PseudoVMV1R_V; 296 SubRegIdx = RISCV::sub_vrm1_0; 297 NF = 2; 298 LMul = RISCVII::LMUL_1; 299 } else if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) { 300 Opc = RISCV::PseudoVMV2R_V; 301 SubRegIdx = RISCV::sub_vrm2_0; 302 NF = 2; 303 LMul = RISCVII::LMUL_2; 304 } else if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) { 305 Opc = RISCV::PseudoVMV4R_V; 306 SubRegIdx = RISCV::sub_vrm4_0; 307 NF = 2; 308 LMul = RISCVII::LMUL_4; 309 } else if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) { 310 Opc = RISCV::PseudoVMV1R_V; 311 SubRegIdx = RISCV::sub_vrm1_0; 312 NF = 3; 313 LMul = RISCVII::LMUL_1; 314 } else if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) { 315 Opc = RISCV::PseudoVMV2R_V; 316 SubRegIdx = RISCV::sub_vrm2_0; 317 NF = 3; 318 LMul = RISCVII::LMUL_2; 319 } else if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) { 320 Opc = RISCV::PseudoVMV1R_V; 321 SubRegIdx = RISCV::sub_vrm1_0; 322 NF = 4; 323 LMul = RISCVII::LMUL_1; 324 } else if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) { 325 Opc = RISCV::PseudoVMV2R_V; 326 SubRegIdx = RISCV::sub_vrm2_0; 327 NF = 4; 328 LMul = RISCVII::LMUL_2; 329 } else if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) { 330 Opc = RISCV::PseudoVMV1R_V; 331 SubRegIdx = RISCV::sub_vrm1_0; 332 NF = 5; 333 LMul = RISCVII::LMUL_1; 334 } else if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) { 335 Opc = RISCV::PseudoVMV1R_V; 336 SubRegIdx = RISCV::sub_vrm1_0; 337 NF = 6; 338 LMul = RISCVII::LMUL_1; 339 } else if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) { 340 Opc = RISCV::PseudoVMV1R_V; 341 SubRegIdx = RISCV::sub_vrm1_0; 342 NF = 7; 343 LMul = RISCVII::LMUL_1; 344 } else if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) { 345 Opc = RISCV::PseudoVMV1R_V; 346 SubRegIdx = RISCV::sub_vrm1_0; 347 NF = 8; 348 LMul = RISCVII::LMUL_1; 349 } else { 350 llvm_unreachable("Impossible reg-to-reg copy"); 351 } 352 353 if (IsScalableVector) { 354 bool UseVMV_V_V = false; 355 MachineBasicBlock::const_iterator DefMBBI; 356 unsigned DefExplicitOpNum; 357 unsigned VIOpc; 358 if (isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) { 359 UseVMV_V_V = true; 360 DefExplicitOpNum = DefMBBI->getNumExplicitOperands(); 361 // We only need to handle LMUL = 1/2/4/8 here because we only define 362 // vector register classes for LMUL = 1/2/4/8. 363 switch (LMul) { 364 default: 365 llvm_unreachable("Impossible LMUL for vector register copy."); 366 case RISCVII::LMUL_1: 367 Opc = RISCV::PseudoVMV_V_V_M1; 368 VIOpc = RISCV::PseudoVMV_V_I_M1; 369 break; 370 case RISCVII::LMUL_2: 371 Opc = RISCV::PseudoVMV_V_V_M2; 372 VIOpc = RISCV::PseudoVMV_V_I_M2; 373 break; 374 case RISCVII::LMUL_4: 375 Opc = RISCV::PseudoVMV_V_V_M4; 376 VIOpc = RISCV::PseudoVMV_V_I_M4; 377 break; 378 case RISCVII::LMUL_8: 379 Opc = RISCV::PseudoVMV_V_V_M8; 380 VIOpc = RISCV::PseudoVMV_V_I_M8; 381 break; 382 } 383 } 384 385 bool UseVMV_V_I = false; 386 if (UseVMV_V_V && (DefMBBI->getOpcode() == VIOpc)) { 387 UseVMV_V_I = true; 388 Opc = VIOpc; 389 } 390 391 if (NF == 1) { 392 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), DstReg); 393 if (UseVMV_V_I) 394 MIB = MIB.add(DefMBBI->getOperand(1)); 395 else 396 MIB = MIB.addReg(SrcReg, getKillRegState(KillSrc)); 397 if (UseVMV_V_V) { 398 // The last two arguments of vector instructions are 399 // AVL, SEW. We also need to append the implicit-use vl and vtype. 400 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 2)); // AVL 401 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 1)); // SEW 402 MIB.addReg(RISCV::VL, RegState::Implicit); 403 MIB.addReg(RISCV::VTYPE, RegState::Implicit); 404 } 405 } else { 406 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 407 408 int I = 0, End = NF, Incr = 1; 409 unsigned SrcEncoding = TRI->getEncodingValue(SrcReg); 410 unsigned DstEncoding = TRI->getEncodingValue(DstReg); 411 unsigned LMulVal; 412 bool Fractional; 413 std::tie(LMulVal, Fractional) = RISCVVType::decodeVLMUL(LMul); 414 assert(!Fractional && "It is impossible be fractional lmul here."); 415 if (forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NF * LMulVal)) { 416 I = NF - 1; 417 End = -1; 418 Incr = -1; 419 } 420 421 for (; I != End; I += Incr) { 422 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), 423 TRI->getSubReg(DstReg, SubRegIdx + I)); 424 if (UseVMV_V_I) 425 MIB = MIB.add(DefMBBI->getOperand(1)); 426 else 427 MIB = MIB.addReg(TRI->getSubReg(SrcReg, SubRegIdx + I), 428 getKillRegState(KillSrc)); 429 if (UseVMV_V_V) { 430 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 2)); // AVL 431 MIB.add(DefMBBI->getOperand(DefExplicitOpNum - 1)); // SEW 432 MIB.addReg(RISCV::VL, RegState::Implicit); 433 MIB.addReg(RISCV::VTYPE, RegState::Implicit); 434 } 435 } 436 } 437 } else { 438 BuildMI(MBB, MBBI, DL, get(Opc), DstReg) 439 .addReg(SrcReg, getKillRegState(KillSrc)) 440 .addReg(SrcReg, getKillRegState(KillSrc)); 441 } 442 } 443 444 void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 445 MachineBasicBlock::iterator I, 446 Register SrcReg, bool IsKill, int FI, 447 const TargetRegisterClass *RC, 448 const TargetRegisterInfo *TRI) const { 449 DebugLoc DL; 450 if (I != MBB.end()) 451 DL = I->getDebugLoc(); 452 453 MachineFunction *MF = MBB.getParent(); 454 MachineFrameInfo &MFI = MF->getFrameInfo(); 455 456 unsigned Opcode; 457 bool IsScalableVector = true; 458 bool IsZvlsseg = true; 459 if (RISCV::GPRRegClass.hasSubClassEq(RC)) { 460 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ? 461 RISCV::SW : RISCV::SD; 462 IsScalableVector = false; 463 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) { 464 Opcode = RISCV::FSH; 465 IsScalableVector = false; 466 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) { 467 Opcode = RISCV::FSW; 468 IsScalableVector = false; 469 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) { 470 Opcode = RISCV::FSD; 471 IsScalableVector = false; 472 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) { 473 Opcode = RISCV::PseudoVSPILL_M1; 474 IsZvlsseg = false; 475 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) { 476 Opcode = RISCV::PseudoVSPILL_M2; 477 IsZvlsseg = false; 478 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) { 479 Opcode = RISCV::PseudoVSPILL_M4; 480 IsZvlsseg = false; 481 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) { 482 Opcode = RISCV::PseudoVSPILL_M8; 483 IsZvlsseg = false; 484 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) 485 Opcode = RISCV::PseudoVSPILL2_M1; 486 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC)) 487 Opcode = RISCV::PseudoVSPILL2_M2; 488 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC)) 489 Opcode = RISCV::PseudoVSPILL2_M4; 490 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) 491 Opcode = RISCV::PseudoVSPILL3_M1; 492 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC)) 493 Opcode = RISCV::PseudoVSPILL3_M2; 494 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) 495 Opcode = RISCV::PseudoVSPILL4_M1; 496 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC)) 497 Opcode = RISCV::PseudoVSPILL4_M2; 498 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) 499 Opcode = RISCV::PseudoVSPILL5_M1; 500 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) 501 Opcode = RISCV::PseudoVSPILL6_M1; 502 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) 503 Opcode = RISCV::PseudoVSPILL7_M1; 504 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) 505 Opcode = RISCV::PseudoVSPILL8_M1; 506 else 507 llvm_unreachable("Can't store this register to stack slot"); 508 509 if (IsScalableVector) { 510 MachineMemOperand *MMO = MF->getMachineMemOperand( 511 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 512 MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); 513 514 MFI.setStackID(FI, TargetStackID::ScalableVector); 515 auto MIB = BuildMI(MBB, I, DL, get(Opcode)) 516 .addReg(SrcReg, getKillRegState(IsKill)) 517 .addFrameIndex(FI) 518 .addMemOperand(MMO); 519 if (IsZvlsseg) { 520 // For spilling/reloading Zvlsseg registers, append the dummy field for 521 // the scaled vector length. The argument will be used when expanding 522 // these pseudo instructions. 523 MIB.addReg(RISCV::X0); 524 } 525 } else { 526 MachineMemOperand *MMO = MF->getMachineMemOperand( 527 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 528 MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); 529 530 BuildMI(MBB, I, DL, get(Opcode)) 531 .addReg(SrcReg, getKillRegState(IsKill)) 532 .addFrameIndex(FI) 533 .addImm(0) 534 .addMemOperand(MMO); 535 } 536 } 537 538 void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 539 MachineBasicBlock::iterator I, 540 Register DstReg, int FI, 541 const TargetRegisterClass *RC, 542 const TargetRegisterInfo *TRI) const { 543 DebugLoc DL; 544 if (I != MBB.end()) 545 DL = I->getDebugLoc(); 546 547 MachineFunction *MF = MBB.getParent(); 548 MachineFrameInfo &MFI = MF->getFrameInfo(); 549 550 unsigned Opcode; 551 bool IsScalableVector = true; 552 bool IsZvlsseg = true; 553 if (RISCV::GPRRegClass.hasSubClassEq(RC)) { 554 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ? 555 RISCV::LW : RISCV::LD; 556 IsScalableVector = false; 557 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) { 558 Opcode = RISCV::FLH; 559 IsScalableVector = false; 560 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) { 561 Opcode = RISCV::FLW; 562 IsScalableVector = false; 563 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) { 564 Opcode = RISCV::FLD; 565 IsScalableVector = false; 566 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) { 567 Opcode = RISCV::PseudoVRELOAD_M1; 568 IsZvlsseg = false; 569 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) { 570 Opcode = RISCV::PseudoVRELOAD_M2; 571 IsZvlsseg = false; 572 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) { 573 Opcode = RISCV::PseudoVRELOAD_M4; 574 IsZvlsseg = false; 575 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) { 576 Opcode = RISCV::PseudoVRELOAD_M8; 577 IsZvlsseg = false; 578 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC)) 579 Opcode = RISCV::PseudoVRELOAD2_M1; 580 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC)) 581 Opcode = RISCV::PseudoVRELOAD2_M2; 582 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC)) 583 Opcode = RISCV::PseudoVRELOAD2_M4; 584 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC)) 585 Opcode = RISCV::PseudoVRELOAD3_M1; 586 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC)) 587 Opcode = RISCV::PseudoVRELOAD3_M2; 588 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC)) 589 Opcode = RISCV::PseudoVRELOAD4_M1; 590 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC)) 591 Opcode = RISCV::PseudoVRELOAD4_M2; 592 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC)) 593 Opcode = RISCV::PseudoVRELOAD5_M1; 594 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC)) 595 Opcode = RISCV::PseudoVRELOAD6_M1; 596 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC)) 597 Opcode = RISCV::PseudoVRELOAD7_M1; 598 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC)) 599 Opcode = RISCV::PseudoVRELOAD8_M1; 600 else 601 llvm_unreachable("Can't load this register from stack slot"); 602 603 if (IsScalableVector) { 604 MachineMemOperand *MMO = MF->getMachineMemOperand( 605 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 606 MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); 607 608 MFI.setStackID(FI, TargetStackID::ScalableVector); 609 auto MIB = BuildMI(MBB, I, DL, get(Opcode), DstReg) 610 .addFrameIndex(FI) 611 .addMemOperand(MMO); 612 if (IsZvlsseg) { 613 // For spilling/reloading Zvlsseg registers, append the dummy field for 614 // the scaled vector length. The argument will be used when expanding 615 // these pseudo instructions. 616 MIB.addReg(RISCV::X0); 617 } 618 } else { 619 MachineMemOperand *MMO = MF->getMachineMemOperand( 620 MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 621 MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); 622 623 BuildMI(MBB, I, DL, get(Opcode), DstReg) 624 .addFrameIndex(FI) 625 .addImm(0) 626 .addMemOperand(MMO); 627 } 628 } 629 630 void RISCVInstrInfo::movImm(MachineBasicBlock &MBB, 631 MachineBasicBlock::iterator MBBI, 632 const DebugLoc &DL, Register DstReg, uint64_t Val, 633 MachineInstr::MIFlag Flag) const { 634 MachineFunction *MF = MBB.getParent(); 635 MachineRegisterInfo &MRI = MF->getRegInfo(); 636 Register SrcReg = RISCV::X0; 637 Register Result = MRI.createVirtualRegister(&RISCV::GPRRegClass); 638 unsigned Num = 0; 639 640 if (!STI.is64Bit() && !isInt<32>(Val)) 641 report_fatal_error("Should only materialize 32-bit constants for RV32"); 642 643 RISCVMatInt::InstSeq Seq = 644 RISCVMatInt::generateInstSeq(Val, STI.getFeatureBits()); 645 assert(!Seq.empty()); 646 647 for (RISCVMatInt::Inst &Inst : Seq) { 648 // Write the final result to DstReg if it's the last instruction in the Seq. 649 // Otherwise, write the result to the temp register. 650 if (++Num == Seq.size()) 651 Result = DstReg; 652 653 if (Inst.Opc == RISCV::LUI) { 654 BuildMI(MBB, MBBI, DL, get(RISCV::LUI), Result) 655 .addImm(Inst.Imm) 656 .setMIFlag(Flag); 657 } else if (Inst.Opc == RISCV::ADD_UW) { 658 BuildMI(MBB, MBBI, DL, get(RISCV::ADD_UW), Result) 659 .addReg(SrcReg, RegState::Kill) 660 .addReg(RISCV::X0) 661 .setMIFlag(Flag); 662 } else if (Inst.Opc == RISCV::SH1ADD || Inst.Opc == RISCV::SH2ADD || 663 Inst.Opc == RISCV::SH3ADD) { 664 BuildMI(MBB, MBBI, DL, get(Inst.Opc), Result) 665 .addReg(SrcReg, RegState::Kill) 666 .addReg(SrcReg, RegState::Kill) 667 .setMIFlag(Flag); 668 } else { 669 BuildMI(MBB, MBBI, DL, get(Inst.Opc), Result) 670 .addReg(SrcReg, RegState::Kill) 671 .addImm(Inst.Imm) 672 .setMIFlag(Flag); 673 } 674 // Only the first instruction has X0 as its source. 675 SrcReg = Result; 676 } 677 } 678 679 static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) { 680 switch (Opc) { 681 default: 682 return RISCVCC::COND_INVALID; 683 case RISCV::BEQ: 684 return RISCVCC::COND_EQ; 685 case RISCV::BNE: 686 return RISCVCC::COND_NE; 687 case RISCV::BLT: 688 return RISCVCC::COND_LT; 689 case RISCV::BGE: 690 return RISCVCC::COND_GE; 691 case RISCV::BLTU: 692 return RISCVCC::COND_LTU; 693 case RISCV::BGEU: 694 return RISCVCC::COND_GEU; 695 } 696 } 697 698 // The contents of values added to Cond are not examined outside of 699 // RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we 700 // push BranchOpcode, Reg1, Reg2. 701 static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target, 702 SmallVectorImpl<MachineOperand> &Cond) { 703 // Block ends with fall-through condbranch. 704 assert(LastInst.getDesc().isConditionalBranch() && 705 "Unknown conditional branch"); 706 Target = LastInst.getOperand(2).getMBB(); 707 unsigned CC = getCondFromBranchOpc(LastInst.getOpcode()); 708 Cond.push_back(MachineOperand::CreateImm(CC)); 709 Cond.push_back(LastInst.getOperand(0)); 710 Cond.push_back(LastInst.getOperand(1)); 711 } 712 713 const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const { 714 switch (CC) { 715 default: 716 llvm_unreachable("Unknown condition code!"); 717 case RISCVCC::COND_EQ: 718 return get(RISCV::BEQ); 719 case RISCVCC::COND_NE: 720 return get(RISCV::BNE); 721 case RISCVCC::COND_LT: 722 return get(RISCV::BLT); 723 case RISCVCC::COND_GE: 724 return get(RISCV::BGE); 725 case RISCVCC::COND_LTU: 726 return get(RISCV::BLTU); 727 case RISCVCC::COND_GEU: 728 return get(RISCV::BGEU); 729 } 730 } 731 732 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) { 733 switch (CC) { 734 default: 735 llvm_unreachable("Unrecognized conditional branch"); 736 case RISCVCC::COND_EQ: 737 return RISCVCC::COND_NE; 738 case RISCVCC::COND_NE: 739 return RISCVCC::COND_EQ; 740 case RISCVCC::COND_LT: 741 return RISCVCC::COND_GE; 742 case RISCVCC::COND_GE: 743 return RISCVCC::COND_LT; 744 case RISCVCC::COND_LTU: 745 return RISCVCC::COND_GEU; 746 case RISCVCC::COND_GEU: 747 return RISCVCC::COND_LTU; 748 } 749 } 750 751 bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 752 MachineBasicBlock *&TBB, 753 MachineBasicBlock *&FBB, 754 SmallVectorImpl<MachineOperand> &Cond, 755 bool AllowModify) const { 756 TBB = FBB = nullptr; 757 Cond.clear(); 758 759 // If the block has no terminators, it just falls into the block after it. 760 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 761 if (I == MBB.end() || !isUnpredicatedTerminator(*I)) 762 return false; 763 764 // Count the number of terminators and find the first unconditional or 765 // indirect branch. 766 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end(); 767 int NumTerminators = 0; 768 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J); 769 J++) { 770 NumTerminators++; 771 if (J->getDesc().isUnconditionalBranch() || 772 J->getDesc().isIndirectBranch()) { 773 FirstUncondOrIndirectBr = J.getReverse(); 774 } 775 } 776 777 // If AllowModify is true, we can erase any terminators after 778 // FirstUncondOrIndirectBR. 779 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) { 780 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) { 781 std::next(FirstUncondOrIndirectBr)->eraseFromParent(); 782 NumTerminators--; 783 } 784 I = FirstUncondOrIndirectBr; 785 } 786 787 // We can't handle blocks that end in an indirect branch. 788 if (I->getDesc().isIndirectBranch()) 789 return true; 790 791 // We can't handle blocks with more than 2 terminators. 792 if (NumTerminators > 2) 793 return true; 794 795 // Handle a single unconditional branch. 796 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) { 797 TBB = getBranchDestBlock(*I); 798 return false; 799 } 800 801 // Handle a single conditional branch. 802 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) { 803 parseCondBranch(*I, TBB, Cond); 804 return false; 805 } 806 807 // Handle a conditional branch followed by an unconditional branch. 808 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() && 809 I->getDesc().isUnconditionalBranch()) { 810 parseCondBranch(*std::prev(I), TBB, Cond); 811 FBB = getBranchDestBlock(*I); 812 return false; 813 } 814 815 // Otherwise, we can't handle this. 816 return true; 817 } 818 819 unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB, 820 int *BytesRemoved) const { 821 if (BytesRemoved) 822 *BytesRemoved = 0; 823 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 824 if (I == MBB.end()) 825 return 0; 826 827 if (!I->getDesc().isUnconditionalBranch() && 828 !I->getDesc().isConditionalBranch()) 829 return 0; 830 831 // Remove the branch. 832 if (BytesRemoved) 833 *BytesRemoved += getInstSizeInBytes(*I); 834 I->eraseFromParent(); 835 836 I = MBB.end(); 837 838 if (I == MBB.begin()) 839 return 1; 840 --I; 841 if (!I->getDesc().isConditionalBranch()) 842 return 1; 843 844 // Remove the branch. 845 if (BytesRemoved) 846 *BytesRemoved += getInstSizeInBytes(*I); 847 I->eraseFromParent(); 848 return 2; 849 } 850 851 // Inserts a branch into the end of the specific MachineBasicBlock, returning 852 // the number of instructions inserted. 853 unsigned RISCVInstrInfo::insertBranch( 854 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, 855 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const { 856 if (BytesAdded) 857 *BytesAdded = 0; 858 859 // Shouldn't be a fall through. 860 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 861 assert((Cond.size() == 3 || Cond.size() == 0) && 862 "RISCV branch conditions have two components!"); 863 864 // Unconditional branch. 865 if (Cond.empty()) { 866 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB); 867 if (BytesAdded) 868 *BytesAdded += getInstSizeInBytes(MI); 869 return 1; 870 } 871 872 // Either a one or two-way conditional branch. 873 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm()); 874 MachineInstr &CondMI = 875 *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB); 876 if (BytesAdded) 877 *BytesAdded += getInstSizeInBytes(CondMI); 878 879 // One-way conditional branch. 880 if (!FBB) 881 return 1; 882 883 // Two-way conditional branch. 884 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB); 885 if (BytesAdded) 886 *BytesAdded += getInstSizeInBytes(MI); 887 return 2; 888 } 889 890 void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 891 MachineBasicBlock &DestBB, 892 MachineBasicBlock &RestoreBB, 893 const DebugLoc &DL, int64_t BrOffset, 894 RegScavenger *RS) const { 895 assert(RS && "RegScavenger required for long branching"); 896 assert(MBB.empty() && 897 "new block should be inserted for expanding unconditional branch"); 898 assert(MBB.pred_size() == 1); 899 900 MachineFunction *MF = MBB.getParent(); 901 MachineRegisterInfo &MRI = MF->getRegInfo(); 902 903 if (!isInt<32>(BrOffset)) 904 report_fatal_error( 905 "Branch offsets outside of the signed 32-bit range not supported"); 906 907 // FIXME: A virtual register must be used initially, as the register 908 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch 909 // uses the same workaround). 910 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass); 911 auto II = MBB.end(); 912 913 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump)) 914 .addReg(ScratchReg, RegState::Define | RegState::Dead) 915 .addMBB(&DestBB, RISCVII::MO_CALL); 916 917 RS->enterBasicBlockEnd(MBB); 918 Register Scav = RS->scavengeRegisterBackwards(RISCV::GPRRegClass, 919 MI.getIterator(), false, 0); 920 // TODO: The case when there is no scavenged register needs special handling. 921 assert(Scav != RISCV::NoRegister && "No register is scavenged!"); 922 MRI.replaceRegWith(ScratchReg, Scav); 923 MRI.clearVirtRegs(); 924 RS->setRegUsed(Scav); 925 } 926 927 bool RISCVInstrInfo::reverseBranchCondition( 928 SmallVectorImpl<MachineOperand> &Cond) const { 929 assert((Cond.size() == 3) && "Invalid branch condition!"); 930 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm()); 931 Cond[0].setImm(getOppositeBranchCondition(CC)); 932 return false; 933 } 934 935 MachineBasicBlock * 936 RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { 937 assert(MI.getDesc().isBranch() && "Unexpected opcode!"); 938 // The branch target is always the last operand. 939 int NumOp = MI.getNumExplicitOperands(); 940 return MI.getOperand(NumOp - 1).getMBB(); 941 } 942 943 bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 944 int64_t BrOffset) const { 945 unsigned XLen = STI.getXLen(); 946 // Ideally we could determine the supported branch offset from the 947 // RISCVII::FormMask, but this can't be used for Pseudo instructions like 948 // PseudoBR. 949 switch (BranchOp) { 950 default: 951 llvm_unreachable("Unexpected opcode!"); 952 case RISCV::BEQ: 953 case RISCV::BNE: 954 case RISCV::BLT: 955 case RISCV::BGE: 956 case RISCV::BLTU: 957 case RISCV::BGEU: 958 return isIntN(13, BrOffset); 959 case RISCV::JAL: 960 case RISCV::PseudoBR: 961 return isIntN(21, BrOffset); 962 case RISCV::PseudoJump: 963 return isIntN(32, SignExtend64(BrOffset + 0x800, XLen)); 964 } 965 } 966 967 unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 968 if (MI.isMetaInstruction()) 969 return 0; 970 971 unsigned Opcode = MI.getOpcode(); 972 973 if (Opcode == TargetOpcode::INLINEASM || 974 Opcode == TargetOpcode::INLINEASM_BR) { 975 const MachineFunction &MF = *MI.getParent()->getParent(); 976 const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget()); 977 return getInlineAsmLength(MI.getOperand(0).getSymbolName(), 978 *TM.getMCAsmInfo()); 979 } 980 981 if (MI.getParent() && MI.getParent()->getParent()) { 982 const auto MF = MI.getMF(); 983 const auto &TM = static_cast<const RISCVTargetMachine &>(MF->getTarget()); 984 const MCRegisterInfo &MRI = *TM.getMCRegisterInfo(); 985 const MCSubtargetInfo &STI = *TM.getMCSubtargetInfo(); 986 const RISCVSubtarget &ST = MF->getSubtarget<RISCVSubtarget>(); 987 if (isCompressibleInst(MI, &ST, MRI, STI)) 988 return 2; 989 } 990 return get(Opcode).getSize(); 991 } 992 993 bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const { 994 const unsigned Opcode = MI.getOpcode(); 995 switch (Opcode) { 996 default: 997 break; 998 case RISCV::FSGNJ_D: 999 case RISCV::FSGNJ_S: 1000 case RISCV::FSGNJ_H: 1001 // The canonical floating-point move is fsgnj rd, rs, rs. 1002 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && 1003 MI.getOperand(1).getReg() == MI.getOperand(2).getReg(); 1004 case RISCV::ADDI: 1005 case RISCV::ORI: 1006 case RISCV::XORI: 1007 return (MI.getOperand(1).isReg() && 1008 MI.getOperand(1).getReg() == RISCV::X0) || 1009 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0); 1010 } 1011 return MI.isAsCheapAsAMove(); 1012 } 1013 1014 Optional<DestSourcePair> 1015 RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 1016 if (MI.isMoveReg()) 1017 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1018 switch (MI.getOpcode()) { 1019 default: 1020 break; 1021 case RISCV::ADDI: 1022 // Operand 1 can be a frameindex but callers expect registers 1023 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() && 1024 MI.getOperand(2).getImm() == 0) 1025 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1026 break; 1027 case RISCV::FSGNJ_D: 1028 case RISCV::FSGNJ_S: 1029 case RISCV::FSGNJ_H: 1030 // The canonical floating-point move is fsgnj rd, rs, rs. 1031 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && 1032 MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 1033 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1034 break; 1035 } 1036 return None; 1037 } 1038 1039 bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI, 1040 StringRef &ErrInfo) const { 1041 const MCInstrInfo *MCII = STI.getInstrInfo(); 1042 MCInstrDesc const &Desc = MCII->get(MI.getOpcode()); 1043 1044 for (auto &OI : enumerate(Desc.operands())) { 1045 unsigned OpType = OI.value().OperandType; 1046 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM && 1047 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) { 1048 const MachineOperand &MO = MI.getOperand(OI.index()); 1049 if (MO.isImm()) { 1050 int64_t Imm = MO.getImm(); 1051 bool Ok; 1052 switch (OpType) { 1053 default: 1054 llvm_unreachable("Unexpected operand type"); 1055 case RISCVOp::OPERAND_UIMM2: 1056 Ok = isUInt<2>(Imm); 1057 break; 1058 case RISCVOp::OPERAND_UIMM3: 1059 Ok = isUInt<3>(Imm); 1060 break; 1061 case RISCVOp::OPERAND_UIMM4: 1062 Ok = isUInt<4>(Imm); 1063 break; 1064 case RISCVOp::OPERAND_UIMM5: 1065 Ok = isUInt<5>(Imm); 1066 break; 1067 case RISCVOp::OPERAND_UIMM7: 1068 Ok = isUInt<7>(Imm); 1069 break; 1070 case RISCVOp::OPERAND_UIMM12: 1071 Ok = isUInt<12>(Imm); 1072 break; 1073 case RISCVOp::OPERAND_SIMM12: 1074 Ok = isInt<12>(Imm); 1075 break; 1076 case RISCVOp::OPERAND_UIMM20: 1077 Ok = isUInt<20>(Imm); 1078 break; 1079 case RISCVOp::OPERAND_UIMMLOG2XLEN: 1080 if (STI.getTargetTriple().isArch64Bit()) 1081 Ok = isUInt<6>(Imm); 1082 else 1083 Ok = isUInt<5>(Imm); 1084 break; 1085 case RISCVOp::OPERAND_RVKRNUM: 1086 Ok = Imm >= 0 && Imm <= 10; 1087 break; 1088 } 1089 if (!Ok) { 1090 ErrInfo = "Invalid immediate"; 1091 return false; 1092 } 1093 } 1094 } 1095 } 1096 1097 return true; 1098 } 1099 1100 // Return true if get the base operand, byte offset of an instruction and the 1101 // memory width. Width is the size of memory that is being loaded/stored. 1102 bool RISCVInstrInfo::getMemOperandWithOffsetWidth( 1103 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, 1104 unsigned &Width, const TargetRegisterInfo *TRI) const { 1105 if (!LdSt.mayLoadOrStore()) 1106 return false; 1107 1108 // Here we assume the standard RISC-V ISA, which uses a base+offset 1109 // addressing mode. You'll need to relax these conditions to support custom 1110 // load/stores instructions. 1111 if (LdSt.getNumExplicitOperands() != 3) 1112 return false; 1113 if (!LdSt.getOperand(1).isReg() || !LdSt.getOperand(2).isImm()) 1114 return false; 1115 1116 if (!LdSt.hasOneMemOperand()) 1117 return false; 1118 1119 Width = (*LdSt.memoperands_begin())->getSize(); 1120 BaseReg = &LdSt.getOperand(1); 1121 Offset = LdSt.getOperand(2).getImm(); 1122 return true; 1123 } 1124 1125 bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint( 1126 const MachineInstr &MIa, const MachineInstr &MIb) const { 1127 assert(MIa.mayLoadOrStore() && "MIa must be a load or store."); 1128 assert(MIb.mayLoadOrStore() && "MIb must be a load or store."); 1129 1130 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() || 1131 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 1132 return false; 1133 1134 // Retrieve the base register, offset from the base register and width. Width 1135 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If 1136 // base registers are identical, and the offset of a lower memory access + 1137 // the width doesn't overlap the offset of a higher memory access, 1138 // then the memory accesses are different. 1139 const TargetRegisterInfo *TRI = STI.getRegisterInfo(); 1140 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; 1141 int64_t OffsetA = 0, OffsetB = 0; 1142 unsigned int WidthA = 0, WidthB = 0; 1143 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && 1144 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { 1145 if (BaseOpA->isIdenticalTo(*BaseOpB)) { 1146 int LowOffset = std::min(OffsetA, OffsetB); 1147 int HighOffset = std::max(OffsetA, OffsetB); 1148 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 1149 if (LowOffset + LowWidth <= HighOffset) 1150 return true; 1151 } 1152 } 1153 return false; 1154 } 1155 1156 std::pair<unsigned, unsigned> 1157 RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 1158 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK; 1159 return std::make_pair(TF & Mask, TF & ~Mask); 1160 } 1161 1162 ArrayRef<std::pair<unsigned, const char *>> 1163 RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 1164 using namespace RISCVII; 1165 static const std::pair<unsigned, const char *> TargetFlags[] = { 1166 {MO_CALL, "riscv-call"}, 1167 {MO_PLT, "riscv-plt"}, 1168 {MO_LO, "riscv-lo"}, 1169 {MO_HI, "riscv-hi"}, 1170 {MO_PCREL_LO, "riscv-pcrel-lo"}, 1171 {MO_PCREL_HI, "riscv-pcrel-hi"}, 1172 {MO_GOT_HI, "riscv-got-hi"}, 1173 {MO_TPREL_LO, "riscv-tprel-lo"}, 1174 {MO_TPREL_HI, "riscv-tprel-hi"}, 1175 {MO_TPREL_ADD, "riscv-tprel-add"}, 1176 {MO_TLS_GOT_HI, "riscv-tls-got-hi"}, 1177 {MO_TLS_GD_HI, "riscv-tls-gd-hi"}}; 1178 return makeArrayRef(TargetFlags); 1179 } 1180 bool RISCVInstrInfo::isFunctionSafeToOutlineFrom( 1181 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const { 1182 const Function &F = MF.getFunction(); 1183 1184 // Can F be deduplicated by the linker? If it can, don't outline from it. 1185 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 1186 return false; 1187 1188 // Don't outline from functions with section markings; the program could 1189 // expect that all the code is in the named section. 1190 if (F.hasSection()) 1191 return false; 1192 1193 // It's safe to outline from MF. 1194 return true; 1195 } 1196 1197 bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, 1198 unsigned &Flags) const { 1199 // More accurate safety checking is done in getOutliningCandidateInfo. 1200 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags); 1201 } 1202 1203 // Enum values indicating how an outlined call should be constructed. 1204 enum MachineOutlinerConstructionID { 1205 MachineOutlinerDefault 1206 }; 1207 1208 outliner::OutlinedFunction RISCVInstrInfo::getOutliningCandidateInfo( 1209 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 1210 1211 // First we need to filter out candidates where the X5 register (IE t0) can't 1212 // be used to setup the function call. 1213 auto CannotInsertCall = [](outliner::Candidate &C) { 1214 const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo(); 1215 1216 C.initLRU(*TRI); 1217 LiveRegUnits LRU = C.LRU; 1218 return !LRU.available(RISCV::X5); 1219 }; 1220 1221 llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall); 1222 1223 // If the sequence doesn't have enough candidates left, then we're done. 1224 if (RepeatedSequenceLocs.size() < 2) 1225 return outliner::OutlinedFunction(); 1226 1227 unsigned SequenceSize = 0; 1228 1229 auto I = RepeatedSequenceLocs[0].front(); 1230 auto E = std::next(RepeatedSequenceLocs[0].back()); 1231 for (; I != E; ++I) 1232 SequenceSize += getInstSizeInBytes(*I); 1233 1234 // call t0, function = 8 bytes. 1235 unsigned CallOverhead = 8; 1236 for (auto &C : RepeatedSequenceLocs) 1237 C.setCallInfo(MachineOutlinerDefault, CallOverhead); 1238 1239 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled. 1240 unsigned FrameOverhead = 4; 1241 if (RepeatedSequenceLocs[0].getMF()->getSubtarget() 1242 .getFeatureBits()[RISCV::FeatureStdExtC]) 1243 FrameOverhead = 2; 1244 1245 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1246 FrameOverhead, MachineOutlinerDefault); 1247 } 1248 1249 outliner::InstrType 1250 RISCVInstrInfo::getOutliningType(MachineBasicBlock::iterator &MBBI, 1251 unsigned Flags) const { 1252 MachineInstr &MI = *MBBI; 1253 MachineBasicBlock *MBB = MI.getParent(); 1254 const TargetRegisterInfo *TRI = 1255 MBB->getParent()->getSubtarget().getRegisterInfo(); 1256 1257 // Positions generally can't safely be outlined. 1258 if (MI.isPosition()) { 1259 // We can manually strip out CFI instructions later. 1260 if (MI.isCFIInstruction()) 1261 return outliner::InstrType::Invisible; 1262 1263 return outliner::InstrType::Illegal; 1264 } 1265 1266 // Don't trust the user to write safe inline assembly. 1267 if (MI.isInlineAsm()) 1268 return outliner::InstrType::Illegal; 1269 1270 // We can't outline branches to other basic blocks. 1271 if (MI.isTerminator() && !MBB->succ_empty()) 1272 return outliner::InstrType::Illegal; 1273 1274 // We need support for tail calls to outlined functions before return 1275 // statements can be allowed. 1276 if (MI.isReturn()) 1277 return outliner::InstrType::Illegal; 1278 1279 // Don't allow modifying the X5 register which we use for return addresses for 1280 // these outlined functions. 1281 if (MI.modifiesRegister(RISCV::X5, TRI) || 1282 MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5)) 1283 return outliner::InstrType::Illegal; 1284 1285 // Make sure the operands don't reference something unsafe. 1286 for (const auto &MO : MI.operands()) 1287 if (MO.isMBB() || MO.isBlockAddress() || MO.isCPI() || MO.isJTI()) 1288 return outliner::InstrType::Illegal; 1289 1290 // Don't allow instructions which won't be materialized to impact outlining 1291 // analysis. 1292 if (MI.isMetaInstruction()) 1293 return outliner::InstrType::Invisible; 1294 1295 return outliner::InstrType::Legal; 1296 } 1297 1298 void RISCVInstrInfo::buildOutlinedFrame( 1299 MachineBasicBlock &MBB, MachineFunction &MF, 1300 const outliner::OutlinedFunction &OF) const { 1301 1302 // Strip out any CFI instructions 1303 bool Changed = true; 1304 while (Changed) { 1305 Changed = false; 1306 auto I = MBB.begin(); 1307 auto E = MBB.end(); 1308 for (; I != E; ++I) { 1309 if (I->isCFIInstruction()) { 1310 I->removeFromParent(); 1311 Changed = true; 1312 break; 1313 } 1314 } 1315 } 1316 1317 MBB.addLiveIn(RISCV::X5); 1318 1319 // Add in a return instruction to the end of the outlined frame. 1320 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR)) 1321 .addReg(RISCV::X0, RegState::Define) 1322 .addReg(RISCV::X5) 1323 .addImm(0)); 1324 } 1325 1326 MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall( 1327 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, 1328 MachineFunction &MF, const outliner::Candidate &C) const { 1329 1330 // Add in a call instruction to the outlined function at the given location. 1331 It = MBB.insert(It, 1332 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5) 1333 .addGlobalAddress(M.getNamedValue(MF.getName()), 0, 1334 RISCVII::MO_CALL)); 1335 return It; 1336 } 1337 1338 // clang-format off 1339 #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL) \ 1340 RISCV::PseudoV##OP##_##TYPE##_##LMUL 1341 1342 #define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE) \ 1343 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1): \ 1344 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2): \ 1345 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4): \ 1346 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8) 1347 1348 #define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE) \ 1349 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2): \ 1350 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE) 1351 1352 #define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE) \ 1353 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4): \ 1354 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE) 1355 1356 #define CASE_VFMA_OPCODE_LMULS(OP, TYPE) \ 1357 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8): \ 1358 case CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE) 1359 1360 #define CASE_VFMA_SPLATS(OP) \ 1361 CASE_VFMA_OPCODE_LMULS_MF4(OP, VF16): \ 1362 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VF32): \ 1363 case CASE_VFMA_OPCODE_LMULS_M1(OP, VF64) 1364 // clang-format on 1365 1366 bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 1367 unsigned &SrcOpIdx1, 1368 unsigned &SrcOpIdx2) const { 1369 const MCInstrDesc &Desc = MI.getDesc(); 1370 if (!Desc.isCommutable()) 1371 return false; 1372 1373 switch (MI.getOpcode()) { 1374 case CASE_VFMA_SPLATS(FMADD): 1375 case CASE_VFMA_SPLATS(FMSUB): 1376 case CASE_VFMA_SPLATS(FMACC): 1377 case CASE_VFMA_SPLATS(FMSAC): 1378 case CASE_VFMA_SPLATS(FNMADD): 1379 case CASE_VFMA_SPLATS(FNMSUB): 1380 case CASE_VFMA_SPLATS(FNMACC): 1381 case CASE_VFMA_SPLATS(FNMSAC): 1382 case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV): 1383 case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV): 1384 case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV): 1385 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV): 1386 case CASE_VFMA_OPCODE_LMULS(MADD, VX): 1387 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX): 1388 case CASE_VFMA_OPCODE_LMULS(MACC, VX): 1389 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): 1390 case CASE_VFMA_OPCODE_LMULS(MACC, VV): 1391 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { 1392 // If the tail policy is undisturbed we can't commute. 1393 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); 1394 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) 1395 return false; 1396 1397 // For these instructions we can only swap operand 1 and operand 3 by 1398 // changing the opcode. 1399 unsigned CommutableOpIdx1 = 1; 1400 unsigned CommutableOpIdx2 = 3; 1401 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 1402 CommutableOpIdx2)) 1403 return false; 1404 return true; 1405 } 1406 case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV): 1407 case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV): 1408 case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV): 1409 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV): 1410 case CASE_VFMA_OPCODE_LMULS(MADD, VV): 1411 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { 1412 // If the tail policy is undisturbed we can't commute. 1413 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags)); 1414 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0) 1415 return false; 1416 1417 // For these instructions we have more freedom. We can commute with the 1418 // other multiplicand or with the addend/subtrahend/minuend. 1419 1420 // Any fixed operand must be from source 1, 2 or 3. 1421 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3) 1422 return false; 1423 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3) 1424 return false; 1425 1426 // It both ops are fixed one must be the tied source. 1427 if (SrcOpIdx1 != CommuteAnyOperandIndex && 1428 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1) 1429 return false; 1430 1431 // Look for two different register operands assumed to be commutable 1432 // regardless of the FMA opcode. The FMA opcode is adjusted later if 1433 // needed. 1434 if (SrcOpIdx1 == CommuteAnyOperandIndex || 1435 SrcOpIdx2 == CommuteAnyOperandIndex) { 1436 // At least one of operands to be commuted is not specified and 1437 // this method is free to choose appropriate commutable operands. 1438 unsigned CommutableOpIdx1 = SrcOpIdx1; 1439 if (SrcOpIdx1 == SrcOpIdx2) { 1440 // Both of operands are not fixed. Set one of commutable 1441 // operands to the tied source. 1442 CommutableOpIdx1 = 1; 1443 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) { 1444 // Only one of the operands is not fixed. 1445 CommutableOpIdx1 = SrcOpIdx2; 1446 } 1447 1448 // CommutableOpIdx1 is well defined now. Let's choose another commutable 1449 // operand and assign its index to CommutableOpIdx2. 1450 unsigned CommutableOpIdx2; 1451 if (CommutableOpIdx1 != 1) { 1452 // If we haven't already used the tied source, we must use it now. 1453 CommutableOpIdx2 = 1; 1454 } else { 1455 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg(); 1456 1457 // The commuted operands should have different registers. 1458 // Otherwise, the commute transformation does not change anything and 1459 // is useless. We use this as a hint to make our decision. 1460 if (Op1Reg != MI.getOperand(2).getReg()) 1461 CommutableOpIdx2 = 2; 1462 else 1463 CommutableOpIdx2 = 3; 1464 } 1465 1466 // Assign the found pair of commutable indices to SrcOpIdx1 and 1467 // SrcOpIdx2 to return those values. 1468 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1, 1469 CommutableOpIdx2)) 1470 return false; 1471 } 1472 1473 return true; 1474 } 1475 } 1476 1477 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 1478 } 1479 1480 #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \ 1481 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \ 1482 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \ 1483 break; 1484 1485 #define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \ 1486 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \ 1487 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \ 1488 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \ 1489 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8) 1490 1491 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \ 1492 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \ 1493 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) 1494 1495 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \ 1496 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \ 1497 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) 1498 1499 #define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \ 1500 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \ 1501 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) 1502 1503 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \ 1504 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VF16) \ 1505 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VF32) \ 1506 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VF64) 1507 1508 MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI, 1509 bool NewMI, 1510 unsigned OpIdx1, 1511 unsigned OpIdx2) const { 1512 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 1513 if (NewMI) 1514 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 1515 return MI; 1516 }; 1517 1518 switch (MI.getOpcode()) { 1519 case CASE_VFMA_SPLATS(FMACC): 1520 case CASE_VFMA_SPLATS(FMADD): 1521 case CASE_VFMA_SPLATS(FMSAC): 1522 case CASE_VFMA_SPLATS(FMSUB): 1523 case CASE_VFMA_SPLATS(FNMACC): 1524 case CASE_VFMA_SPLATS(FNMADD): 1525 case CASE_VFMA_SPLATS(FNMSAC): 1526 case CASE_VFMA_SPLATS(FNMSUB): 1527 case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV): 1528 case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV): 1529 case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV): 1530 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV): 1531 case CASE_VFMA_OPCODE_LMULS(MADD, VX): 1532 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX): 1533 case CASE_VFMA_OPCODE_LMULS(MACC, VX): 1534 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX): 1535 case CASE_VFMA_OPCODE_LMULS(MACC, VV): 1536 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): { 1537 // It only make sense to toggle these between clobbering the 1538 // addend/subtrahend/minuend one of the multiplicands. 1539 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index"); 1540 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index"); 1541 unsigned Opc; 1542 switch (MI.getOpcode()) { 1543 default: 1544 llvm_unreachable("Unexpected opcode"); 1545 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD) 1546 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC) 1547 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB) 1548 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC) 1549 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD) 1550 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC) 1551 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB) 1552 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC) 1553 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMACC, FMADD, VV) 1554 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSAC, FMSUB, VV) 1555 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMACC, FNMADD, VV) 1556 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSAC, FNMSUB, VV) 1557 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX) 1558 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX) 1559 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX) 1560 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX) 1561 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV) 1562 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV) 1563 } 1564 1565 auto &WorkingMI = cloneIfNew(MI); 1566 WorkingMI.setDesc(get(Opc)); 1567 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1568 OpIdx1, OpIdx2); 1569 } 1570 case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV): 1571 case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV): 1572 case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV): 1573 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV): 1574 case CASE_VFMA_OPCODE_LMULS(MADD, VV): 1575 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): { 1576 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index"); 1577 // If one of the operands, is the addend we need to change opcode. 1578 // Otherwise we're just swapping 2 of the multiplicands. 1579 if (OpIdx1 == 3 || OpIdx2 == 3) { 1580 unsigned Opc; 1581 switch (MI.getOpcode()) { 1582 default: 1583 llvm_unreachable("Unexpected opcode"); 1584 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMADD, FMACC, VV) 1585 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSUB, FMSAC, VV) 1586 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMADD, FNMACC, VV) 1587 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSUB, FNMSAC, VV) 1588 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV) 1589 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV) 1590 } 1591 1592 auto &WorkingMI = cloneIfNew(MI); 1593 WorkingMI.setDesc(get(Opc)); 1594 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1595 OpIdx1, OpIdx2); 1596 } 1597 // Let the default code handle it. 1598 break; 1599 } 1600 } 1601 1602 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 1603 } 1604 1605 #undef CASE_VFMA_CHANGE_OPCODE_SPLATS 1606 #undef CASE_VFMA_CHANGE_OPCODE_LMULS 1607 #undef CASE_VFMA_CHANGE_OPCODE_COMMON 1608 #undef CASE_VFMA_SPLATS 1609 #undef CASE_VFMA_OPCODE_LMULS 1610 #undef CASE_VFMA_OPCODE_COMMON 1611 1612 // clang-format off 1613 #define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \ 1614 RISCV::PseudoV##OP##_##LMUL##_TIED 1615 1616 #define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \ 1617 CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \ 1618 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \ 1619 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \ 1620 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \ 1621 case CASE_WIDEOP_OPCODE_COMMON(OP, M4) 1622 1623 #define CASE_WIDEOP_OPCODE_LMULS(OP) \ 1624 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \ 1625 case CASE_WIDEOP_OPCODE_LMULS_MF4(OP) 1626 // clang-format on 1627 1628 #define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \ 1629 case RISCV::PseudoV##OP##_##LMUL##_TIED: \ 1630 NewOpc = RISCV::PseudoV##OP##_##LMUL; \ 1631 break; 1632 1633 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \ 1634 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \ 1635 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \ 1636 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \ 1637 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \ 1638 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4) 1639 1640 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \ 1641 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \ 1642 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) 1643 1644 MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI, 1645 LiveVariables *LV, 1646 LiveIntervals *LIS) const { 1647 switch (MI.getOpcode()) { 1648 default: 1649 break; 1650 case CASE_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV): 1651 case CASE_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): 1652 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV): 1653 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV): 1654 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV): 1655 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): { 1656 // clang-format off 1657 unsigned NewOpc; 1658 switch (MI.getOpcode()) { 1659 default: 1660 llvm_unreachable("Unexpected opcode"); 1661 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV) 1662 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV) 1663 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV) 1664 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV) 1665 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV) 1666 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV) 1667 } 1668 // clang-format on 1669 1670 MachineBasicBlock &MBB = *MI.getParent(); 1671 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 1672 .add(MI.getOperand(0)) 1673 .add(MI.getOperand(1)) 1674 .add(MI.getOperand(2)) 1675 .add(MI.getOperand(3)) 1676 .add(MI.getOperand(4)); 1677 MIB.copyImplicitOps(MI); 1678 1679 if (LV) { 1680 unsigned NumOps = MI.getNumOperands(); 1681 for (unsigned I = 1; I < NumOps; ++I) { 1682 MachineOperand &Op = MI.getOperand(I); 1683 if (Op.isReg() && Op.isKill()) 1684 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 1685 } 1686 } 1687 1688 if (LIS) { 1689 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB); 1690 1691 if (MI.getOperand(0).isEarlyClobber()) { 1692 // Use operand 1 was tied to early-clobber def operand 0, so its live 1693 // interval could have ended at an early-clobber slot. Now they are not 1694 // tied we need to update it to the normal register slot. 1695 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg()); 1696 LiveRange::Segment *S = LI.getSegmentContaining(Idx); 1697 if (S->end == Idx.getRegSlot(true)) 1698 S->end = Idx.getRegSlot(); 1699 } 1700 } 1701 1702 return MIB; 1703 } 1704 } 1705 1706 return nullptr; 1707 } 1708 1709 #undef CASE_WIDEOP_CHANGE_OPCODE_LMULS 1710 #undef CASE_WIDEOP_CHANGE_OPCODE_COMMON 1711 #undef CASE_WIDEOP_OPCODE_LMULS 1712 #undef CASE_WIDEOP_OPCODE_COMMON 1713 1714 Register RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, 1715 MachineBasicBlock &MBB, 1716 MachineBasicBlock::iterator II, 1717 const DebugLoc &DL, 1718 int64_t Amount, 1719 MachineInstr::MIFlag Flag) const { 1720 assert(Amount > 0 && "There is no need to get VLEN scaled value."); 1721 assert(Amount % 8 == 0 && 1722 "Reserve the stack by the multiple of one vector size."); 1723 1724 MachineRegisterInfo &MRI = MF.getRegInfo(); 1725 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo(); 1726 int64_t NumOfVReg = Amount / 8; 1727 1728 Register VL = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1729 BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), VL) 1730 .setMIFlag(Flag); 1731 assert(isInt<32>(NumOfVReg) && 1732 "Expect the number of vector registers within 32-bits."); 1733 if (isPowerOf2_32(NumOfVReg)) { 1734 uint32_t ShiftAmount = Log2_32(NumOfVReg); 1735 if (ShiftAmount == 0) 1736 return VL; 1737 BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), VL) 1738 .addReg(VL, RegState::Kill) 1739 .addImm(ShiftAmount) 1740 .setMIFlag(Flag); 1741 } else if (isPowerOf2_32(NumOfVReg - 1)) { 1742 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1743 uint32_t ShiftAmount = Log2_32(NumOfVReg - 1); 1744 BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), ScaledRegister) 1745 .addReg(VL) 1746 .addImm(ShiftAmount) 1747 .setMIFlag(Flag); 1748 BuildMI(MBB, II, DL, TII->get(RISCV::ADD), VL) 1749 .addReg(ScaledRegister, RegState::Kill) 1750 .addReg(VL, RegState::Kill) 1751 .setMIFlag(Flag); 1752 } else if (isPowerOf2_32(NumOfVReg + 1)) { 1753 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1754 uint32_t ShiftAmount = Log2_32(NumOfVReg + 1); 1755 BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), ScaledRegister) 1756 .addReg(VL) 1757 .addImm(ShiftAmount) 1758 .setMIFlag(Flag); 1759 BuildMI(MBB, II, DL, TII->get(RISCV::SUB), VL) 1760 .addReg(ScaledRegister, RegState::Kill) 1761 .addReg(VL, RegState::Kill) 1762 .setMIFlag(Flag); 1763 } else { 1764 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass); 1765 if (!isInt<12>(NumOfVReg)) 1766 movImm(MBB, II, DL, N, NumOfVReg); 1767 else { 1768 BuildMI(MBB, II, DL, TII->get(RISCV::ADDI), N) 1769 .addReg(RISCV::X0) 1770 .addImm(NumOfVReg) 1771 .setMIFlag(Flag); 1772 } 1773 if (!MF.getSubtarget<RISCVSubtarget>().hasStdExtM()) 1774 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ 1775 MF.getFunction(), 1776 "M-extension must be enabled to calculate the vscaled size/offset."}); 1777 BuildMI(MBB, II, DL, TII->get(RISCV::MUL), VL) 1778 .addReg(VL, RegState::Kill) 1779 .addReg(N, RegState::Kill) 1780 .setMIFlag(Flag); 1781 } 1782 1783 return VL; 1784 } 1785 1786 static bool isRVVWholeLoadStore(unsigned Opcode) { 1787 switch (Opcode) { 1788 default: 1789 return false; 1790 case RISCV::VS1R_V: 1791 case RISCV::VS2R_V: 1792 case RISCV::VS4R_V: 1793 case RISCV::VS8R_V: 1794 case RISCV::VL1RE8_V: 1795 case RISCV::VL2RE8_V: 1796 case RISCV::VL4RE8_V: 1797 case RISCV::VL8RE8_V: 1798 case RISCV::VL1RE16_V: 1799 case RISCV::VL2RE16_V: 1800 case RISCV::VL4RE16_V: 1801 case RISCV::VL8RE16_V: 1802 case RISCV::VL1RE32_V: 1803 case RISCV::VL2RE32_V: 1804 case RISCV::VL4RE32_V: 1805 case RISCV::VL8RE32_V: 1806 case RISCV::VL1RE64_V: 1807 case RISCV::VL2RE64_V: 1808 case RISCV::VL4RE64_V: 1809 case RISCV::VL8RE64_V: 1810 return true; 1811 } 1812 } 1813 1814 bool RISCVInstrInfo::isRVVSpill(const MachineInstr &MI, bool CheckFIs) const { 1815 // RVV lacks any support for immediate addressing for stack addresses, so be 1816 // conservative. 1817 unsigned Opcode = MI.getOpcode(); 1818 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) && 1819 !isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode)) 1820 return false; 1821 return !CheckFIs || any_of(MI.operands(), [](const MachineOperand &MO) { 1822 return MO.isFI(); 1823 }); 1824 } 1825 1826 Optional<std::pair<unsigned, unsigned>> 1827 RISCVInstrInfo::isRVVSpillForZvlsseg(unsigned Opcode) const { 1828 switch (Opcode) { 1829 default: 1830 return None; 1831 case RISCV::PseudoVSPILL2_M1: 1832 case RISCV::PseudoVRELOAD2_M1: 1833 return std::make_pair(2u, 1u); 1834 case RISCV::PseudoVSPILL2_M2: 1835 case RISCV::PseudoVRELOAD2_M2: 1836 return std::make_pair(2u, 2u); 1837 case RISCV::PseudoVSPILL2_M4: 1838 case RISCV::PseudoVRELOAD2_M4: 1839 return std::make_pair(2u, 4u); 1840 case RISCV::PseudoVSPILL3_M1: 1841 case RISCV::PseudoVRELOAD3_M1: 1842 return std::make_pair(3u, 1u); 1843 case RISCV::PseudoVSPILL3_M2: 1844 case RISCV::PseudoVRELOAD3_M2: 1845 return std::make_pair(3u, 2u); 1846 case RISCV::PseudoVSPILL4_M1: 1847 case RISCV::PseudoVRELOAD4_M1: 1848 return std::make_pair(4u, 1u); 1849 case RISCV::PseudoVSPILL4_M2: 1850 case RISCV::PseudoVRELOAD4_M2: 1851 return std::make_pair(4u, 2u); 1852 case RISCV::PseudoVSPILL5_M1: 1853 case RISCV::PseudoVRELOAD5_M1: 1854 return std::make_pair(5u, 1u); 1855 case RISCV::PseudoVSPILL6_M1: 1856 case RISCV::PseudoVRELOAD6_M1: 1857 return std::make_pair(6u, 1u); 1858 case RISCV::PseudoVSPILL7_M1: 1859 case RISCV::PseudoVRELOAD7_M1: 1860 return std::make_pair(7u, 1u); 1861 case RISCV::PseudoVSPILL8_M1: 1862 case RISCV::PseudoVRELOAD8_M1: 1863 return std::make_pair(8u, 1u); 1864 } 1865 } 1866