1//===-- SIInstructions.td - SI Instruction Definitions --------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// This file was originally auto-generated from a GPU register header file and 9// all the instruction definitions were originally commented out. Instructions 10// that are not yet supported remain commented out. 11//===----------------------------------------------------------------------===// 12 13class GCNPat<dag pattern, dag result> : Pat<pattern, result>, GCNPredicateControl { 14 15} 16 17class UniformSextInreg<ValueType VT> : PatFrag< 18 (ops node:$src), 19 (sext_inreg $src, VT), 20 [{ return !N->isDivergent(); }]>; 21 22class DivergentSextInreg<ValueType VT> : PatFrag< 23 (ops node:$src), 24 (sext_inreg $src, VT), 25 [{ return N->isDivergent(); }]>; 26 27include "SOPInstructions.td" 28include "VOPInstructions.td" 29include "SMInstructions.td" 30include "FLATInstructions.td" 31include "BUFInstructions.td" 32include "EXPInstructions.td" 33include "DSDIRInstructions.td" 34include "VINTERPInstructions.td" 35 36//===----------------------------------------------------------------------===// 37// VINTRP Instructions 38//===----------------------------------------------------------------------===// 39 40// Used to inject printing of "_e32" suffix for VI (there are "_e64" variants for VI) 41def VINTRPDst : VINTRPDstOperand <VGPR_32>; 42 43let Uses = [MODE, M0, EXEC] in { 44 45// FIXME: Specify SchedRW for VINTRP instructions. 46 47multiclass V_INTERP_P1_F32_m : VINTRP_m < 48 0x00000000, 49 (outs VINTRPDst:$vdst), 50 (ins VGPR_32:$vsrc, InterpAttr:$attr, InterpAttrChan:$attrchan), 51 "v_interp_p1_f32$vdst, $vsrc, $attr$attrchan", 52 [(set f32:$vdst, (int_amdgcn_interp_p1 f32:$vsrc, 53 (i32 timm:$attrchan), (i32 timm:$attr), M0))] 54>; 55 56let OtherPredicates = [has32BankLDS, isNotGFX90APlus] in { 57 58defm V_INTERP_P1_F32 : V_INTERP_P1_F32_m; 59 60} // End OtherPredicates = [has32BankLDS, isNotGFX90APlus] 61 62let OtherPredicates = [has16BankLDS, isNotGFX90APlus], 63 Constraints = "@earlyclobber $vdst", isAsmParserOnly=1 in { 64 65defm V_INTERP_P1_F32_16bank : V_INTERP_P1_F32_m; 66 67} // End OtherPredicates = [has32BankLDS, isNotGFX90APlus], 68 // Constraints = "@earlyclobber $vdst", isAsmParserOnly=1 69 70let OtherPredicates = [isNotGFX90APlus] in { 71let DisableEncoding = "$src0", Constraints = "$src0 = $vdst" in { 72 73defm V_INTERP_P2_F32 : VINTRP_m < 74 0x00000001, 75 (outs VINTRPDst:$vdst), 76 (ins VGPR_32:$src0, VGPR_32:$vsrc, InterpAttr:$attr, 77 InterpAttrChan:$attrchan), 78 "v_interp_p2_f32$vdst, $vsrc, $attr$attrchan", 79 [(set f32:$vdst, (int_amdgcn_interp_p2 f32:$src0, f32:$vsrc, 80 (i32 timm:$attrchan), (i32 timm:$attr), M0))]>; 81 82} // End DisableEncoding = "$src0", Constraints = "$src0 = $vdst" 83 84defm V_INTERP_MOV_F32 : VINTRP_m < 85 0x00000002, 86 (outs VINTRPDst:$vdst), 87 (ins InterpSlot:$vsrc, InterpAttr:$attr, InterpAttrChan:$attrchan), 88 "v_interp_mov_f32$vdst, $vsrc, $attr$attrchan", 89 [(set f32:$vdst, (int_amdgcn_interp_mov (i32 timm:$vsrc), 90 (i32 timm:$attrchan), (i32 timm:$attr), M0))]>; 91 92} // End OtherPredicates = [isNotGFX90APlus] 93 94} // End Uses = [MODE, M0, EXEC] 95 96//===----------------------------------------------------------------------===// 97// Pseudo Instructions 98//===----------------------------------------------------------------------===// 99 100// Insert a branch to an endpgm block to use as a fallback trap. 101def ENDPGM_TRAP : SPseudoInstSI< 102 (outs), (ins), 103 [(AMDGPUendpgm_trap)], 104 "ENDPGM_TRAP"> { 105 let hasSideEffects = 1; 106 let usesCustomInserter = 1; 107} 108 109def ATOMIC_FENCE : SPseudoInstSI< 110 (outs), (ins i32imm:$ordering, i32imm:$scope), 111 [(atomic_fence (i32 timm:$ordering), (i32 timm:$scope))], 112 "ATOMIC_FENCE $ordering, $scope"> { 113 let hasSideEffects = 1; 114} 115 116let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in { 117 118// For use in patterns 119def V_CNDMASK_B64_PSEUDO : VOP3Common <(outs VReg_64:$vdst), 120 (ins VSrc_b64:$src0, VSrc_b64:$src1, SSrc_b64:$src2), "", []> { 121 let isPseudo = 1; 122 let isCodeGenOnly = 1; 123 let usesCustomInserter = 1; 124} 125 126// 64-bit vector move instruction. This is mainly used by the 127// SIFoldOperands pass to enable folding of inline immediates. 128def V_MOV_B64_PSEUDO : VPseudoInstSI <(outs VReg_64:$vdst), 129 (ins VSrc_b64:$src0)> { 130 let isReMaterializable = 1; 131 let isAsCheapAsAMove = 1; 132 let isMoveImm = 1; 133 let SchedRW = [Write64Bit]; 134 let Size = 4; 135 let UseNamedOperandTable = 1; 136} 137 138// 64-bit vector move with dpp. Expanded post-RA. 139def V_MOV_B64_DPP_PSEUDO : VOP_DPP_Pseudo <"v_mov_b64_dpp", VOP_I64_I64> { 140 let Size = 16; // Requires two 8-byte v_mov_b32_dpp to complete. 141} 142 143// 64-bit scalar move immediate instruction. This is used to avoid subregs 144// initialization and allow rematerialization. 145def S_MOV_B64_IMM_PSEUDO : SPseudoInstSI <(outs SReg_64:$sdst), 146 (ins i64imm:$src0)> { 147 let isReMaterializable = 1; 148 let isAsCheapAsAMove = 1; 149 let isMoveImm = 1; 150 let SchedRW = [WriteSALU, Write64Bit]; 151 let Size = 4; 152 let Uses = []; 153 let UseNamedOperandTable = 1; 154} 155 156// Pseudoinstruction for @llvm.amdgcn.wqm. It is turned into a copy after the 157// WQM pass processes it. 158def WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 159 160// Pseudoinstruction for @llvm.amdgcn.softwqm. Like @llvm.amdgcn.wqm it is 161// turned into a copy by WQM pass, but does not seed WQM requirements. 162def SOFT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 163 164// Pseudoinstruction for @llvm.amdgcn.strict.wwm. It is turned into a copy post-RA, so 165// that the @earlyclobber is respected. The @earlyclobber is to make sure that 166// the instruction that defines $src0 (which is run in Whole Wave Mode) doesn't 167// accidentally clobber inactive channels of $vdst. 168let Constraints = "@earlyclobber $vdst" in { 169def STRICT_WWM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 170def STRICT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 171} 172 173} // End let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] 174 175def WWM_COPY : SPseudoInstSI < 176 (outs unknown:$dst), (ins unknown:$src)> { 177 let hasSideEffects = 0; 178 let isAsCheapAsAMove = 1; 179 let isConvergent = 1; 180} 181 182def ENTER_STRICT_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> { 183 let Uses = [EXEC]; 184 let Defs = [EXEC, SCC]; 185 let hasSideEffects = 0; 186 let mayLoad = 0; 187 let mayStore = 0; 188} 189 190def EXIT_STRICT_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> { 191 let hasSideEffects = 0; 192 let mayLoad = 0; 193 let mayStore = 0; 194} 195 196def ENTER_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> { 197 let Uses = [EXEC]; 198 let Defs = [EXEC, SCC]; 199 let hasSideEffects = 0; 200 let mayLoad = 0; 201 let mayStore = 0; 202} 203 204def EXIT_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> { 205 let hasSideEffects = 0; 206 let mayLoad = 0; 207 let mayStore = 0; 208} 209 210let usesCustomInserter = 1 in { 211def S_INVERSE_BALLOT_U32 : SPseudoInstSI <(outs SReg_32:$sdst), (ins SSrc_b32:$mask)>; 212 213def S_INVERSE_BALLOT_U64 : SPseudoInstSI <(outs SReg_64:$sdst), (ins SSrc_b64:$mask)>; 214} // End usesCustomInserter = 1 215 216// PSEUDO_WM is treated like STRICT_WWM/STRICT_WQM without exec changes. 217def ENTER_PSEUDO_WM : SPseudoInstSI <(outs), (ins)> { 218 let Uses = [EXEC]; 219 let Defs = [EXEC]; 220 let hasSideEffects = 0; 221 let mayLoad = 0; 222 let mayStore = 0; 223} 224 225def EXIT_PSEUDO_WM : SPseudoInstSI <(outs), (ins)> { 226 let hasSideEffects = 0; 227 let mayLoad = 0; 228 let mayStore = 0; 229} 230 231// Pseudo instructions used for @llvm.fptrunc.round upward 232// and @llvm.fptrunc.round downward. 233// These intrinsics will be legalized to G_FPTRUNC_ROUND_UPWARD 234// and G_FPTRUNC_ROUND_DOWNWARD before being lowered to 235// FPTRUNC_UPWARD_PSEUDO and FPTRUNC_DOWNWARD_PSEUDO. 236// The final codegen is done in the ModeRegister pass. 237let Uses = [MODE, EXEC] in { 238def FPTRUNC_UPWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst), 239 (ins VGPR_32:$src0), 240 [(set f16:$vdst, (SIfptrunc_round_upward f32:$src0))]>; 241 242def FPTRUNC_DOWNWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst), 243 (ins VGPR_32:$src0), 244 [(set f16:$vdst, (SIfptrunc_round_downward f32:$src0))]>; 245} // End Uses = [MODE, EXEC] 246 247// Invert the exec mask and overwrite the inactive lanes of dst with inactive, 248// restoring it after we're done. 249let Defs = [SCC], isConvergent = 1 in { 250def V_SET_INACTIVE_B32 : VPseudoInstSI <(outs VGPR_32:$vdst), 251 (ins VSrc_b32: $src, VSrc_b32:$inactive), 252 [(set i32:$vdst, (int_amdgcn_set_inactive i32:$src, i32:$inactive))]> { 253} 254 255def V_SET_INACTIVE_B64 : VPseudoInstSI <(outs VReg_64:$vdst), 256 (ins VSrc_b64: $src, VSrc_b64:$inactive), 257 [(set i64:$vdst, (int_amdgcn_set_inactive i64:$src, i64:$inactive))]> { 258} 259} // End Defs = [SCC] 260 261def : GCNPat<(i32 (int_amdgcn_set_inactive_chain_arg i32:$src, i32:$inactive)), 262 (V_SET_INACTIVE_B32 VGPR_32:$src, VGPR_32:$inactive)>; 263 264def : GCNPat<(i64 (int_amdgcn_set_inactive_chain_arg i64:$src, i64:$inactive)), 265 (V_SET_INACTIVE_B64 VReg_64:$src, VReg_64:$inactive)>; 266 267let usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in { 268 def WAVE_REDUCE_UMIN_PSEUDO_U32 : VPseudoInstSI <(outs SGPR_32:$sdst), 269 (ins VSrc_b32: $src, VSrc_b32:$strategy), 270 [(set i32:$sdst, (int_amdgcn_wave_reduce_umin i32:$src, i32:$strategy))]> { 271 } 272 273 def WAVE_REDUCE_UMAX_PSEUDO_U32 : VPseudoInstSI <(outs SGPR_32:$sdst), 274 (ins VSrc_b32: $src, VSrc_b32:$strategy), 275 [(set i32:$sdst, (int_amdgcn_wave_reduce_umax i32:$src, i32:$strategy))]> { 276 } 277} 278 279let usesCustomInserter = 1, Defs = [VCC] in { 280def V_ADD_U64_PSEUDO : VPseudoInstSI < 281 (outs VReg_64:$vdst), (ins VSrc_b64:$src0, VSrc_b64:$src1), 282 [(set VReg_64:$vdst, (DivergentBinFrag<add> i64:$src0, i64:$src1))] 283>; 284 285def V_SUB_U64_PSEUDO : VPseudoInstSI < 286 (outs VReg_64:$vdst), (ins VSrc_b64:$src0, VSrc_b64:$src1), 287 [(set VReg_64:$vdst, (DivergentBinFrag<sub> i64:$src0, i64:$src1))] 288>; 289} // End usesCustomInserter = 1, Defs = [VCC] 290 291let usesCustomInserter = 1, Defs = [SCC] in { 292def S_ADD_U64_PSEUDO : SPseudoInstSI < 293 (outs SReg_64:$sdst), (ins SSrc_b64:$src0, SSrc_b64:$src1), 294 [(set SReg_64:$sdst, (UniformBinFrag<add> i64:$src0, i64:$src1))] 295>; 296 297def S_SUB_U64_PSEUDO : SPseudoInstSI < 298 (outs SReg_64:$sdst), (ins SSrc_b64:$src0, SSrc_b64:$src1), 299 [(set SReg_64:$sdst, (UniformBinFrag<sub> i64:$src0, i64:$src1))] 300>; 301 302def S_ADD_CO_PSEUDO : SPseudoInstSI < 303 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1, SSrc_i1:$scc_in) 304>; 305 306def S_SUB_CO_PSEUDO : SPseudoInstSI < 307 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1, SSrc_i1:$scc_in) 308>; 309 310def S_UADDO_PSEUDO : SPseudoInstSI < 311 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1) 312>; 313 314def S_USUBO_PSEUDO : SPseudoInstSI < 315 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1) 316>; 317 318let OtherPredicates = [HasShaderCyclesHiLoRegisters] in 319def GET_SHADERCYCLESHILO : SPseudoInstSI< 320 (outs SReg_64:$sdst), (ins), 321 [(set SReg_64:$sdst, (i64 (readcyclecounter)))] 322>; 323 324} // End usesCustomInserter = 1, Defs = [SCC] 325 326let usesCustomInserter = 1 in { 327def GET_GROUPSTATICSIZE : SPseudoInstSI <(outs SReg_32:$sdst), (ins), 328 [(set SReg_32:$sdst, (int_amdgcn_groupstaticsize))]>; 329} // End let usesCustomInserter = 1, SALU = 1 330 331// Wrap an instruction by duplicating it, except for setting isTerminator. 332class WrapTerminatorInst<SOP_Pseudo base_inst> : SPseudoInstSI< 333 base_inst.OutOperandList, 334 base_inst.InOperandList> { 335 let Uses = base_inst.Uses; 336 let Defs = base_inst.Defs; 337 let isTerminator = 1; 338 let isAsCheapAsAMove = base_inst.isAsCheapAsAMove; 339 let hasSideEffects = base_inst.hasSideEffects; 340 let UseNamedOperandTable = base_inst.UseNamedOperandTable; 341 let CodeSize = base_inst.CodeSize; 342 let SchedRW = base_inst.SchedRW; 343} 344 345let WaveSizePredicate = isWave64 in { 346def S_MOV_B64_term : WrapTerminatorInst<S_MOV_B64>; 347def S_XOR_B64_term : WrapTerminatorInst<S_XOR_B64>; 348def S_OR_B64_term : WrapTerminatorInst<S_OR_B64>; 349def S_ANDN2_B64_term : WrapTerminatorInst<S_ANDN2_B64>; 350def S_AND_B64_term : WrapTerminatorInst<S_AND_B64>; 351def S_AND_SAVEEXEC_B64_term : WrapTerminatorInst<S_AND_SAVEEXEC_B64>; 352} 353 354let WaveSizePredicate = isWave32 in { 355def S_MOV_B32_term : WrapTerminatorInst<S_MOV_B32>; 356def S_XOR_B32_term : WrapTerminatorInst<S_XOR_B32>; 357def S_OR_B32_term : WrapTerminatorInst<S_OR_B32>; 358def S_ANDN2_B32_term : WrapTerminatorInst<S_ANDN2_B32>; 359def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>; 360def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>; 361} 362 363 364def WAVE_BARRIER : SPseudoInstSI<(outs), (ins), 365 [(int_amdgcn_wave_barrier)]> { 366 let SchedRW = []; 367 let hasNoSchedulingInfo = 1; 368 let hasSideEffects = 1; 369 let mayLoad = 0; 370 let mayStore = 0; 371 let isConvergent = 1; 372 let FixedSize = 1; 373 let Size = 0; 374 let isMeta = 1; 375} 376 377def SCHED_BARRIER : SPseudoInstSI<(outs), (ins i32imm:$mask), 378 [(int_amdgcn_sched_barrier (i32 timm:$mask))]> { 379 let SchedRW = []; 380 let hasNoSchedulingInfo = 1; 381 let hasSideEffects = 1; 382 let mayLoad = 0; 383 let mayStore = 0; 384 let isConvergent = 1; 385 let FixedSize = 1; 386 let Size = 0; 387 let isMeta = 1; 388} 389 390def SCHED_GROUP_BARRIER : SPseudoInstSI< 391 (outs), 392 (ins i32imm:$mask, i32imm:$size, i32imm:$syncid), 393 [(int_amdgcn_sched_group_barrier (i32 timm:$mask), (i32 timm:$size), (i32 timm:$syncid))]> { 394 let SchedRW = []; 395 let hasNoSchedulingInfo = 1; 396 let hasSideEffects = 1; 397 let mayLoad = 0; 398 let mayStore = 0; 399 let isConvergent = 1; 400 let FixedSize = 1; 401 let Size = 0; 402 let isMeta = 1; 403} 404 405def IGLP_OPT : SPseudoInstSI<(outs), (ins i32imm:$mask), 406 [(int_amdgcn_iglp_opt (i32 timm:$mask))]> { 407 let SchedRW = []; 408 let hasNoSchedulingInfo = 1; 409 let hasSideEffects = 1; 410 let mayLoad = 0; 411 let mayStore = 0; 412 let isConvergent = 1; 413 let FixedSize = 1; 414 let Size = 0; 415 let isMeta = 1; 416} 417 418// SI pseudo instructions. These are used by the CFG structurizer pass 419// and should be lowered to ISA instructions prior to codegen. 420 421// As we have enhanced control flow intrinsics to work under unstructured CFG, 422// duplicating such intrinsics can be actually treated as legal. On the contrary, 423// by making them non-duplicable, we are observing better code generation result. 424// So we choose to mark them non-duplicable in hope of getting better code 425// generation as well as simplied CFG during Machine IR optimization stage. 426 427let isTerminator = 1, isNotDuplicable = 1 in { 428 429let OtherPredicates = [EnableLateCFGStructurize] in { 430 def SI_NON_UNIFORM_BRCOND_PSEUDO : CFPseudoInstSI < 431 (outs), 432 (ins SReg_1:$vcc, brtarget:$target), 433 [(brcond i1:$vcc, bb:$target)]> { 434 let Size = 12; 435} 436} 437 438def SI_IF: CFPseudoInstSI < 439 (outs SReg_1:$dst), (ins SReg_1:$vcc, brtarget:$target), 440 [(set i1:$dst, (AMDGPUif i1:$vcc, bb:$target))], 1, 1> { 441 let Constraints = ""; 442 let Size = 12; 443 let hasSideEffects = 1; 444 let IsNeverUniform = 1; 445} 446 447def SI_ELSE : CFPseudoInstSI < 448 (outs SReg_1:$dst), 449 (ins SReg_1:$src, brtarget:$target), [], 1, 1> { 450 let Size = 12; 451 let hasSideEffects = 1; 452 let IsNeverUniform = 1; 453} 454 455def SI_WATERFALL_LOOP : CFPseudoInstSI < 456 (outs), 457 (ins brtarget:$target), [], 1> { 458 let Size = 8; 459 let isBranch = 1; 460 let Defs = []; 461} 462 463def SI_LOOP : CFPseudoInstSI < 464 (outs), (ins SReg_1:$saved, brtarget:$target), 465 [(AMDGPUloop i1:$saved, bb:$target)], 1, 1> { 466 let Size = 8; 467 let isBranch = 1; 468 let hasSideEffects = 1; 469 let IsNeverUniform = 1; 470} 471 472} // End isTerminator = 1 473 474def SI_END_CF : CFPseudoInstSI < 475 (outs), (ins SReg_1:$saved), [], 1, 1> { 476 let Size = 4; 477 let isAsCheapAsAMove = 1; 478 let isReMaterializable = 1; 479 let hasSideEffects = 1; 480 let isNotDuplicable = 1; // Not a hard requirement, see long comments above for details. 481 let mayLoad = 1; // FIXME: Should not need memory flags 482 let mayStore = 1; 483} 484 485def SI_IF_BREAK : CFPseudoInstSI < 486 (outs SReg_1:$dst), (ins SReg_1:$vcc, SReg_1:$src), []> { 487 let Size = 4; 488 let isNotDuplicable = 1; // Not a hard requirement, see long comments above for details. 489 let isAsCheapAsAMove = 1; 490 let isReMaterializable = 1; 491} 492 493// Branch to the early termination block of the shader if SCC is 0. 494// This uses SCC from a previous SALU operation, i.e. the update of 495// a mask of live lanes after a kill/demote operation. 496// Only valid in pixel shaders. 497def SI_EARLY_TERMINATE_SCC0 : SPseudoInstSI <(outs), (ins)> { 498 let Uses = [EXEC,SCC]; 499} 500 501let Uses = [EXEC] in { 502 503multiclass PseudoInstKill <dag ins> { 504 // Even though this pseudo can usually be expanded without an SCC def, we 505 // conservatively assume that it has an SCC def, both because it is sometimes 506 // required in degenerate cases (when V_CMPX cannot be used due to constant 507 // bus limitations) and because it allows us to avoid having to track SCC 508 // liveness across basic blocks. 509 let Defs = [EXEC,SCC] in 510 def _PSEUDO : PseudoInstSI <(outs), ins> { 511 let isConvergent = 1; 512 let usesCustomInserter = 1; 513 } 514 515 let Defs = [EXEC,SCC] in 516 def _TERMINATOR : SPseudoInstSI <(outs), ins> { 517 let isTerminator = 1; 518 } 519} 520 521defm SI_KILL_I1 : PseudoInstKill <(ins SCSrc_i1:$src, i1imm:$killvalue)>; 522let Defs = [VCC] in 523defm SI_KILL_F32_COND_IMM : PseudoInstKill <(ins VSrc_b32:$src0, i32imm:$src1, i32imm:$cond)>; 524 525let Defs = [EXEC,VCC] in 526def SI_ILLEGAL_COPY : SPseudoInstSI < 527 (outs unknown:$dst), (ins unknown:$src), 528 [], " ; illegal copy $src to $dst">; 529 530} // End Uses = [EXEC], Defs = [EXEC,VCC] 531 532// Branch on undef scc. Used to avoid intermediate copy from 533// IMPLICIT_DEF to SCC. 534def SI_BR_UNDEF : SPseudoInstSI <(outs), (ins SOPPBrTarget:$simm16)> { 535 let isTerminator = 1; 536 let usesCustomInserter = 1; 537 let isBranch = 1; 538} 539 540def SI_PS_LIVE : PseudoInstSI < 541 (outs SReg_1:$dst), (ins), 542 [(set i1:$dst, (int_amdgcn_ps_live))]> { 543 let SALU = 1; 544} 545 546let Uses = [EXEC] in { 547def SI_LIVE_MASK : PseudoInstSI < 548 (outs SReg_1:$dst), (ins), 549 [(set i1:$dst, (int_amdgcn_live_mask))]> { 550 let SALU = 1; 551} 552let Defs = [EXEC,SCC] in { 553// Demote: Turn a pixel shader thread into a helper lane. 554def SI_DEMOTE_I1 : SPseudoInstSI <(outs), (ins SCSrc_i1:$src, i1imm:$killvalue)>; 555} // End Defs = [EXEC,SCC] 556} // End Uses = [EXEC] 557 558def SI_MASKED_UNREACHABLE : SPseudoInstSI <(outs), (ins), 559 [(int_amdgcn_unreachable)], 560 "; divergent unreachable"> { 561 let Size = 0; 562 let hasNoSchedulingInfo = 1; 563 let FixedSize = 1; 564 let isMeta = 1; 565 let maybeAtomic = 0; 566} 567 568// Used as an isel pseudo to directly emit initialization with an 569// s_mov_b32 rather than a copy of another initialized 570// register. MachineCSE skips copies, and we don't want to have to 571// fold operands before it runs. 572def SI_INIT_M0 : SPseudoInstSI <(outs), (ins SSrc_b32:$src)> { 573 let Defs = [M0]; 574 let usesCustomInserter = 1; 575 let isAsCheapAsAMove = 1; 576 let isReMaterializable = 1; 577} 578 579def SI_INIT_EXEC : SPseudoInstSI < 580 (outs), (ins i64imm:$src), 581 [(int_amdgcn_init_exec (i64 timm:$src))]> { 582 let Defs = [EXEC]; 583 let isAsCheapAsAMove = 1; 584} 585 586def SI_INIT_EXEC_FROM_INPUT : SPseudoInstSI < 587 (outs), (ins SSrc_b32:$input, i32imm:$shift), 588 [(int_amdgcn_init_exec_from_input i32:$input, (i32 timm:$shift))]> { 589 let Defs = [EXEC]; 590} 591 592// Return for returning shaders to a shader variant epilog. 593def SI_RETURN_TO_EPILOG : SPseudoInstSI < 594 (outs), (ins variable_ops), [(AMDGPUreturn_to_epilog)]> { 595 let isTerminator = 1; 596 let isBarrier = 1; 597 let isReturn = 1; 598 let hasNoSchedulingInfo = 1; 599 let DisableWQM = 1; 600 let FixedSize = 1; 601 602 // TODO: Should this be true? 603 let isMeta = 0; 604} 605 606// Return for returning function calls. 607def SI_RETURN : SPseudoInstSI < 608 (outs), (ins), [(AMDGPUret_glue)], 609 "; return"> { 610 let isTerminator = 1; 611 let isBarrier = 1; 612 let isReturn = 1; 613 let SchedRW = [WriteBranch]; 614} 615 616// Return for returning function calls without output register. 617// 618// This version is only needed so we can fill in the output register 619// in the custom inserter. 620def SI_CALL_ISEL : SPseudoInstSI < 621 (outs), (ins SSrc_b64:$src0, unknown:$callee), 622 [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> { 623 let Size = 4; 624 let isCall = 1; 625 let SchedRW = [WriteBranch]; 626 let usesCustomInserter = 1; 627 // TODO: Should really base this on the call target 628 let isConvergent = 1; 629} 630 631def : GCNPat< 632 (AMDGPUcall i64:$src0, (i64 0)), 633 (SI_CALL_ISEL $src0, (i64 0)) 634>; 635 636// Wrapper around s_swappc_b64 with extra $callee parameter to track 637// the called function after regalloc. 638def SI_CALL : SPseudoInstSI < 639 (outs SReg_64:$dst), (ins SSrc_b64:$src0, unknown:$callee)> { 640 let Size = 4; 641 let FixedSize = 1; 642 let isCall = 1; 643 let UseNamedOperandTable = 1; 644 let SchedRW = [WriteBranch]; 645 // TODO: Should really base this on the call target 646 let isConvergent = 1; 647} 648 649class SI_TCRETURN_Pseudo<RegisterClass rc, SDNode sd> : SPseudoInstSI <(outs), 650 (ins rc:$src0, unknown:$callee, i32imm:$fpdiff), 651 [(sd i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> { 652 let Size = 4; 653 let FixedSize = 1; 654 let isCall = 1; 655 let isTerminator = 1; 656 let isReturn = 1; 657 let isBarrier = 1; 658 let UseNamedOperandTable = 1; 659 let SchedRW = [WriteBranch]; 660 // TODO: Should really base this on the call target 661 let isConvergent = 1; 662} 663 664// Tail call handling pseudo 665def SI_TCRETURN : SI_TCRETURN_Pseudo<CCR_SGPR_64, AMDGPUtc_return>; 666def SI_TCRETURN_GFX : SI_TCRETURN_Pseudo<Gfx_CCR_SGPR_64, AMDGPUtc_return_gfx>; 667 668// Handle selecting indirect tail calls 669def : GCNPat< 670 (AMDGPUtc_return i64:$src0, (i64 0), (i32 timm:$fpdiff)), 671 (SI_TCRETURN CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff) 672>; 673 674// Handle selecting indirect tail calls for AMDGPU_gfx 675def : GCNPat< 676 (AMDGPUtc_return_gfx i64:$src0, (i64 0), (i32 timm:$fpdiff)), 677 (SI_TCRETURN_GFX Gfx_CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff) 678>; 679 680// Pseudo for the llvm.amdgcn.cs.chain intrinsic. 681// This is essentially a tail call, but it also takes a mask to put in EXEC 682// right before jumping to the callee. 683class SI_CS_CHAIN_TC< 684 ValueType execvt, Predicate wavesizepred, 685 RegisterOperand execrc = getSOPSrcForVT<execvt>.ret> 686 : SPseudoInstSI <(outs), 687 (ins CCR_SGPR_64:$src0, unknown:$callee, i32imm:$fpdiff, execrc:$exec)> { 688 let FixedSize = 0; 689 let isCall = 1; 690 let isTerminator = 1; 691 let isBarrier = 1; 692 let isReturn = 1; 693 let UseNamedOperandTable = 1; 694 let SchedRW = [WriteBranch]; 695 let isConvergent = 1; 696 697 let WaveSizePredicate = wavesizepred; 698} 699 700def SI_CS_CHAIN_TC_W32 : SI_CS_CHAIN_TC<i32, isWave32>; 701def SI_CS_CHAIN_TC_W64 : SI_CS_CHAIN_TC<i64, isWave64>; 702 703// Handle selecting direct & indirect calls via SI_CS_CHAIN_TC_W32/64 704multiclass si_cs_chain_tc_pattern< 705 dag callee, ValueType execvt, RegisterOperand execrc, Instruction tc> { 706def : GCNPat< 707 (AMDGPUtc_return_chain i64:$src0, callee, (i32 timm:$fpdiff), execvt:$exec), 708 (tc CCR_SGPR_64:$src0, callee, i32imm:$fpdiff, execrc:$exec) 709>; 710} 711 712multiclass si_cs_chain_tc_patterns< 713 ValueType execvt, 714 RegisterOperand execrc = getSOPSrcForVT<execvt>.ret, 715 Instruction tc = !if(!eq(execvt, i32), SI_CS_CHAIN_TC_W32, SI_CS_CHAIN_TC_W64) 716 > { 717 defm direct: si_cs_chain_tc_pattern<(tglobaladdr:$callee), execvt, execrc, tc>; 718 defm indirect: si_cs_chain_tc_pattern<(i64 0), execvt, execrc, tc>; 719} 720 721defm : si_cs_chain_tc_patterns<i32>; 722defm : si_cs_chain_tc_patterns<i64>; 723 724def ADJCALLSTACKUP : SPseudoInstSI< 725 (outs), (ins i32imm:$amt0, i32imm:$amt1), 726 [(callseq_start timm:$amt0, timm:$amt1)], 727 "; adjcallstackup $amt0 $amt1"> { 728 let Size = 8; // Worst case. (s_add_u32 + constant) 729 let FixedSize = 1; 730 let hasSideEffects = 1; 731 let usesCustomInserter = 1; 732 let SchedRW = [WriteSALU]; 733 let Defs = [SCC]; 734} 735 736def ADJCALLSTACKDOWN : SPseudoInstSI< 737 (outs), (ins i32imm:$amt1, i32imm:$amt2), 738 [(callseq_end timm:$amt1, timm:$amt2)], 739 "; adjcallstackdown $amt1"> { 740 let Size = 8; // Worst case. (s_add_u32 + constant) 741 let hasSideEffects = 1; 742 let usesCustomInserter = 1; 743 let SchedRW = [WriteSALU]; 744 let Defs = [SCC]; 745} 746 747let Defs = [M0, EXEC, SCC], 748 UseNamedOperandTable = 1 in { 749 750// SI_INDIRECT_SRC/DST are only used by legacy SelectionDAG indirect 751// addressing implementation. 752class SI_INDIRECT_SRC<RegisterClass rc> : VPseudoInstSI < 753 (outs VGPR_32:$vdst), 754 (ins rc:$src, VS_32:$idx, i32imm:$offset)> { 755 let usesCustomInserter = 1; 756} 757 758class SI_INDIRECT_DST<RegisterClass rc> : VPseudoInstSI < 759 (outs rc:$vdst), 760 (ins rc:$src, VS_32:$idx, i32imm:$offset, VGPR_32:$val)> { 761 let Constraints = "$src = $vdst"; 762 let usesCustomInserter = 1; 763} 764 765def SI_INDIRECT_SRC_V1 : SI_INDIRECT_SRC<VGPR_32>; 766def SI_INDIRECT_SRC_V2 : SI_INDIRECT_SRC<VReg_64>; 767def SI_INDIRECT_SRC_V4 : SI_INDIRECT_SRC<VReg_128>; 768def SI_INDIRECT_SRC_V8 : SI_INDIRECT_SRC<VReg_256>; 769def SI_INDIRECT_SRC_V9 : SI_INDIRECT_SRC<VReg_288>; 770def SI_INDIRECT_SRC_V10 : SI_INDIRECT_SRC<VReg_320>; 771def SI_INDIRECT_SRC_V11 : SI_INDIRECT_SRC<VReg_352>; 772def SI_INDIRECT_SRC_V12 : SI_INDIRECT_SRC<VReg_384>; 773def SI_INDIRECT_SRC_V16 : SI_INDIRECT_SRC<VReg_512>; 774def SI_INDIRECT_SRC_V32 : SI_INDIRECT_SRC<VReg_1024>; 775 776def SI_INDIRECT_DST_V1 : SI_INDIRECT_DST<VGPR_32>; 777def SI_INDIRECT_DST_V2 : SI_INDIRECT_DST<VReg_64>; 778def SI_INDIRECT_DST_V4 : SI_INDIRECT_DST<VReg_128>; 779def SI_INDIRECT_DST_V8 : SI_INDIRECT_DST<VReg_256>; 780def SI_INDIRECT_DST_V9 : SI_INDIRECT_DST<VReg_288>; 781def SI_INDIRECT_DST_V10 : SI_INDIRECT_DST<VReg_320>; 782def SI_INDIRECT_DST_V11 : SI_INDIRECT_DST<VReg_352>; 783def SI_INDIRECT_DST_V12 : SI_INDIRECT_DST<VReg_384>; 784def SI_INDIRECT_DST_V16 : SI_INDIRECT_DST<VReg_512>; 785def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>; 786 787} // End Uses = [EXEC], Defs = [M0, EXEC] 788 789// This is a pseudo variant of the v_movreld_b32 instruction in which the 790// vector operand appears only twice, once as def and once as use. Using this 791// pseudo avoids problems with the Two Address instructions pass. 792class INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc, 793 RegisterOperand val_ty> : PseudoInstSI < 794 (outs rc:$vdst), (ins rc:$vsrc, val_ty:$val, i32imm:$subreg)> { 795 let Constraints = "$vsrc = $vdst"; 796 let Uses = [M0]; 797} 798 799class V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> : 800 INDIRECT_REG_WRITE_MOVREL_pseudo<rc, VSrc_b32> { 801 let VALU = 1; 802 let VOP1 = 1; 803 let Uses = [M0, EXEC]; 804} 805 806class S_INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc, 807 RegisterOperand val_ty> : 808 INDIRECT_REG_WRITE_MOVREL_pseudo<rc, val_ty> { 809 let SALU = 1; 810 let SOP1 = 1; 811 let Uses = [M0]; 812} 813 814class S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> : 815 S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b32>; 816class S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<RegisterClass rc> : 817 S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b64>; 818 819def V_INDIRECT_REG_WRITE_MOVREL_B32_V1 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VGPR_32>; 820def V_INDIRECT_REG_WRITE_MOVREL_B32_V2 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_64>; 821def V_INDIRECT_REG_WRITE_MOVREL_B32_V3 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_96>; 822def V_INDIRECT_REG_WRITE_MOVREL_B32_V4 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_128>; 823def V_INDIRECT_REG_WRITE_MOVREL_B32_V5 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_160>; 824def V_INDIRECT_REG_WRITE_MOVREL_B32_V8 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_256>; 825def V_INDIRECT_REG_WRITE_MOVREL_B32_V9 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_288>; 826def V_INDIRECT_REG_WRITE_MOVREL_B32_V10 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_320>; 827def V_INDIRECT_REG_WRITE_MOVREL_B32_V11 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_352>; 828def V_INDIRECT_REG_WRITE_MOVREL_B32_V12 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_384>; 829def V_INDIRECT_REG_WRITE_MOVREL_B32_V16 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_512>; 830def V_INDIRECT_REG_WRITE_MOVREL_B32_V32 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_1024>; 831 832def S_INDIRECT_REG_WRITE_MOVREL_B32_V1 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_32>; 833def S_INDIRECT_REG_WRITE_MOVREL_B32_V2 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_64>; 834def S_INDIRECT_REG_WRITE_MOVREL_B32_V3 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_96>; 835def S_INDIRECT_REG_WRITE_MOVREL_B32_V4 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_128>; 836def S_INDIRECT_REG_WRITE_MOVREL_B32_V5 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_160>; 837def S_INDIRECT_REG_WRITE_MOVREL_B32_V8 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_256>; 838def S_INDIRECT_REG_WRITE_MOVREL_B32_V9 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_288>; 839def S_INDIRECT_REG_WRITE_MOVREL_B32_V10 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_320>; 840def S_INDIRECT_REG_WRITE_MOVREL_B32_V11 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_352>; 841def S_INDIRECT_REG_WRITE_MOVREL_B32_V12 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_384>; 842def S_INDIRECT_REG_WRITE_MOVREL_B32_V16 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_512>; 843def S_INDIRECT_REG_WRITE_MOVREL_B32_V32 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_1024>; 844 845def S_INDIRECT_REG_WRITE_MOVREL_B64_V1 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_64>; 846def S_INDIRECT_REG_WRITE_MOVREL_B64_V2 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_128>; 847def S_INDIRECT_REG_WRITE_MOVREL_B64_V4 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_256>; 848def S_INDIRECT_REG_WRITE_MOVREL_B64_V8 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_512>; 849def S_INDIRECT_REG_WRITE_MOVREL_B64_V16 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_1024>; 850 851// These variants of V_INDIRECT_REG_READ/WRITE use VGPR indexing. By using these 852// pseudos we avoid spills or copies being inserted within indirect sequences 853// that switch the VGPR indexing mode. Spills to accvgprs could be effected by 854// this mode switching. 855 856class V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI < 857 (outs rc:$vdst), (ins rc:$vsrc, VSrc_b32:$val, SSrc_b32:$idx, i32imm:$subreg)> { 858 let Constraints = "$vsrc = $vdst"; 859 let VALU = 1; 860 let Uses = [M0, EXEC]; 861 let Defs = [M0]; 862} 863 864def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VGPR_32>; 865def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_64>; 866def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_96>; 867def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_128>; 868def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_160>; 869def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_256>; 870def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_288>; 871def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_320>; 872def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_352>; 873def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_384>; 874def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_512>; 875def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_1024>; 876 877class V_INDIRECT_REG_READ_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI < 878 (outs VGPR_32:$vdst), (ins rc:$vsrc, SSrc_b32:$idx, i32imm:$subreg)> { 879 let VALU = 1; 880 let Uses = [M0, EXEC]; 881 let Defs = [M0]; 882} 883 884def V_INDIRECT_REG_READ_GPR_IDX_B32_V1 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VGPR_32>; 885def V_INDIRECT_REG_READ_GPR_IDX_B32_V2 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_64>; 886def V_INDIRECT_REG_READ_GPR_IDX_B32_V3 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_96>; 887def V_INDIRECT_REG_READ_GPR_IDX_B32_V4 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_128>; 888def V_INDIRECT_REG_READ_GPR_IDX_B32_V5 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_160>; 889def V_INDIRECT_REG_READ_GPR_IDX_B32_V8 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_256>; 890def V_INDIRECT_REG_READ_GPR_IDX_B32_V9 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_288>; 891def V_INDIRECT_REG_READ_GPR_IDX_B32_V10 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_320>; 892def V_INDIRECT_REG_READ_GPR_IDX_B32_V11 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_352>; 893def V_INDIRECT_REG_READ_GPR_IDX_B32_V12 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_384>; 894def V_INDIRECT_REG_READ_GPR_IDX_B32_V16 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_512>; 895def V_INDIRECT_REG_READ_GPR_IDX_B32_V32 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_1024>; 896 897multiclass SI_SPILL_SGPR <RegisterClass sgpr_class> { 898 let UseNamedOperandTable = 1, SGPRSpill = 1, Uses = [EXEC] in { 899 def _SAVE : PseudoInstSI < 900 (outs), 901 (ins sgpr_class:$data, i32imm:$addr)> { 902 let mayStore = 1; 903 let mayLoad = 0; 904 } 905 906 def _RESTORE : PseudoInstSI < 907 (outs sgpr_class:$data), 908 (ins i32imm:$addr)> { 909 let mayStore = 0; 910 let mayLoad = 1; 911 } 912 } // End UseNamedOperandTable = 1 913} 914 915// You cannot use M0 as the output of v_readlane_b32 instructions or 916// use it in the sdata operand of SMEM instructions. We still need to 917// be able to spill the physical register m0, so allow it for 918// SI_SPILL_32_* instructions. 919defm SI_SPILL_S32 : SI_SPILL_SGPR <SReg_32>; 920defm SI_SPILL_S64 : SI_SPILL_SGPR <SReg_64>; 921defm SI_SPILL_S96 : SI_SPILL_SGPR <SReg_96>; 922defm SI_SPILL_S128 : SI_SPILL_SGPR <SReg_128>; 923defm SI_SPILL_S160 : SI_SPILL_SGPR <SReg_160>; 924defm SI_SPILL_S192 : SI_SPILL_SGPR <SReg_192>; 925defm SI_SPILL_S224 : SI_SPILL_SGPR <SReg_224>; 926defm SI_SPILL_S256 : SI_SPILL_SGPR <SReg_256>; 927defm SI_SPILL_S288 : SI_SPILL_SGPR <SReg_288>; 928defm SI_SPILL_S320 : SI_SPILL_SGPR <SReg_320>; 929defm SI_SPILL_S352 : SI_SPILL_SGPR <SReg_352>; 930defm SI_SPILL_S384 : SI_SPILL_SGPR <SReg_384>; 931defm SI_SPILL_S512 : SI_SPILL_SGPR <SReg_512>; 932defm SI_SPILL_S1024 : SI_SPILL_SGPR <SReg_1024>; 933 934let SGPRSpill = 1, VALU = 1, isConvergent = 1 in { 935def SI_SPILL_S32_TO_VGPR : PseudoInstSI <(outs VGPR_32:$vdst), 936 (ins SReg_32:$src0, i32imm:$src1, VGPR_32:$vdst_in)> { 937 let Size = 4; 938 let FixedSize = 1; 939 let IsNeverUniform = 1; 940 let hasSideEffects = 0; 941 let mayLoad = 0; 942 let mayStore = 0; 943 let Constraints = "$vdst = $vdst_in"; 944} 945 946def SI_RESTORE_S32_FROM_VGPR : PseudoInstSI <(outs SReg_32:$sdst), 947 (ins VGPR_32:$src0, i32imm:$src1)> { 948 let Size = 4; 949 let FixedSize = 1; 950 let hasSideEffects = 0; 951 let mayLoad = 0; 952 let mayStore = 0; 953} 954} // End SGPRSpill = 1, VALU = 1, isConvergent = 1 955 956// VGPR or AGPR spill instructions. In case of AGPR spilling a temp register 957// needs to be used and an extra instruction to move between VGPR and AGPR. 958// UsesTmp adds to the total size of an expanded spill in this case. 959multiclass SI_SPILL_VGPR <RegisterClass vgpr_class, bit UsesTmp = 0> { 960 let UseNamedOperandTable = 1, VGPRSpill = 1, 961 SchedRW = [WriteVMEM] in { 962 def _SAVE : VPseudoInstSI < 963 (outs), 964 (ins vgpr_class:$vdata, i32imm:$vaddr, 965 SReg_32:$soffset, i32imm:$offset)> { 966 let mayStore = 1; 967 let mayLoad = 0; 968 // (2 * 4) + (8 * num_subregs) bytes maximum 969 int MaxSize = !add(!shl(!srl(vgpr_class.Size, 5), !add(UsesTmp, 3)), 8); 970 // Size field is unsigned char and cannot fit more. 971 let Size = !if(!le(MaxSize, 256), MaxSize, 252); 972 } 973 974 def _RESTORE : VPseudoInstSI < 975 (outs vgpr_class:$vdata), 976 (ins i32imm:$vaddr, 977 SReg_32:$soffset, i32imm:$offset)> { 978 let mayStore = 0; 979 let mayLoad = 1; 980 981 // (2 * 4) + (8 * num_subregs) bytes maximum 982 int MaxSize = !add(!shl(!srl(vgpr_class.Size, 5), !add(UsesTmp, 3)), 8); 983 // Size field is unsigned char and cannot fit more. 984 let Size = !if(!le(MaxSize, 256), MaxSize, 252); 985 } 986 } // End UseNamedOperandTable = 1, VGPRSpill = 1, SchedRW = [WriteVMEM] 987} 988 989defm SI_SPILL_V32 : SI_SPILL_VGPR <VGPR_32>; 990defm SI_SPILL_V64 : SI_SPILL_VGPR <VReg_64>; 991defm SI_SPILL_V96 : SI_SPILL_VGPR <VReg_96>; 992defm SI_SPILL_V128 : SI_SPILL_VGPR <VReg_128>; 993defm SI_SPILL_V160 : SI_SPILL_VGPR <VReg_160>; 994defm SI_SPILL_V192 : SI_SPILL_VGPR <VReg_192>; 995defm SI_SPILL_V224 : SI_SPILL_VGPR <VReg_224>; 996defm SI_SPILL_V256 : SI_SPILL_VGPR <VReg_256>; 997defm SI_SPILL_V288 : SI_SPILL_VGPR <VReg_288>; 998defm SI_SPILL_V320 : SI_SPILL_VGPR <VReg_320>; 999defm SI_SPILL_V352 : SI_SPILL_VGPR <VReg_352>; 1000defm SI_SPILL_V384 : SI_SPILL_VGPR <VReg_384>; 1001defm SI_SPILL_V512 : SI_SPILL_VGPR <VReg_512>; 1002defm SI_SPILL_V1024 : SI_SPILL_VGPR <VReg_1024>; 1003 1004defm SI_SPILL_A32 : SI_SPILL_VGPR <AGPR_32, 1>; 1005defm SI_SPILL_A64 : SI_SPILL_VGPR <AReg_64, 1>; 1006defm SI_SPILL_A96 : SI_SPILL_VGPR <AReg_96, 1>; 1007defm SI_SPILL_A128 : SI_SPILL_VGPR <AReg_128, 1>; 1008defm SI_SPILL_A160 : SI_SPILL_VGPR <AReg_160, 1>; 1009defm SI_SPILL_A192 : SI_SPILL_VGPR <AReg_192, 1>; 1010defm SI_SPILL_A224 : SI_SPILL_VGPR <AReg_224, 1>; 1011defm SI_SPILL_A256 : SI_SPILL_VGPR <AReg_256, 1>; 1012defm SI_SPILL_A288 : SI_SPILL_VGPR <AReg_288, 1>; 1013defm SI_SPILL_A320 : SI_SPILL_VGPR <AReg_320, 1>; 1014defm SI_SPILL_A352 : SI_SPILL_VGPR <AReg_352, 1>; 1015defm SI_SPILL_A384 : SI_SPILL_VGPR <AReg_384, 1>; 1016defm SI_SPILL_A512 : SI_SPILL_VGPR <AReg_512, 1>; 1017defm SI_SPILL_A1024 : SI_SPILL_VGPR <AReg_1024, 1>; 1018 1019defm SI_SPILL_AV32 : SI_SPILL_VGPR <AV_32, 1>; 1020defm SI_SPILL_AV64 : SI_SPILL_VGPR <AV_64, 1>; 1021defm SI_SPILL_AV96 : SI_SPILL_VGPR <AV_96, 1>; 1022defm SI_SPILL_AV128 : SI_SPILL_VGPR <AV_128, 1>; 1023defm SI_SPILL_AV160 : SI_SPILL_VGPR <AV_160, 1>; 1024defm SI_SPILL_AV192 : SI_SPILL_VGPR <AV_192, 1>; 1025defm SI_SPILL_AV224 : SI_SPILL_VGPR <AV_224, 1>; 1026defm SI_SPILL_AV256 : SI_SPILL_VGPR <AV_256, 1>; 1027defm SI_SPILL_AV288 : SI_SPILL_VGPR <AV_288, 1>; 1028defm SI_SPILL_AV320 : SI_SPILL_VGPR <AV_320, 1>; 1029defm SI_SPILL_AV352 : SI_SPILL_VGPR <AV_352, 1>; 1030defm SI_SPILL_AV384 : SI_SPILL_VGPR <AV_384, 1>; 1031defm SI_SPILL_AV512 : SI_SPILL_VGPR <AV_512, 1>; 1032defm SI_SPILL_AV1024 : SI_SPILL_VGPR <AV_1024, 1>; 1033 1034let isConvergent = 1 in { 1035 defm SI_SPILL_WWM_V32 : SI_SPILL_VGPR <VGPR_32>; 1036 defm SI_SPILL_WWM_AV32 : SI_SPILL_VGPR <AV_32, 1>; 1037} 1038 1039def SI_PC_ADD_REL_OFFSET : SPseudoInstSI < 1040 (outs SReg_64:$dst), 1041 (ins si_ga:$ptr_lo, si_ga:$ptr_hi), 1042 [(set SReg_64:$dst, 1043 (i64 (SIpc_add_rel_offset tglobaladdr:$ptr_lo, tglobaladdr:$ptr_hi)))]> { 1044 let Defs = [SCC]; 1045} 1046 1047def : GCNPat < 1048 (SIpc_add_rel_offset tglobaladdr:$ptr_lo, 0), 1049 (SI_PC_ADD_REL_OFFSET $ptr_lo, (i32 0)) 1050>; 1051 1052def : GCNPat< 1053 (AMDGPUtrap timm:$trapid), 1054 (S_TRAP $trapid) 1055>; 1056 1057def : GCNPat< 1058 (AMDGPUelse i1:$src, bb:$target), 1059 (SI_ELSE $src, $target) 1060>; 1061 1062def : Pat < 1063 (int_amdgcn_kill i1:$src), 1064 (SI_KILL_I1_PSEUDO SCSrc_i1:$src, 0) 1065>; 1066 1067def : Pat < 1068 (int_amdgcn_kill (i1 (not i1:$src))), 1069 (SI_KILL_I1_PSEUDO SCSrc_i1:$src, -1) 1070>; 1071 1072def : Pat < 1073 (int_amdgcn_kill (i1 (setcc f32:$src, InlineImmFP32:$imm, cond:$cond))), 1074 (SI_KILL_F32_COND_IMM_PSEUDO VSrc_b32:$src, (bitcast_fpimm_to_i32 $imm), (cond_as_i32imm $cond)) 1075>; 1076 1077def : Pat < 1078 (int_amdgcn_wqm_demote i1:$src), 1079 (SI_DEMOTE_I1 SCSrc_i1:$src, 0) 1080>; 1081 1082def : Pat < 1083 (int_amdgcn_wqm_demote (i1 (not i1:$src))), 1084 (SI_DEMOTE_I1 SCSrc_i1:$src, -1) 1085>; 1086 1087 // TODO: we could add more variants for other types of conditionals 1088 1089def : Pat < 1090 (i64 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))), 1091 (COPY $src) // Return the SGPRs representing i1 src 1092>; 1093 1094def : Pat < 1095 (i32 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))), 1096 (COPY $src) // Return the SGPRs representing i1 src 1097>; 1098 1099//===----------------------------------------------------------------------===// 1100// VOP1 Patterns 1101//===----------------------------------------------------------------------===// 1102 1103multiclass f16_fp_Pats<Instruction cvt_f16_f32_inst_e64, Instruction cvt_f32_f16_inst_e64> { 1104 // f16_to_fp patterns 1105 def : GCNPat < 1106 (f32 (any_f16_to_fp i32:$src0)), 1107 (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src0) 1108 >; 1109 1110 def : GCNPat < 1111 (f32 (f16_to_fp (and_oneuse i32:$src0, 0x7fff))), 1112 (cvt_f32_f16_inst_e64 SRCMODS.ABS, $src0) 1113 >; 1114 1115 def : GCNPat < 1116 (f32 (f16_to_fp (i32 (srl_oneuse (and_oneuse i32:$src0, 0x7fff0000), (i32 16))))), 1117 (cvt_f32_f16_inst_e64 SRCMODS.ABS, (i32 (V_LSHRREV_B32_e64 (i32 16), i32:$src0))) 1118 >; 1119 1120 def : GCNPat < 1121 (f32 (f16_to_fp (or_oneuse i32:$src0, 0x8000))), 1122 (cvt_f32_f16_inst_e64 SRCMODS.NEG_ABS, $src0) 1123 >; 1124 1125 def : GCNPat < 1126 (f32 (f16_to_fp (xor_oneuse i32:$src0, 0x8000))), 1127 (cvt_f32_f16_inst_e64 SRCMODS.NEG, $src0) 1128 >; 1129 1130 def : GCNPat < 1131 (f64 (any_fpextend f16:$src)), 1132 (V_CVT_F64_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src)) 1133 >; 1134 1135 // fp_to_fp16 patterns 1136 def : GCNPat < 1137 (i32 (AMDGPUfp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), 1138 (cvt_f16_f32_inst_e64 $src0_modifiers, f32:$src0) 1139 >; 1140 1141 def : GCNPat < 1142 (i32 (fp_to_sint f16:$src)), 1143 (V_CVT_I32_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, VSrc_b32:$src)) 1144 >; 1145 1146 def : GCNPat < 1147 (i32 (fp_to_uint f16:$src)), 1148 (V_CVT_U32_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, VSrc_b32:$src)) 1149 >; 1150 1151 def : GCNPat < 1152 (f16 (sint_to_fp i32:$src)), 1153 (cvt_f16_f32_inst_e64 SRCMODS.NONE, (V_CVT_F32_I32_e32 VSrc_b32:$src)) 1154 >; 1155 1156 def : GCNPat < 1157 (f16 (uint_to_fp i32:$src)), 1158 (cvt_f16_f32_inst_e64 SRCMODS.NONE, (V_CVT_F32_U32_e32 VSrc_b32:$src)) 1159 >; 1160 1161 // This is only used on targets without half support 1162 // TODO: Introduce strict variant of AMDGPUfp_to_f16 and share custom lowering 1163 def : GCNPat < 1164 (i32 (strict_fp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), 1165 (cvt_f16_f32_inst_e64 $src0_modifiers, f32:$src0) 1166 >; 1167} 1168 1169let SubtargetPredicate = NotHasTrue16BitInsts in 1170defm : f16_fp_Pats<V_CVT_F16_F32_e64, V_CVT_F32_F16_e64>; 1171 1172let SubtargetPredicate = HasTrue16BitInsts in 1173defm : f16_fp_Pats<V_CVT_F16_F32_t16_e64, V_CVT_F32_F16_t16_e64>; 1174 1175//===----------------------------------------------------------------------===// 1176// VOP2 Patterns 1177//===----------------------------------------------------------------------===// 1178 1179// NoMods pattern used for mac. If there are any source modifiers then it's 1180// better to select mad instead of mac. 1181class FMADPat <ValueType vt, Instruction inst> 1182 : GCNPat <(vt (any_fmad (vt (VOP3NoMods vt:$src0)), 1183 (vt (VOP3NoMods vt:$src1)), 1184 (vt (VOP3NoMods vt:$src2)))), 1185 (inst SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 1186 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1187>; 1188 1189// Prefer mac form when there are no modifiers. 1190let AddedComplexity = 9 in { 1191let OtherPredicates = [HasMadMacF32Insts] in 1192def : FMADPat <f32, V_MAC_F32_e64>; 1193 1194// Don't allow source modifiers. If there are any source modifiers then it's 1195// better to select mad instead of mac. 1196let SubtargetPredicate = isGFX6GFX7GFX10, 1197 OtherPredicates = [HasMadMacF32Insts, NoFP32Denormals] in 1198def : GCNPat < 1199 (f32 (fadd (AMDGPUfmul_legacy (VOP3NoMods f32:$src0), 1200 (VOP3NoMods f32:$src1)), 1201 (VOP3NoMods f32:$src2))), 1202 (V_MAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 1203 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1204>; 1205 1206// Don't allow source modifiers. If there are any source modifiers then it's 1207// better to select fma instead of fmac. 1208let SubtargetPredicate = HasFmaLegacy32 in 1209def : GCNPat < 1210 (f32 (int_amdgcn_fma_legacy (VOP3NoMods f32:$src0), 1211 (VOP3NoMods f32:$src1), 1212 (VOP3NoMods f32:$src2))), 1213 (V_FMAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 1214 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1215>; 1216 1217let SubtargetPredicate = Has16BitInsts in 1218def : FMADPat <f16, V_MAC_F16_e64>; 1219} // AddedComplexity = 9 1220 1221let OtherPredicates = [HasMadMacF32Insts, NoFP32Denormals] in 1222def : GCNPat < 1223 (f32 (fadd (AMDGPUfmul_legacy (VOP3Mods f32:$src0, i32:$src0_mod), 1224 (VOP3Mods f32:$src1, i32:$src1_mod)), 1225 (VOP3Mods f32:$src2, i32:$src2_mod))), 1226 (V_MAD_LEGACY_F32_e64 $src0_mod, $src0, $src1_mod, $src1, 1227 $src2_mod, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1228>; 1229 1230class VOPSelectModsPat <ValueType vt> : GCNPat < 1231 (vt (select i1:$src0, (VOP3ModsNonCanonicalizing vt:$src1, i32:$src1_mods), 1232 (VOP3ModsNonCanonicalizing vt:$src2, i32:$src2_mods))), 1233 (V_CNDMASK_B32_e64 FP32InputMods:$src2_mods, VSrc_b32:$src2, 1234 FP32InputMods:$src1_mods, VSrc_b32:$src1, SSrc_i1:$src0) 1235>; 1236 1237class VOPSelectPat <ValueType vt> : GCNPat < 1238 (vt (select i1:$src0, vt:$src1, vt:$src2)), 1239 (V_CNDMASK_B32_e64 0, VSrc_b32:$src2, 0, VSrc_b32:$src1, SSrc_i1:$src0) 1240>; 1241 1242def : VOPSelectModsPat <i32>; 1243def : VOPSelectModsPat <f32>; 1244def : VOPSelectPat <f16>; 1245def : VOPSelectPat <i16>; 1246 1247let AddedComplexity = 1 in { 1248def : GCNPat < 1249 (i32 (add (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)), i32:$val)), 1250 (V_BCNT_U32_B32_e64 $popcnt, $val) 1251>; 1252} 1253 1254def : GCNPat < 1255 (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)), 1256 (V_BCNT_U32_B32_e64 VSrc_b32:$popcnt, (i32 0)) 1257>; 1258 1259def : GCNPat < 1260 (i16 (add (i16 (trunc (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)))), i16:$val)), 1261 (V_BCNT_U32_B32_e64 $popcnt, $val) 1262>; 1263 1264def : GCNPat < 1265 (i64 (DivergentUnaryFrag<ctpop> i64:$src)), 1266 (REG_SEQUENCE VReg_64, 1267 (V_BCNT_U32_B32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub1)), 1268 (i32 (V_BCNT_U32_B32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0)))), sub0, 1269 (i32 (V_MOV_B32_e32 (i32 0))), sub1) 1270>; 1271 1272/********** ============================================ **********/ 1273/********** Extraction, Insertion, Building and Casting **********/ 1274/********** ============================================ **********/ 1275 1276// Special case for 2 element vectors. REQ_SEQUENCE produces better code 1277// than an INSERT_SUBREG. 1278multiclass Insert_Element_V2<RegisterClass RC, ValueType elem_type, ValueType vec_type> { 1279 def : GCNPat < 1280 (insertelt vec_type:$vec, elem_type:$elem, 0), 1281 (REG_SEQUENCE RC, $elem, sub0, (elem_type (EXTRACT_SUBREG $vec, sub1)), sub1) 1282 >; 1283 1284 def : GCNPat < 1285 (insertelt vec_type:$vec, elem_type:$elem, 1), 1286 (REG_SEQUENCE RC, (elem_type (EXTRACT_SUBREG $vec, sub0)), sub0, $elem, sub1) 1287 >; 1288} 1289 1290foreach Index = 0-1 in { 1291 def Extract_Element_v2i32_#Index : Extract_Element < 1292 i32, v2i32, Index, !cast<SubRegIndex>(sub#Index) 1293 >; 1294 1295 def Extract_Element_v2f32_#Index : Extract_Element < 1296 f32, v2f32, Index, !cast<SubRegIndex>(sub#Index) 1297 >; 1298} 1299 1300defm : Insert_Element_V2 <SReg_64, i32, v2i32>; 1301defm : Insert_Element_V2 <SReg_64, f32, v2f32>; 1302 1303foreach Index = 0-2 in { 1304 def Extract_Element_v3i32_#Index : Extract_Element < 1305 i32, v3i32, Index, !cast<SubRegIndex>(sub#Index) 1306 >; 1307 def Insert_Element_v3i32_#Index : Insert_Element < 1308 i32, v3i32, Index, !cast<SubRegIndex>(sub#Index) 1309 >; 1310 1311 def Extract_Element_v3f32_#Index : Extract_Element < 1312 f32, v3f32, Index, !cast<SubRegIndex>(sub#Index) 1313 >; 1314 def Insert_Element_v3f32_#Index : Insert_Element < 1315 f32, v3f32, Index, !cast<SubRegIndex>(sub#Index) 1316 >; 1317} 1318 1319foreach Index = 0-3 in { 1320 def Extract_Element_v4i32_#Index : Extract_Element < 1321 i32, v4i32, Index, !cast<SubRegIndex>(sub#Index) 1322 >; 1323 def Insert_Element_v4i32_#Index : Insert_Element < 1324 i32, v4i32, Index, !cast<SubRegIndex>(sub#Index) 1325 >; 1326 1327 def Extract_Element_v4f32_#Index : Extract_Element < 1328 f32, v4f32, Index, !cast<SubRegIndex>(sub#Index) 1329 >; 1330 def Insert_Element_v4f32_#Index : Insert_Element < 1331 f32, v4f32, Index, !cast<SubRegIndex>(sub#Index) 1332 >; 1333} 1334 1335foreach Index = 0-4 in { 1336 def Extract_Element_v5i32_#Index : Extract_Element < 1337 i32, v5i32, Index, !cast<SubRegIndex>(sub#Index) 1338 >; 1339 def Insert_Element_v5i32_#Index : Insert_Element < 1340 i32, v5i32, Index, !cast<SubRegIndex>(sub#Index) 1341 >; 1342 1343 def Extract_Element_v5f32_#Index : Extract_Element < 1344 f32, v5f32, Index, !cast<SubRegIndex>(sub#Index) 1345 >; 1346 def Insert_Element_v5f32_#Index : Insert_Element < 1347 f32, v5f32, Index, !cast<SubRegIndex>(sub#Index) 1348 >; 1349} 1350 1351foreach Index = 0-5 in { 1352 def Extract_Element_v6i32_#Index : Extract_Element < 1353 i32, v6i32, Index, !cast<SubRegIndex>(sub#Index) 1354 >; 1355 def Insert_Element_v6i32_#Index : Insert_Element < 1356 i32, v6i32, Index, !cast<SubRegIndex>(sub#Index) 1357 >; 1358 1359 def Extract_Element_v6f32_#Index : Extract_Element < 1360 f32, v6f32, Index, !cast<SubRegIndex>(sub#Index) 1361 >; 1362 def Insert_Element_v6f32_#Index : Insert_Element < 1363 f32, v6f32, Index, !cast<SubRegIndex>(sub#Index) 1364 >; 1365} 1366 1367foreach Index = 0-6 in { 1368 def Extract_Element_v7i32_#Index : Extract_Element < 1369 i32, v7i32, Index, !cast<SubRegIndex>(sub#Index) 1370 >; 1371 def Insert_Element_v7i32_#Index : Insert_Element < 1372 i32, v7i32, Index, !cast<SubRegIndex>(sub#Index) 1373 >; 1374 1375 def Extract_Element_v7f32_#Index : Extract_Element < 1376 f32, v7f32, Index, !cast<SubRegIndex>(sub#Index) 1377 >; 1378 def Insert_Element_v7f32_#Index : Insert_Element < 1379 f32, v7f32, Index, !cast<SubRegIndex>(sub#Index) 1380 >; 1381} 1382 1383foreach Index = 0-7 in { 1384 def Extract_Element_v8i32_#Index : Extract_Element < 1385 i32, v8i32, Index, !cast<SubRegIndex>(sub#Index) 1386 >; 1387 def Insert_Element_v8i32_#Index : Insert_Element < 1388 i32, v8i32, Index, !cast<SubRegIndex>(sub#Index) 1389 >; 1390 1391 def Extract_Element_v8f32_#Index : Extract_Element < 1392 f32, v8f32, Index, !cast<SubRegIndex>(sub#Index) 1393 >; 1394 def Insert_Element_v8f32_#Index : Insert_Element < 1395 f32, v8f32, Index, !cast<SubRegIndex>(sub#Index) 1396 >; 1397} 1398 1399foreach Index = 0-8 in { 1400 def Extract_Element_v9i32_#Index : Extract_Element < 1401 i32, v9i32, Index, !cast<SubRegIndex>(sub#Index) 1402 >; 1403 def Insert_Element_v9i32_#Index : Insert_Element < 1404 i32, v9i32, Index, !cast<SubRegIndex>(sub#Index) 1405 >; 1406 1407 def Extract_Element_v9f32_#Index : Extract_Element < 1408 f32, v9f32, Index, !cast<SubRegIndex>(sub#Index) 1409 >; 1410 def Insert_Element_v9f32_#Index : Insert_Element < 1411 f32, v9f32, Index, !cast<SubRegIndex>(sub#Index) 1412 >; 1413} 1414 1415foreach Index = 0-9 in { 1416 def Extract_Element_v10i32_#Index : Extract_Element < 1417 i32, v10i32, Index, !cast<SubRegIndex>(sub#Index) 1418 >; 1419 def Insert_Element_v10i32_#Index : Insert_Element < 1420 i32, v10i32, Index, !cast<SubRegIndex>(sub#Index) 1421 >; 1422 1423 def Extract_Element_v10f32_#Index : Extract_Element < 1424 f32, v10f32, Index, !cast<SubRegIndex>(sub#Index) 1425 >; 1426 def Insert_Element_v10f32_#Index : Insert_Element < 1427 f32, v10f32, Index, !cast<SubRegIndex>(sub#Index) 1428 >; 1429} 1430 1431foreach Index = 0-10 in { 1432 def Extract_Element_v11i32_#Index : Extract_Element < 1433 i32, v11i32, Index, !cast<SubRegIndex>(sub#Index) 1434 >; 1435 def Insert_Element_v11i32_#Index : Insert_Element < 1436 i32, v11i32, Index, !cast<SubRegIndex>(sub#Index) 1437 >; 1438 1439 def Extract_Element_v11f32_#Index : Extract_Element < 1440 f32, v11f32, Index, !cast<SubRegIndex>(sub#Index) 1441 >; 1442 def Insert_Element_v11f32_#Index : Insert_Element < 1443 f32, v11f32, Index, !cast<SubRegIndex>(sub#Index) 1444 >; 1445} 1446 1447foreach Index = 0-11 in { 1448 def Extract_Element_v12i32_#Index : Extract_Element < 1449 i32, v12i32, Index, !cast<SubRegIndex>(sub#Index) 1450 >; 1451 def Insert_Element_v12i32_#Index : Insert_Element < 1452 i32, v12i32, Index, !cast<SubRegIndex>(sub#Index) 1453 >; 1454 1455 def Extract_Element_v12f32_#Index : Extract_Element < 1456 f32, v12f32, Index, !cast<SubRegIndex>(sub#Index) 1457 >; 1458 def Insert_Element_v12f32_#Index : Insert_Element < 1459 f32, v12f32, Index, !cast<SubRegIndex>(sub#Index) 1460 >; 1461} 1462 1463foreach Index = 0-15 in { 1464 def Extract_Element_v16i32_#Index : Extract_Element < 1465 i32, v16i32, Index, !cast<SubRegIndex>(sub#Index) 1466 >; 1467 def Insert_Element_v16i32_#Index : Insert_Element < 1468 i32, v16i32, Index, !cast<SubRegIndex>(sub#Index) 1469 >; 1470 1471 def Extract_Element_v16f32_#Index : Extract_Element < 1472 f32, v16f32, Index, !cast<SubRegIndex>(sub#Index) 1473 >; 1474 def Insert_Element_v16f32_#Index : Insert_Element < 1475 f32, v16f32, Index, !cast<SubRegIndex>(sub#Index) 1476 >; 1477} 1478 1479 1480foreach Index = 0-31 in { 1481 def Extract_Element_v32i32_#Index : Extract_Element < 1482 i32, v32i32, Index, !cast<SubRegIndex>(sub#Index) 1483 >; 1484 1485 def Insert_Element_v32i32_#Index : Insert_Element < 1486 i32, v32i32, Index, !cast<SubRegIndex>(sub#Index) 1487 >; 1488 1489 def Extract_Element_v32f32_#Index : Extract_Element < 1490 f32, v32f32, Index, !cast<SubRegIndex>(sub#Index) 1491 >; 1492 1493 def Insert_Element_v32f32_#Index : Insert_Element < 1494 f32, v32f32, Index, !cast<SubRegIndex>(sub#Index) 1495 >; 1496} 1497 1498// FIXME: Why do only some of these type combinations for SReg and 1499// VReg? 1500// 16-bit bitcast 1501def : BitConvert <i16, f16, VGPR_32>; 1502def : BitConvert <f16, i16, VGPR_32>; 1503def : BitConvert <f16, bf16, VGPR_32>; 1504def : BitConvert <bf16, f16, VGPR_32>; 1505 1506def : BitConvert <i16, f16, SReg_32>; 1507def : BitConvert <f16, i16, SReg_32>; 1508def : BitConvert <f16, bf16, SReg_32>; 1509def : BitConvert <bf16, f16, SReg_32>; 1510 1511def : BitConvert <i16, bf16, VGPR_32>; 1512def : BitConvert <bf16, i16, VGPR_32>; 1513def : BitConvert <i16, bf16, SReg_32>; 1514def : BitConvert <bf16, i16, SReg_32>; 1515 1516// 32-bit bitcast 1517def : BitConvert <i32, f32, VGPR_32>; 1518def : BitConvert <f32, i32, VGPR_32>; 1519def : BitConvert <i32, f32, SReg_32>; 1520def : BitConvert <f32, i32, SReg_32>; 1521def : BitConvert <v2i16, i32, SReg_32>; 1522def : BitConvert <i32, v2i16, SReg_32>; 1523def : BitConvert <v2f16, i32, SReg_32>; 1524def : BitConvert <i32, v2f16, SReg_32>; 1525def : BitConvert <v2i16, v2f16, SReg_32>; 1526def : BitConvert <v2f16, v2i16, SReg_32>; 1527def : BitConvert <v2f16, f32, SReg_32>; 1528def : BitConvert <f32, v2f16, SReg_32>; 1529def : BitConvert <v2i16, f32, SReg_32>; 1530def : BitConvert <f32, v2i16, SReg_32>; 1531def : BitConvert <v2bf16, i32, SReg_32>; 1532def : BitConvert <i32, v2bf16, SReg_32>; 1533def : BitConvert <v2bf16, i32, VGPR_32>; 1534def : BitConvert <i32, v2bf16, VGPR_32>; 1535def : BitConvert <v2bf16, v2i16, SReg_32>; 1536def : BitConvert <v2i16, v2bf16, SReg_32>; 1537def : BitConvert <v2bf16, v2i16, VGPR_32>; 1538def : BitConvert <v2i16, v2bf16, VGPR_32>; 1539def : BitConvert <v2bf16, v2f16, SReg_32>; 1540def : BitConvert <v2f16, v2bf16, SReg_32>; 1541def : BitConvert <v2bf16, v2f16, VGPR_32>; 1542def : BitConvert <v2f16, v2bf16, VGPR_32>; 1543def : BitConvert <f32, v2bf16, VGPR_32>; 1544def : BitConvert <v2bf16, f32, VGPR_32>; 1545def : BitConvert <f32, v2bf16, SReg_32>; 1546def : BitConvert <v2bf16, f32, SReg_32>; 1547 1548 1549// 64-bit bitcast 1550def : BitConvert <i64, f64, VReg_64>; 1551def : BitConvert <f64, i64, VReg_64>; 1552def : BitConvert <v2i32, v2f32, VReg_64>; 1553def : BitConvert <v2f32, v2i32, VReg_64>; 1554def : BitConvert <i64, v2i32, VReg_64>; 1555def : BitConvert <v2i32, i64, VReg_64>; 1556def : BitConvert <i64, v2f32, VReg_64>; 1557def : BitConvert <v2f32, i64, VReg_64>; 1558def : BitConvert <f64, v2f32, VReg_64>; 1559def : BitConvert <v2f32, f64, VReg_64>; 1560def : BitConvert <f64, v2i32, VReg_64>; 1561def : BitConvert <v2i32, f64, VReg_64>; 1562def : BitConvert <v4i16, v4f16, VReg_64>; 1563def : BitConvert <v4f16, v4i16, VReg_64>; 1564def : BitConvert <v4bf16, v2i32, VReg_64>; 1565def : BitConvert <v2i32, v4bf16, VReg_64>; 1566def : BitConvert <v4bf16, i64, VReg_64>; 1567def : BitConvert <i64, v4bf16, VReg_64>; 1568def : BitConvert <v4bf16, v4i16, VReg_64>; 1569def : BitConvert <v4i16, v4bf16, VReg_64>; 1570def : BitConvert <v4bf16, v4f16, VReg_64>; 1571def : BitConvert <v4f16, v4bf16, VReg_64>; 1572def : BitConvert <v4bf16, v2f32, VReg_64>; 1573def : BitConvert <v2f32, v4bf16, VReg_64>; 1574def : BitConvert <v4bf16, f64, VReg_64>; 1575def : BitConvert <f64, v4bf16, VReg_64>; 1576 1577 1578// FIXME: Make SGPR 1579def : BitConvert <v2i32, v4f16, VReg_64>; 1580def : BitConvert <v4f16, v2i32, VReg_64>; 1581def : BitConvert <v2i32, v4f16, VReg_64>; 1582def : BitConvert <v2i32, v4i16, VReg_64>; 1583def : BitConvert <v4i16, v2i32, VReg_64>; 1584def : BitConvert <v2f32, v4f16, VReg_64>; 1585def : BitConvert <v4f16, v2f32, VReg_64>; 1586def : BitConvert <v2f32, v4i16, VReg_64>; 1587def : BitConvert <v4i16, v2f32, VReg_64>; 1588def : BitConvert <v4i16, f64, VReg_64>; 1589def : BitConvert <v4f16, f64, VReg_64>; 1590def : BitConvert <f64, v4i16, VReg_64>; 1591def : BitConvert <f64, v4f16, VReg_64>; 1592def : BitConvert <v4i16, i64, VReg_64>; 1593def : BitConvert <v4f16, i64, VReg_64>; 1594def : BitConvert <i64, v4i16, VReg_64>; 1595def : BitConvert <i64, v4f16, VReg_64>; 1596 1597def : BitConvert <v4i32, v4f32, VReg_128>; 1598def : BitConvert <v4f32, v4i32, VReg_128>; 1599 1600// 96-bit bitcast 1601def : BitConvert <v3i32, v3f32, SGPR_96>; 1602def : BitConvert <v3f32, v3i32, SGPR_96>; 1603 1604// 128-bit bitcast 1605def : BitConvert <v2i64, v4i32, SReg_128>; 1606def : BitConvert <v4i32, v2i64, SReg_128>; 1607def : BitConvert <v2f64, v4f32, VReg_128>; 1608def : BitConvert <v2f64, v4i32, VReg_128>; 1609def : BitConvert <v4f32, v2f64, VReg_128>; 1610def : BitConvert <v4i32, v2f64, VReg_128>; 1611def : BitConvert <v2i64, v2f64, VReg_128>; 1612def : BitConvert <v2f64, v2i64, VReg_128>; 1613def : BitConvert <v4f32, v2i64, VReg_128>; 1614def : BitConvert <v2i64, v4f32, VReg_128>; 1615def : BitConvert <v8i16, v4i32, SReg_128>; 1616def : BitConvert <v4i32, v8i16, SReg_128>; 1617def : BitConvert <v8f16, v4f32, VReg_128>; 1618def : BitConvert <v8f16, v4i32, VReg_128>; 1619def : BitConvert <v4f32, v8f16, VReg_128>; 1620def : BitConvert <v4i32, v8f16, VReg_128>; 1621def : BitConvert <v8i16, v8f16, VReg_128>; 1622def : BitConvert <v8f16, v8i16, VReg_128>; 1623def : BitConvert <v4f32, v8i16, VReg_128>; 1624def : BitConvert <v8i16, v4f32, VReg_128>; 1625def : BitConvert <v8i16, v8f16, SReg_128>; 1626def : BitConvert <v8i16, v2i64, SReg_128>; 1627def : BitConvert <v8i16, v2f64, SReg_128>; 1628def : BitConvert <v8f16, v2i64, SReg_128>; 1629def : BitConvert <v8f16, v2f64, SReg_128>; 1630def : BitConvert <v8f16, v8i16, SReg_128>; 1631def : BitConvert <v2i64, v8i16, SReg_128>; 1632def : BitConvert <v2f64, v8i16, SReg_128>; 1633def : BitConvert <v2i64, v8f16, SReg_128>; 1634def : BitConvert <v2f64, v8f16, SReg_128>; 1635 1636def : BitConvert <v4i32, v8bf16, SReg_128>; 1637def : BitConvert <v8bf16, v4i32, SReg_128>; 1638def : BitConvert <v4i32, v8bf16, VReg_128>; 1639def : BitConvert <v8bf16, v4i32, VReg_128>; 1640 1641def : BitConvert <v4f32, v8bf16, SReg_128>; 1642def : BitConvert <v8bf16, v4f32, SReg_128>; 1643def : BitConvert <v4f32, v8bf16, VReg_128>; 1644def : BitConvert <v8bf16, v4f32, VReg_128>; 1645 1646def : BitConvert <v8i16, v8bf16, SReg_128>; 1647def : BitConvert <v8bf16, v8i16, SReg_128>; 1648def : BitConvert <v8i16, v8bf16, VReg_128>; 1649def : BitConvert <v8bf16, v8i16, VReg_128>; 1650 1651def : BitConvert <v8f16, v8bf16, SReg_128>; 1652def : BitConvert <v8bf16, v8f16, SReg_128>; 1653def : BitConvert <v8f16, v8bf16, VReg_128>; 1654def : BitConvert <v8bf16, v8f16, VReg_128>; 1655 1656def : BitConvert <v2f64, v8bf16, SReg_128>; 1657def : BitConvert <v8bf16, v2f64, SReg_128>; 1658def : BitConvert <v2f64, v8bf16, VReg_128>; 1659def : BitConvert <v8bf16, v2f64, VReg_128>; 1660 1661def : BitConvert <v2i64, v8bf16, SReg_128>; 1662def : BitConvert <v8bf16, v2i64, SReg_128>; 1663def : BitConvert <v2i64, v8bf16, VReg_128>; 1664def : BitConvert <v8bf16, v2i64, VReg_128>; 1665 1666 1667// 160-bit bitcast 1668def : BitConvert <v5i32, v5f32, SReg_160>; 1669def : BitConvert <v5f32, v5i32, SReg_160>; 1670def : BitConvert <v5i32, v5f32, VReg_160>; 1671def : BitConvert <v5f32, v5i32, VReg_160>; 1672 1673// 192-bit bitcast 1674def : BitConvert <v6i32, v6f32, SReg_192>; 1675def : BitConvert <v6f32, v6i32, SReg_192>; 1676def : BitConvert <v6i32, v6f32, VReg_192>; 1677def : BitConvert <v6f32, v6i32, VReg_192>; 1678def : BitConvert <v3i64, v3f64, VReg_192>; 1679def : BitConvert <v3f64, v3i64, VReg_192>; 1680def : BitConvert <v3i64, v6i32, VReg_192>; 1681def : BitConvert <v3i64, v6f32, VReg_192>; 1682def : BitConvert <v3f64, v6i32, VReg_192>; 1683def : BitConvert <v3f64, v6f32, VReg_192>; 1684def : BitConvert <v6i32, v3i64, VReg_192>; 1685def : BitConvert <v6f32, v3i64, VReg_192>; 1686def : BitConvert <v6i32, v3f64, VReg_192>; 1687def : BitConvert <v6f32, v3f64, VReg_192>; 1688 1689// 224-bit bitcast 1690def : BitConvert <v7i32, v7f32, SReg_224>; 1691def : BitConvert <v7f32, v7i32, SReg_224>; 1692def : BitConvert <v7i32, v7f32, VReg_224>; 1693def : BitConvert <v7f32, v7i32, VReg_224>; 1694 1695// 256-bit bitcast 1696def : BitConvert <v8i32, v8f32, SReg_256>; 1697def : BitConvert <v8f32, v8i32, SReg_256>; 1698def : BitConvert <v8i32, v8f32, VReg_256>; 1699def : BitConvert <v8f32, v8i32, VReg_256>; 1700def : BitConvert <v4i64, v4f64, VReg_256>; 1701def : BitConvert <v4f64, v4i64, VReg_256>; 1702def : BitConvert <v4i64, v8i32, VReg_256>; 1703def : BitConvert <v4i64, v8f32, VReg_256>; 1704def : BitConvert <v4f64, v8i32, VReg_256>; 1705def : BitConvert <v4f64, v8f32, VReg_256>; 1706def : BitConvert <v8i32, v4i64, VReg_256>; 1707def : BitConvert <v8f32, v4i64, VReg_256>; 1708def : BitConvert <v8i32, v4f64, VReg_256>; 1709def : BitConvert <v8f32, v4f64, VReg_256>; 1710def : BitConvert <v16i16, v16f16, SReg_256>; 1711def : BitConvert <v16f16, v16i16, SReg_256>; 1712def : BitConvert <v16i16, v16f16, VReg_256>; 1713def : BitConvert <v16f16, v16i16, VReg_256>; 1714def : BitConvert <v16f16, v8i32, VReg_256>; 1715def : BitConvert <v16i16, v8i32, VReg_256>; 1716def : BitConvert <v16f16, v8f32, VReg_256>; 1717def : BitConvert <v16i16, v8f32, VReg_256>; 1718def : BitConvert <v8i32, v16f16, VReg_256>; 1719def : BitConvert <v8i32, v16i16, VReg_256>; 1720def : BitConvert <v8f32, v16f16, VReg_256>; 1721def : BitConvert <v8f32, v16i16, VReg_256>; 1722def : BitConvert <v16f16, v4i64, VReg_256>; 1723def : BitConvert <v16i16, v4i64, VReg_256>; 1724def : BitConvert <v16f16, v4f64, VReg_256>; 1725def : BitConvert <v16i16, v4f64, VReg_256>; 1726def : BitConvert <v4i64, v16f16, VReg_256>; 1727def : BitConvert <v4i64, v16i16, VReg_256>; 1728def : BitConvert <v4f64, v16f16, VReg_256>; 1729def : BitConvert <v4f64, v16i16, VReg_256>; 1730 1731 1732def : BitConvert <v8i32, v16bf16, VReg_256>; 1733def : BitConvert <v16bf16, v8i32, VReg_256>; 1734def : BitConvert <v8f32, v16bf16, VReg_256>; 1735def : BitConvert <v16bf16, v8f32, VReg_256>; 1736def : BitConvert <v4i64, v16bf16, VReg_256>; 1737def : BitConvert <v16bf16, v4i64, VReg_256>; 1738def : BitConvert <v4f64, v16bf16, VReg_256>; 1739def : BitConvert <v16bf16, v4f64, VReg_256>; 1740 1741 1742 1743def : BitConvert <v16i16, v16bf16, SReg_256>; 1744def : BitConvert <v16bf16, v16i16, SReg_256>; 1745def : BitConvert <v16i16, v16bf16, VReg_256>; 1746def : BitConvert <v16bf16, v16i16, VReg_256>; 1747 1748def : BitConvert <v16f16, v16bf16, SReg_256>; 1749def : BitConvert <v16bf16, v16f16, SReg_256>; 1750def : BitConvert <v16f16, v16bf16, VReg_256>; 1751def : BitConvert <v16bf16, v16f16, VReg_256>; 1752 1753 1754 1755 1756// 288-bit bitcast 1757def : BitConvert <v9i32, v9f32, SReg_288>; 1758def : BitConvert <v9f32, v9i32, SReg_288>; 1759def : BitConvert <v9i32, v9f32, VReg_288>; 1760def : BitConvert <v9f32, v9i32, VReg_288>; 1761 1762// 320-bit bitcast 1763def : BitConvert <v10i32, v10f32, SReg_320>; 1764def : BitConvert <v10f32, v10i32, SReg_320>; 1765def : BitConvert <v10i32, v10f32, VReg_320>; 1766def : BitConvert <v10f32, v10i32, VReg_320>; 1767 1768// 320-bit bitcast 1769def : BitConvert <v11i32, v11f32, SReg_352>; 1770def : BitConvert <v11f32, v11i32, SReg_352>; 1771def : BitConvert <v11i32, v11f32, VReg_352>; 1772def : BitConvert <v11f32, v11i32, VReg_352>; 1773 1774// 384-bit bitcast 1775def : BitConvert <v12i32, v12f32, SReg_384>; 1776def : BitConvert <v12f32, v12i32, SReg_384>; 1777def : BitConvert <v12i32, v12f32, VReg_384>; 1778def : BitConvert <v12f32, v12i32, VReg_384>; 1779 1780// 512-bit bitcast 1781def : BitConvert <v32f16, v32i16, VReg_512>; 1782def : BitConvert <v32i16, v32f16, VReg_512>; 1783def : BitConvert <v32f16, v16i32, VReg_512>; 1784def : BitConvert <v32f16, v16f32, VReg_512>; 1785def : BitConvert <v16f32, v32f16, VReg_512>; 1786def : BitConvert <v16i32, v32f16, VReg_512>; 1787def : BitConvert <v32i16, v16i32, VReg_512>; 1788def : BitConvert <v32i16, v16f32, VReg_512>; 1789def : BitConvert <v16f32, v32i16, VReg_512>; 1790def : BitConvert <v16i32, v32i16, VReg_512>; 1791def : BitConvert <v16i32, v16f32, VReg_512>; 1792def : BitConvert <v16f32, v16i32, VReg_512>; 1793def : BitConvert <v8i64, v8f64, VReg_512>; 1794def : BitConvert <v8f64, v8i64, VReg_512>; 1795def : BitConvert <v8i64, v16i32, VReg_512>; 1796def : BitConvert <v8f64, v16i32, VReg_512>; 1797def : BitConvert <v16i32, v8i64, VReg_512>; 1798def : BitConvert <v16i32, v8f64, VReg_512>; 1799def : BitConvert <v8i64, v16f32, VReg_512>; 1800def : BitConvert <v8f64, v16f32, VReg_512>; 1801def : BitConvert <v16f32, v8i64, VReg_512>; 1802def : BitConvert <v16f32, v8f64, VReg_512>; 1803 1804 1805 1806def : BitConvert <v32bf16, v32i16, VReg_512>; 1807def : BitConvert <v32i16, v32bf16, VReg_512>; 1808def : BitConvert <v32bf16, v32i16, SReg_512>; 1809def : BitConvert <v32i16, v32bf16, SReg_512>; 1810 1811def : BitConvert <v32bf16, v32f16, VReg_512>; 1812def : BitConvert <v32f16, v32bf16, VReg_512>; 1813def : BitConvert <v32bf16, v32f16, SReg_512>; 1814def : BitConvert <v32f16, v32bf16, SReg_512>; 1815 1816def : BitConvert <v32bf16, v16i32, VReg_512>; 1817def : BitConvert <v16i32, v32bf16, VReg_512>; 1818def : BitConvert <v32bf16, v16i32, SReg_512>; 1819def : BitConvert <v16i32, v32bf16, SReg_512>; 1820 1821def : BitConvert <v32bf16, v16f32, VReg_512>; 1822def : BitConvert <v16f32, v32bf16, VReg_512>; 1823def : BitConvert <v32bf16, v16f32, SReg_512>; 1824def : BitConvert <v16f32, v32bf16, SReg_512>; 1825 1826def : BitConvert <v32bf16, v8f64, VReg_512>; 1827def : BitConvert <v8f64, v32bf16, VReg_512>; 1828def : BitConvert <v32bf16, v8f64, SReg_512>; 1829def : BitConvert <v8f64, v32bf16, SReg_512>; 1830 1831def : BitConvert <v32bf16, v8i64, VReg_512>; 1832def : BitConvert <v8i64, v32bf16, VReg_512>; 1833def : BitConvert <v32bf16, v8i64, SReg_512>; 1834def : BitConvert <v8i64, v32bf16, SReg_512>; 1835 1836// 1024-bit bitcast 1837def : BitConvert <v32i32, v32f32, VReg_1024>; 1838def : BitConvert <v32f32, v32i32, VReg_1024>; 1839def : BitConvert <v16i64, v16f64, VReg_1024>; 1840def : BitConvert <v16f64, v16i64, VReg_1024>; 1841def : BitConvert <v16i64, v32i32, VReg_1024>; 1842def : BitConvert <v32i32, v16i64, VReg_1024>; 1843def : BitConvert <v16f64, v32f32, VReg_1024>; 1844def : BitConvert <v32f32, v16f64, VReg_1024>; 1845def : BitConvert <v16i64, v32f32, VReg_1024>; 1846def : BitConvert <v32i32, v16f64, VReg_1024>; 1847def : BitConvert <v16f64, v32i32, VReg_1024>; 1848def : BitConvert <v32f32, v16i64, VReg_1024>; 1849 1850 1851/********** =================== **********/ 1852/********** Src & Dst modifiers **********/ 1853/********** =================== **********/ 1854 1855 1856// If denormals are not enabled, it only impacts the compare of the 1857// inputs. The output result is not flushed. 1858class ClampPat<Instruction inst, ValueType vt> : GCNPat < 1859 (vt (AMDGPUclamp (VOP3Mods vt:$src0, i32:$src0_modifiers))), 1860 (inst i32:$src0_modifiers, vt:$src0, 1861 i32:$src0_modifiers, vt:$src0, DSTCLAMP.ENABLE, DSTOMOD.NONE) 1862>; 1863 1864def : ClampPat<V_MAX_F32_e64, f32>; 1865let SubtargetPredicate = isNotGFX12Plus in 1866def : ClampPat<V_MAX_F64_e64, f64>; 1867let SubtargetPredicate = isGFX12Plus in 1868def : ClampPat<V_MAX_NUM_F64_e64, f64>; 1869let SubtargetPredicate = NotHasTrue16BitInsts in 1870def : ClampPat<V_MAX_F16_e64, f16>; 1871let SubtargetPredicate = UseRealTrue16Insts in 1872def : ClampPat<V_MAX_F16_t16_e64, f16>; 1873let SubtargetPredicate = UseFakeTrue16Insts in 1874def : ClampPat<V_MAX_F16_fake16_e64, f16>; 1875 1876let SubtargetPredicate = HasVOP3PInsts in { 1877def : GCNPat < 1878 (v2f16 (AMDGPUclamp (VOP3PMods v2f16:$src0, i32:$src0_modifiers))), 1879 (V_PK_MAX_F16 $src0_modifiers, $src0, 1880 $src0_modifiers, $src0, DSTCLAMP.ENABLE) 1881>; 1882} 1883 1884 1885/********** ================================ **********/ 1886/********** Floating point absolute/negative **********/ 1887/********** ================================ **********/ 1888 1889def : GCNPat < 1890 (UniformUnaryFrag<fneg> (fabs (f32 SReg_32:$src))), 1891 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000))) // Set sign bit 1892>; 1893 1894def : GCNPat < 1895 (UniformUnaryFrag<fabs> (f32 SReg_32:$src)), 1896 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fffffff))) 1897>; 1898 1899def : GCNPat < 1900 (UniformUnaryFrag<fneg> (f32 SReg_32:$src)), 1901 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000))) 1902>; 1903 1904def : GCNPat < 1905 (UniformUnaryFrag<fneg> (f16 SReg_32:$src)), 1906 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) 1907>; 1908 1909def : GCNPat < 1910 (UniformUnaryFrag<fabs> (f16 SReg_32:$src)), 1911 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00007fff))) 1912>; 1913 1914def : GCNPat < 1915 (UniformUnaryFrag<fneg> (fabs (f16 SReg_32:$src))), 1916 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) // Set sign bit 1917>; 1918 1919def : GCNPat < 1920 (UniformUnaryFrag<fneg> (v2f16 SReg_32:$src)), 1921 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) 1922>; 1923 1924def : GCNPat < 1925 (UniformUnaryFrag<fabs> (v2f16 SReg_32:$src)), 1926 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fff7fff))) 1927>; 1928 1929// This is really (fneg (fabs v2f16:$src)) 1930// 1931// fabs is not reported as free because there is modifier for it in 1932// VOP3P instructions, so it is turned into the bit op. 1933def : GCNPat < 1934 (UniformUnaryFrag<fneg> (v2f16 (bitconvert (and_oneuse (i32 SReg_32:$src), 0x7fff7fff)))), 1935 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit 1936>; 1937 1938def : GCNPat < 1939 (UniformUnaryFrag<fneg> (v2f16 (fabs SReg_32:$src))), 1940 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit 1941>; 1942 1943 1944// COPY_TO_REGCLASS is needed to avoid using SCC from S_XOR_B32 instead 1945// of the real value. 1946def : GCNPat < 1947 (UniformUnaryFrag<fneg> (v2f32 SReg_64:$src)), 1948 (v2f32 (REG_SEQUENCE SReg_64, 1949 (f32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub0)), 1950 (i32 (S_MOV_B32 (i32 0x80000000)))), 1951 SReg_32)), sub0, 1952 (f32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub1)), 1953 (i32 (S_MOV_B32 (i32 0x80000000)))), 1954 SReg_32)), sub1)) 1955>; 1956 1957def : GCNPat < 1958 (UniformUnaryFrag<fabs> (v2f32 SReg_64:$src)), 1959 (v2f32 (REG_SEQUENCE SReg_64, 1960 (f32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG $src, sub0)), 1961 (i32 (S_MOV_B32 (i32 0x7fffffff)))), 1962 SReg_32)), sub0, 1963 (f32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG $src, sub1)), 1964 (i32 (S_MOV_B32 (i32 0x7fffffff)))), 1965 SReg_32)), sub1)) 1966>; 1967 1968def : GCNPat < 1969 (UniformUnaryFrag<fneg> (fabs (v2f32 SReg_64:$src))), 1970 (v2f32 (REG_SEQUENCE SReg_64, 1971 (f32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG $src, sub0)), 1972 (i32 (S_MOV_B32 (i32 0x80000000)))), 1973 SReg_32)), sub0, 1974 (f32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG $src, sub1)), 1975 (i32 (S_MOV_B32 (i32 0x80000000)))), 1976 SReg_32)), sub1)) 1977>; 1978 1979// FIXME: Use S_BITSET0_B32/B64? 1980def : GCNPat < 1981 (UniformUnaryFrag<fabs> (f64 SReg_64:$src)), 1982 (REG_SEQUENCE SReg_64, 1983 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)), 1984 sub0, 1985 (i32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)), 1986 (S_MOV_B32 (i32 0x7fffffff))), SReg_32)), // Set sign bit. 1987 sub1) 1988>; 1989 1990def : GCNPat < 1991 (UniformUnaryFrag<fneg> (f64 SReg_64:$src)), 1992 (REG_SEQUENCE SReg_64, 1993 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)), 1994 sub0, 1995 (i32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)), 1996 (i32 (S_MOV_B32 (i32 0x80000000)))), SReg_32)), 1997 sub1) 1998>; 1999 2000def : GCNPat < 2001 (UniformUnaryFrag<fneg> (fabs (f64 SReg_64:$src))), 2002 (REG_SEQUENCE SReg_64, 2003 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)), 2004 sub0, 2005 (i32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)), 2006 (S_MOV_B32 (i32 0x80000000))), SReg_32)),// Set sign bit. 2007 sub1) 2008>; 2009 2010 2011def : GCNPat < 2012 (fneg (fabs (f32 VGPR_32:$src))), 2013 (V_OR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src) // Set sign bit 2014>; 2015 2016def : GCNPat < 2017 (fabs (f32 VGPR_32:$src)), 2018 (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), VGPR_32:$src) 2019>; 2020 2021def : GCNPat < 2022 (fneg (f32 VGPR_32:$src)), 2023 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src) 2024>; 2025 2026def : GCNPat < 2027 (fabs (f16 VGPR_32:$src)), 2028 (V_AND_B32_e64 (S_MOV_B32 (i32 0x00007fff)), VGPR_32:$src) 2029>; 2030 2031def : GCNPat < 2032 (fneg (f16 VGPR_32:$src)), 2033 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) 2034>; 2035 2036def : GCNPat < 2037 (fneg (fabs (f16 VGPR_32:$src))), 2038 (V_OR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) // Set sign bit 2039>; 2040 2041def : GCNPat < 2042 (fneg (v2f16 VGPR_32:$src)), 2043 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src) 2044>; 2045 2046def : GCNPat < 2047 (fabs (v2f16 VGPR_32:$src)), 2048 (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fff7fff)), VGPR_32:$src) 2049>; 2050 2051def : GCNPat < 2052 (fneg (v2f16 (fabs VGPR_32:$src))), 2053 (V_OR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src) 2054>; 2055 2056def : GCNPat < 2057 (fabs (f64 VReg_64:$src)), 2058 (REG_SEQUENCE VReg_64, 2059 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)), 2060 sub0, 2061 (V_AND_B32_e64 (i32 (S_MOV_B32 (i32 0x7fffffff))), 2062 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))), 2063 sub1) 2064>; 2065 2066def : GCNPat < 2067 (fneg (f64 VReg_64:$src)), 2068 (REG_SEQUENCE VReg_64, 2069 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)), 2070 sub0, 2071 (V_XOR_B32_e64 (i32 (S_MOV_B32 (i32 0x80000000))), 2072 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))), 2073 sub1) 2074>; 2075 2076def : GCNPat < 2077 (fneg (fabs (f64 VReg_64:$src))), 2078 (REG_SEQUENCE VReg_64, 2079 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)), 2080 sub0, 2081 (V_OR_B32_e64 (i32 (S_MOV_B32 (i32 0x80000000))), 2082 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))), 2083 sub1) 2084>; 2085 2086def : GCNPat < 2087 (DivergentUnaryFrag<fneg> (v2f32 VReg_64:$src)), 2088 (V_PK_ADD_F32 11 /* OP_SEL_1 | NEG_LO | HEG_HI */, VReg_64:$src, 2089 11 /* OP_SEL_1 | NEG_LO | HEG_HI */, 0, 2090 0, 0, 0, 0, 0) 2091> { 2092 let SubtargetPredicate = HasPackedFP32Ops; 2093} 2094 2095foreach fp16vt = [f16, bf16] in { 2096 2097def : GCNPat < 2098 (fcopysign fp16vt:$src0, fp16vt:$src1), 2099 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, $src1) 2100>; 2101 2102def : GCNPat < 2103 (fcopysign f32:$src0, fp16vt:$src1), 2104 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, 2105 (V_LSHLREV_B32_e64 (i32 16), $src1)) 2106>; 2107 2108def : GCNPat < 2109 (fcopysign f64:$src0, fp16vt:$src1), 2110 (REG_SEQUENCE SReg_64, 2111 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 2112 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)), 2113 (V_LSHLREV_B32_e64 (i32 16), $src1)), sub1) 2114>; 2115 2116def : GCNPat < 2117 (fcopysign fp16vt:$src0, f32:$src1), 2118 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, 2119 (V_LSHRREV_B32_e64 (i32 16), $src1)) 2120>; 2121 2122def : GCNPat < 2123 (fcopysign fp16vt:$src0, f64:$src1), 2124 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, 2125 (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1))) 2126>; 2127} // End foreach fp16vt = [f16, bf16] 2128 2129/********** ================== **********/ 2130/********** Immediate Patterns **********/ 2131/********** ================== **********/ 2132 2133def : GCNPat < 2134 (VGPRImm<(i32 imm)>:$imm), 2135 (V_MOV_B32_e32 imm:$imm) 2136>; 2137 2138def : GCNPat < 2139 (VGPRImm<(f32 fpimm)>:$imm), 2140 (V_MOV_B32_e32 (f32 (bitcast_fpimm_to_i32 $imm))) 2141>; 2142 2143def : GCNPat < 2144 (i32 imm:$imm), 2145 (S_MOV_B32 imm:$imm) 2146>; 2147 2148def : GCNPat < 2149 (VGPRImm<(SIlds tglobaladdr:$ga)>), 2150 (V_MOV_B32_e32 $ga) 2151>; 2152 2153def : GCNPat < 2154 (SIlds tglobaladdr:$ga), 2155 (S_MOV_B32 $ga) 2156>; 2157 2158// FIXME: Workaround for ordering issue with peephole optimizer where 2159// a register class copy interferes with immediate folding. Should 2160// use s_mov_b32, which can be shrunk to s_movk_i32 2161def : GCNPat < 2162 (VGPRImm<(f16 fpimm)>:$imm), 2163 (V_MOV_B32_e32 (f16 (bitcast_fpimm_to_i32 $imm))) 2164>; 2165 2166def : GCNPat < 2167 (VGPRImm<(bf16 fpimm)>:$imm), 2168 (V_MOV_B32_e32 (bf16 (bitcast_fpimm_to_i32 $imm))) 2169>; 2170 2171// V_MOV_B64_PSEUDO and S_MOV_B64_IMM_PSEUDO can be used with any 64-bit 2172// immediate and wil be expanded as needed, but we will only use these patterns 2173// for values which can be encoded. 2174def : GCNPat < 2175 (VGPRImm<(i64 imm)>:$imm), 2176 (V_MOV_B64_PSEUDO imm:$imm) 2177>; 2178 2179def : GCNPat < 2180 (VGPRImm<(f64 fpimm)>:$imm), 2181 (V_MOV_B64_PSEUDO (f64 (bitcast_fpimm_to_i64 $imm))) 2182>; 2183 2184def : GCNPat < 2185 (i64 imm:$imm), 2186 (S_MOV_B64_IMM_PSEUDO imm:$imm) 2187>; 2188 2189def : GCNPat < 2190 (f64 fpimm:$imm), 2191 (S_MOV_B64_IMM_PSEUDO (i64 (bitcast_fpimm_to_i64 fpimm:$imm))) 2192>; 2193 2194def : GCNPat < 2195 (f32 fpimm:$imm), 2196 (S_MOV_B32 (f32 (bitcast_fpimm_to_i32 $imm))) 2197>; 2198 2199def : GCNPat < 2200 (f16 fpimm:$imm), 2201 (S_MOV_B32 (i32 (bitcast_fpimm_to_i32 $imm))) 2202>; 2203 2204def : GCNPat < 2205 (bf16 fpimm:$imm), 2206 (S_MOV_B32 (i32 (bitcast_fpimm_to_i32 $imm))) 2207>; 2208 2209def : GCNPat < 2210 (p5 frameindex:$fi), 2211 (V_MOV_B32_e32 (p5 (frameindex_to_targetframeindex $fi))) 2212>; 2213 2214def : GCNPat < 2215 (p5 frameindex:$fi), 2216 (S_MOV_B32 (p5 (frameindex_to_targetframeindex $fi))) 2217>; 2218 2219def : GCNPat < 2220 (i64 InlineImm64:$imm), 2221 (S_MOV_B64 InlineImm64:$imm) 2222>; 2223 2224// XXX - Should this use a s_cmp to set SCC? 2225 2226// Set to sign-extended 64-bit value (true = -1, false = 0) 2227def : GCNPat < 2228 (i1 imm:$imm), 2229 (S_MOV_B64 (i64 (as_i64imm $imm))) 2230> { 2231 let WaveSizePredicate = isWave64; 2232} 2233 2234def : GCNPat < 2235 (i1 imm:$imm), 2236 (S_MOV_B32 (i32 (as_i32imm $imm))) 2237> { 2238 let WaveSizePredicate = isWave32; 2239} 2240 2241def : GCNPat < 2242 (f64 InlineImmFP64:$imm), 2243 (S_MOV_B64 (f64 (bitcast_fpimm_to_i64 InlineImmFP64:$imm))) 2244>; 2245 2246/********** ================== **********/ 2247/********** Intrinsic Patterns **********/ 2248/********** ================== **********/ 2249 2250def : GCNPat < 2251 (f32 (fpow (VOP3Mods f32:$src0, i32:$src0_mods), (VOP3Mods f32:$src1, i32:$src1_mods))), 2252 (V_EXP_F32_e64 SRCMODS.NONE, (V_MUL_LEGACY_F32_e64 $src1_mods, $src1, SRCMODS.NONE, (V_LOG_F32_e64 $src0_mods, $src0), 0, 0)) 2253>; 2254 2255def : GCNPat < 2256 (i32 (sext i1:$src0)), 2257 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2258 /*src1mod*/(i32 0), /*src1*/(i32 -1), i1:$src0) 2259>; 2260 2261class Ext32Pat <SDNode ext> : GCNPat < 2262 (i32 (ext i1:$src0)), 2263 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2264 /*src1mod*/(i32 0), /*src1*/(i32 1), i1:$src0) 2265>; 2266 2267def : Ext32Pat <zext>; 2268def : Ext32Pat <anyext>; 2269 2270// The multiplication scales from [0,1) to the unsigned integer range, 2271// rounding down a bit to avoid unwanted overflow. 2272def : GCNPat < 2273 (AMDGPUurecip i32:$src0), 2274 (V_CVT_U32_F32_e32 2275 (V_MUL_F32_e32 (i32 CONST.FP_4294966784), 2276 (V_RCP_IFLAG_F32_e32 (V_CVT_F32_U32_e32 $src0)))) 2277>; 2278 2279//===----------------------------------------------------------------------===// 2280// VOP3 Patterns 2281//===----------------------------------------------------------------------===// 2282 2283def : IMad24Pat<V_MAD_I32_I24_e64, 1>; 2284def : UMad24Pat<V_MAD_U32_U24_e64, 1>; 2285 2286// BFI patterns 2287 2288def BFIImm32 : PatFrag< 2289 (ops node:$x, node:$y, node:$z), 2290 (i32 (DivergentBinFrag<or> (and node:$y, node:$x), (and node:$z, imm))), 2291 [{ 2292 auto *X = dyn_cast<ConstantSDNode>(N->getOperand(0)->getOperand(1)); 2293 auto *NotX = dyn_cast<ConstantSDNode>(N->getOperand(1)->getOperand(1)); 2294 return X && NotX && 2295 ~(unsigned)X->getZExtValue() == (unsigned)NotX->getZExtValue(); 2296 }] 2297>; 2298 2299 2300// Definition from ISA doc: 2301// (y & x) | (z & ~x) 2302def : AMDGPUPatIgnoreCopies < 2303 (DivergentBinFrag<or> (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))), 2304 (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), 2305 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), 2306 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) 2307>; 2308 2309// (y & C) | (z & ~C) 2310def : AMDGPUPatIgnoreCopies < 2311 (BFIImm32 i32:$x, i32:$y, i32:$z), 2312 (V_BFI_B32_e64 VSrc_b32:$x, VSrc_b32:$y, VSrc_b32:$z) 2313>; 2314 2315// 64-bit version 2316def : AMDGPUPatIgnoreCopies < 2317 (DivergentBinFrag<or> (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))), 2318 (REG_SEQUENCE VReg_64, 2319 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), 2320 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), 2321 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, 2322 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), 2323 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), 2324 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) 2325>; 2326 2327// SHA-256 Ch function 2328// z ^ (x & (y ^ z)) 2329def : AMDGPUPatIgnoreCopies < 2330 (DivergentBinFrag<xor> i32:$z, (and i32:$x, (xor i32:$y, i32:$z))), 2331 (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), 2332 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), 2333 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) 2334>; 2335 2336// 64-bit version 2337def : AMDGPUPatIgnoreCopies < 2338 (DivergentBinFrag<xor> i64:$z, (and i64:$x, (xor i64:$y, i64:$z))), 2339 (REG_SEQUENCE VReg_64, 2340 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), 2341 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), 2342 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, 2343 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), 2344 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), 2345 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) 2346>; 2347 2348def : AMDGPUPat < 2349 (fcopysign f32:$src0, f32:$src1), 2350 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, $src1) 2351>; 2352 2353def : AMDGPUPat < 2354 (fcopysign f32:$src0, f64:$src1), 2355 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, 2356 (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1))) 2357>; 2358 2359def : AMDGPUPat < 2360 (fcopysign f64:$src0, f64:$src1), 2361 (REG_SEQUENCE SReg_64, 2362 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 2363 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), 2364 (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)), 2365 (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1))), sub1) 2366>; 2367 2368def : AMDGPUPat < 2369 (fcopysign f64:$src0, f32:$src1), 2370 (REG_SEQUENCE SReg_64, 2371 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 2372 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), 2373 (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)), 2374 $src1), sub1) 2375>; 2376 2377def : ROTRPattern <V_ALIGNBIT_B32_e64>; 2378 2379def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))), 2380 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)), 2381 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>; 2382 2383def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))), 2384 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)), 2385 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>; 2386 2387/********** ====================== **********/ 2388/********** Indirect addressing **********/ 2389/********** ====================== **********/ 2390 2391multiclass SI_INDIRECT_Pattern <ValueType vt, ValueType eltvt, string VecSize> { 2392 // Extract with offset 2393 def : GCNPat< 2394 (eltvt (extractelt vt:$src, (MOVRELOffset i32:$idx, (i32 imm:$offset)))), 2395 (!cast<Instruction>("SI_INDIRECT_SRC_"#VecSize) $src, $idx, imm:$offset) 2396 >; 2397 2398 // Insert with offset 2399 def : GCNPat< 2400 (insertelt vt:$src, eltvt:$val, (MOVRELOffset i32:$idx, (i32 imm:$offset))), 2401 (!cast<Instruction>("SI_INDIRECT_DST_"#VecSize) $src, $idx, imm:$offset, $val) 2402 >; 2403} 2404 2405defm : SI_INDIRECT_Pattern <v2f32, f32, "V2">; 2406defm : SI_INDIRECT_Pattern <v4f32, f32, "V4">; 2407defm : SI_INDIRECT_Pattern <v8f32, f32, "V8">; 2408defm : SI_INDIRECT_Pattern <v9f32, f32, "V9">; 2409defm : SI_INDIRECT_Pattern <v10f32, f32, "V10">; 2410defm : SI_INDIRECT_Pattern <v11f32, f32, "V11">; 2411defm : SI_INDIRECT_Pattern <v12f32, f32, "V12">; 2412defm : SI_INDIRECT_Pattern <v16f32, f32, "V16">; 2413defm : SI_INDIRECT_Pattern <v32f32, f32, "V32">; 2414 2415defm : SI_INDIRECT_Pattern <v2i32, i32, "V2">; 2416defm : SI_INDIRECT_Pattern <v4i32, i32, "V4">; 2417defm : SI_INDIRECT_Pattern <v8i32, i32, "V8">; 2418defm : SI_INDIRECT_Pattern <v9i32, i32, "V9">; 2419defm : SI_INDIRECT_Pattern <v10i32, i32, "V10">; 2420defm : SI_INDIRECT_Pattern <v11i32, i32, "V11">; 2421defm : SI_INDIRECT_Pattern <v12i32, i32, "V12">; 2422defm : SI_INDIRECT_Pattern <v16i32, i32, "V16">; 2423defm : SI_INDIRECT_Pattern <v32i32, i32, "V32">; 2424 2425//===----------------------------------------------------------------------===// 2426// SAD Patterns 2427//===----------------------------------------------------------------------===// 2428 2429def : GCNPat < 2430 (add (sub_oneuse (umax i32:$src0, i32:$src1), 2431 (umin i32:$src0, i32:$src1)), 2432 i32:$src2), 2433 (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0)) 2434>; 2435 2436def : GCNPat < 2437 (add (select_oneuse (i1 (setugt i32:$src0, i32:$src1)), 2438 (sub i32:$src0, i32:$src1), 2439 (sub i32:$src1, i32:$src0)), 2440 i32:$src2), 2441 (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0)) 2442>; 2443 2444//===----------------------------------------------------------------------===// 2445// Conversion Patterns 2446//===----------------------------------------------------------------------===// 2447def : GCNPat<(i32 (UniformSextInreg<i1> i32:$src)), 2448 (S_BFE_I32 i32:$src, (i32 65536))>; // 0 | 1 << 16 2449 2450// Handle sext_inreg in i64 2451def : GCNPat < 2452 (i64 (UniformSextInreg<i1> i64:$src)), 2453 (S_BFE_I64 i64:$src, (i32 0x10000)) // 0 | 1 << 16 2454>; 2455 2456def : GCNPat < 2457 (i16 (UniformSextInreg<i1> i16:$src)), 2458 (S_BFE_I32 $src, (i32 0x00010000)) // 0 | 1 << 16 2459>; 2460 2461def : GCNPat < 2462 (i16 (UniformSextInreg<i8> i16:$src)), 2463 (S_BFE_I32 $src, (i32 0x80000)) // 0 | 8 << 16 2464>; 2465 2466def : GCNPat < 2467 (i64 (UniformSextInreg<i8> i64:$src)), 2468 (S_BFE_I64 i64:$src, (i32 0x80000)) // 0 | 8 << 16 2469>; 2470 2471def : GCNPat < 2472 (i64 (UniformSextInreg<i16> i64:$src)), 2473 (S_BFE_I64 i64:$src, (i32 0x100000)) // 0 | 16 << 16 2474>; 2475 2476def : GCNPat < 2477 (i64 (UniformSextInreg<i32> i64:$src)), 2478 (S_BFE_I64 i64:$src, (i32 0x200000)) // 0 | 32 << 16 2479>; 2480 2481def : GCNPat< 2482 (i32 (DivergentSextInreg<i1> i32:$src)), 2483 (V_BFE_I32_e64 i32:$src, (i32 0), (i32 1))>; 2484 2485def : GCNPat < 2486 (i16 (DivergentSextInreg<i1> i16:$src)), 2487 (V_BFE_I32_e64 $src, (i32 0), (i32 1)) 2488>; 2489 2490def : GCNPat < 2491 (i16 (DivergentSextInreg<i8> i16:$src)), 2492 (V_BFE_I32_e64 $src, (i32 0), (i32 8)) 2493>; 2494 2495def : GCNPat< 2496 (i32 (DivergentSextInreg<i8> i32:$src)), 2497 (V_BFE_I32_e64 i32:$src, (i32 0), (i32 8)) 2498>; 2499 2500def : GCNPat < 2501 (i32 (DivergentSextInreg<i16> i32:$src)), 2502 (V_BFE_I32_e64 $src, (i32 0), (i32 16)) 2503>; 2504 2505def : GCNPat < 2506 (i64 (DivergentSextInreg<i1> i64:$src)), 2507 (REG_SEQUENCE VReg_64, 2508 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1)), sub0, 2509 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1))), sub1) 2510>; 2511 2512def : GCNPat < 2513 (i64 (DivergentSextInreg<i8> i64:$src)), 2514 (REG_SEQUENCE VReg_64, 2515 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8)), sub0, 2516 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8))), sub1) 2517>; 2518 2519def : GCNPat < 2520 (i64 (DivergentSextInreg<i16> i64:$src)), 2521 (REG_SEQUENCE VReg_64, 2522 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16)), sub0, 2523 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16))), sub1) 2524>; 2525 2526def : GCNPat < 2527 (i64 (DivergentSextInreg<i32> i64:$src)), 2528 (REG_SEQUENCE VReg_64, 2529 (i32 (EXTRACT_SUBREG i64:$src, sub0)), sub0, 2530 (V_ASHRREV_I32_e32 (i32 31), (i32 (EXTRACT_SUBREG i64:$src, sub0))), sub1) 2531>; 2532 2533def : GCNPat < 2534 (i64 (zext i32:$src)), 2535 (REG_SEQUENCE SReg_64, $src, sub0, (S_MOV_B32 (i32 0)), sub1) 2536>; 2537 2538def : GCNPat < 2539 (i64 (anyext i32:$src)), 2540 (REG_SEQUENCE SReg_64, $src, sub0, (i32 (IMPLICIT_DEF)), sub1) 2541>; 2542 2543class ZExt_i64_i1_Pat <SDNode ext> : GCNPat < 2544 (i64 (ext i1:$src)), 2545 (REG_SEQUENCE VReg_64, 2546 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2547 /*src1mod*/(i32 0), /*src1*/(i32 1), $src), 2548 sub0, (S_MOV_B32 (i32 0)), sub1) 2549>; 2550 2551 2552def : ZExt_i64_i1_Pat<zext>; 2553def : ZExt_i64_i1_Pat<anyext>; 2554 2555// FIXME: We need to use COPY_TO_REGCLASS to work-around the fact that 2556// REG_SEQUENCE patterns don't support instructions with multiple outputs. 2557def : GCNPat < 2558 (i64 (UniformUnaryFrag<sext> i32:$src)), 2559 (REG_SEQUENCE SReg_64, $src, sub0, 2560 (i32 (COPY_TO_REGCLASS (S_ASHR_I32 $src, (i32 31)), SReg_32_XM0)), sub1) 2561>; 2562 2563def : GCNPat < 2564 (i64 (DivergentUnaryFrag<sext> i32:$src)), 2565 (REG_SEQUENCE VReg_64, $src, sub0, 2566 (i32 (COPY_TO_REGCLASS (V_ASHRREV_I32_e64 (i32 31), $src), VGPR_32)), sub1) 2567>; 2568 2569def : GCNPat < 2570 (i64 (sext i1:$src)), 2571 (REG_SEQUENCE VReg_64, 2572 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2573 /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub0, 2574 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2575 /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub1) 2576>; 2577 2578class FPToI1Pat<Instruction Inst, int KOne, ValueType kone_type, ValueType vt, SDPatternOperator fp_to_int> : GCNPat < 2579 (i1 (fp_to_int (vt (VOP3Mods vt:$src0, i32:$src0_modifiers)))), 2580 (i1 (Inst 0, (kone_type KOne), $src0_modifiers, $src0, DSTCLAMP.NONE)) 2581>; 2582 2583let OtherPredicates = [NotHasTrue16BitInsts] in { 2584 def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>; 2585 def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>; 2586} // end OtherPredicates = [NotHasTrue16BitInsts] 2587 2588let OtherPredicates = [HasTrue16BitInsts] in { 2589 def : FPToI1Pat<V_CMP_EQ_F16_t16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>; 2590 def : FPToI1Pat<V_CMP_EQ_F16_t16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>; 2591} // end OtherPredicates = [HasTrue16BitInsts] 2592 2593def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_ONE, i32, f32, fp_to_uint>; 2594def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_NEG_ONE, i32, f32, fp_to_sint>; 2595def : FPToI1Pat<V_CMP_EQ_F64_e64, CONST.FP64_ONE, i64, f64, fp_to_uint>; 2596def : FPToI1Pat<V_CMP_EQ_F64_e64, CONST.FP64_NEG_ONE, i64, f64, fp_to_sint>; 2597 2598// If we need to perform a logical operation on i1 values, we need to 2599// use vector comparisons since there is only one SCC register. Vector 2600// comparisons may write to a pair of SGPRs or a single SGPR, so treat 2601// these as 32 or 64-bit comparisons. When legalizing SGPR copies, 2602// instructions resulting in the copies from SCC to these instructions 2603// will be moved to the VALU. 2604 2605let WaveSizePredicate = isWave64 in { 2606def : GCNPat < 2607 (i1 (and i1:$src0, i1:$src1)), 2608 (S_AND_B64 $src0, $src1) 2609>; 2610 2611def : GCNPat < 2612 (i1 (or i1:$src0, i1:$src1)), 2613 (S_OR_B64 $src0, $src1) 2614>; 2615 2616def : GCNPat < 2617 (i1 (xor i1:$src0, i1:$src1)), 2618 (S_XOR_B64 $src0, $src1) 2619>; 2620 2621def : GCNPat < 2622 (i1 (add i1:$src0, i1:$src1)), 2623 (S_XOR_B64 $src0, $src1) 2624>; 2625 2626def : GCNPat < 2627 (i1 (sub i1:$src0, i1:$src1)), 2628 (S_XOR_B64 $src0, $src1) 2629>; 2630 2631let AddedComplexity = 1 in { 2632def : GCNPat < 2633 (i1 (add i1:$src0, (i1 -1))), 2634 (S_NOT_B64 $src0) 2635>; 2636 2637def : GCNPat < 2638 (i1 (sub i1:$src0, (i1 -1))), 2639 (S_NOT_B64 $src0) 2640>; 2641} 2642} // end isWave64 2643 2644let WaveSizePredicate = isWave32 in { 2645def : GCNPat < 2646 (i1 (and i1:$src0, i1:$src1)), 2647 (S_AND_B32 $src0, $src1) 2648>; 2649 2650def : GCNPat < 2651 (i1 (or i1:$src0, i1:$src1)), 2652 (S_OR_B32 $src0, $src1) 2653>; 2654 2655def : GCNPat < 2656 (i1 (xor i1:$src0, i1:$src1)), 2657 (S_XOR_B32 $src0, $src1) 2658>; 2659 2660def : GCNPat < 2661 (i1 (add i1:$src0, i1:$src1)), 2662 (S_XOR_B32 $src0, $src1) 2663>; 2664 2665def : GCNPat < 2666 (i1 (sub i1:$src0, i1:$src1)), 2667 (S_XOR_B32 $src0, $src1) 2668>; 2669 2670let AddedComplexity = 1 in { 2671def : GCNPat < 2672 (i1 (add i1:$src0, (i1 -1))), 2673 (S_NOT_B32 $src0) 2674>; 2675 2676def : GCNPat < 2677 (i1 (sub i1:$src0, (i1 -1))), 2678 (S_NOT_B32 $src0) 2679>; 2680} 2681} // end isWave32 2682 2683def : GCNPat < 2684 (i32 (DivergentBinFrag<xor> i32:$src0, (i32 -1))), 2685 (V_NOT_B32_e32 $src0) 2686>; 2687 2688def : GCNPat < 2689 (i64 (DivergentBinFrag<xor> i64:$src0, (i64 -1))), 2690 (REG_SEQUENCE VReg_64, 2691 (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub0))), sub0, 2692 (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub1))), sub1 2693 ) 2694>; 2695 2696let SubtargetPredicate = NotHasTrue16BitInsts in 2697def : GCNPat < 2698 (f16 (sint_to_fp i1:$src)), 2699 (V_CVT_F16_F32_e32 ( 2700 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2701 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE), 2702 SSrc_i1:$src)) 2703>; 2704 2705let SubtargetPredicate = HasTrue16BitInsts in 2706def : GCNPat < 2707 (f16 (sint_to_fp i1:$src)), 2708 (V_CVT_F16_F32_t16_e32 ( 2709 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2710 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE), 2711 SSrc_i1:$src)) 2712>; 2713 2714let SubtargetPredicate = NotHasTrue16BitInsts in 2715def : GCNPat < 2716 (f16 (uint_to_fp i1:$src)), 2717 (V_CVT_F16_F32_e32 ( 2718 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2719 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE), 2720 SSrc_i1:$src)) 2721>; 2722let SubtargetPredicate = HasTrue16BitInsts in 2723def : GCNPat < 2724 (f16 (uint_to_fp i1:$src)), 2725 (V_CVT_F16_F32_t16_e32 ( 2726 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2727 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE), 2728 SSrc_i1:$src)) 2729>; 2730 2731def : GCNPat < 2732 (f32 (sint_to_fp i1:$src)), 2733 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2734 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE), 2735 SSrc_i1:$src) 2736>; 2737 2738def : GCNPat < 2739 (f32 (uint_to_fp i1:$src)), 2740 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2741 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE), 2742 SSrc_i1:$src) 2743>; 2744 2745def : GCNPat < 2746 (f64 (sint_to_fp i1:$src)), 2747 (V_CVT_F64_I32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2748 /*src1mod*/(i32 0), /*src1*/(i32 -1), 2749 SSrc_i1:$src)) 2750>; 2751 2752def : GCNPat < 2753 (f64 (uint_to_fp i1:$src)), 2754 (V_CVT_F64_U32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2755 /*src1mod*/(i32 0), /*src1*/(i32 1), 2756 SSrc_i1:$src)) 2757>; 2758 2759//===----------------------------------------------------------------------===// 2760// Miscellaneous Patterns 2761//===----------------------------------------------------------------------===// 2762 2763// Eliminate a zero extension from an fp16 operation if it already 2764// zeros the high bits of the 32-bit register. 2765// 2766// This is complicated on gfx9+. Some instructions maintain the legacy 2767// zeroing behavior, but others preserve the high bits. Some have a 2768// control bit to change the behavior. We can't simply say with 2769// certainty what the source behavior is without more context on how 2770// the src is lowered. e.g. fptrunc + fma may be lowered to a 2771// v_fma_mix* instruction which does not zero, or may not. 2772def : GCNPat< 2773 (i32 (DivergentUnaryFrag<abs> i32:$src)), 2774 (V_MAX_I32_e64 (V_SUB_CO_U32_e32 (i32 0), $src), $src)>; 2775 2776let AddedComplexity = 1 in { 2777def : GCNPat< 2778 (i32 (DivergentUnaryFrag<abs> i32:$src)), 2779 (V_MAX_I32_e64 (V_SUB_U32_e32 (i32 0), $src), $src)>{ 2780 let SubtargetPredicate = HasAddNoCarryInsts; 2781} 2782} // AddedComplexity = 1 2783 2784def : GCNPat< 2785 (i32 (DivergentUnaryFrag<zext> i16:$src)), 2786 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src) 2787>; 2788 2789def : GCNPat< 2790 (i64 (DivergentUnaryFrag<zext> i16:$src)), 2791 (REG_SEQUENCE VReg_64, 2792 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src), sub0, 2793 (S_MOV_B32 (i32 0)), sub1) 2794>; 2795 2796def : GCNPat< 2797 (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))), 2798 (COPY VSrc_b16:$src)>; 2799 2800def : GCNPat < 2801 (i32 (trunc i64:$a)), 2802 (EXTRACT_SUBREG $a, sub0) 2803>; 2804 2805def : GCNPat < 2806 (i1 (UniformUnaryFrag<trunc> i32:$a)), 2807 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), $a), (i32 1)) 2808>; 2809 2810def : GCNPat < 2811 (i1 (UniformUnaryFrag<trunc> i16:$a)), 2812 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), $a), (i32 1)) 2813>; 2814 2815def : GCNPat < 2816 (i1 (UniformUnaryFrag<trunc> i64:$a)), 2817 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), 2818 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1)) 2819>; 2820 2821def : GCNPat < 2822 (i1 (DivergentUnaryFrag<trunc> i32:$a)), 2823 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) 2824>; 2825 2826def : GCNPat < 2827 (i1 (DivergentUnaryFrag<trunc> i16:$a)), 2828 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) 2829>; 2830 2831def IMMBitSelConst : SDNodeXForm<imm, [{ 2832 return CurDAG->getTargetConstant(1ULL << N->getZExtValue(), SDLoc(N), 2833 MVT::i32); 2834}]>; 2835 2836// Matching separate SRL and TRUNC instructions 2837// with dependent operands (SRL dest is source of TRUNC) 2838// generates three instructions. However, by using bit shifts, 2839// the V_LSHRREV_B32_e64 result can be directly used in the 2840// operand of the V_AND_B32_e64 instruction: 2841// (trunc i32 (srl i32 $a, i32 $b)) -> 2842// v_and_b32_e64 $a, (1 << $b), $a 2843// v_cmp_ne_u32_e64 $a, 0, $a 2844 2845// Handle the VALU case. 2846def : GCNPat < 2847 (i1 (DivergentUnaryFrag<trunc> (i32 (srl i32:$a, (i32 imm:$b))))), 2848 (V_CMP_NE_U32_e64 (V_AND_B32_e64 (i32 (IMMBitSelConst $b)), $a), 2849 (i32 0)) 2850>; 2851 2852// Handle the scalar case. 2853def : GCNPat < 2854 (i1 (UniformUnaryFrag<trunc> (i32 (srl i32:$a, (i32 imm:$b))))), 2855 (S_CMP_LG_U32 (S_AND_B32 (i32 (IMMBitSelConst $b)), $a), 2856 (i32 0)) 2857>; 2858 2859def : GCNPat < 2860 (i1 (DivergentUnaryFrag<trunc> i64:$a)), 2861 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), 2862 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1)) 2863>; 2864 2865def : GCNPat < 2866 (i32 (bswap i32:$a)), 2867 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)), 2868 (V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 24)), 2869 (V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 8))) 2870>; 2871 2872// FIXME: This should have been narrowed to i32 during legalization. 2873// This pattern should also be skipped for GlobalISel 2874def : GCNPat < 2875 (i64 (bswap i64:$a)), 2876 (REG_SEQUENCE VReg_64, 2877 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)), 2878 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2879 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2880 (i32 24)), 2881 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2882 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2883 (i32 8))), 2884 sub0, 2885 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)), 2886 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2887 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2888 (i32 24)), 2889 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2890 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2891 (i32 8))), 2892 sub1) 2893>; 2894 2895// FIXME: The AddedComplexity should not be needed, but in GlobalISel 2896// the BFI pattern ends up taking precedence without it. 2897let SubtargetPredicate = isGFX8Plus, AddedComplexity = 1 in { 2898// Magic number: 3 | (2 << 8) | (1 << 16) | (0 << 24) 2899// 2900// My reading of the manual suggests we should be using src0 for the 2901// register value, but this is what seems to work. 2902def : GCNPat < 2903 (i32 (bswap i32:$a)), 2904 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x00010203))) 2905>; 2906 2907// FIXME: This should have been narrowed to i32 during legalization. 2908// This pattern should also be skipped for GlobalISel 2909def : GCNPat < 2910 (i64 (bswap i64:$a)), 2911 (REG_SEQUENCE VReg_64, 2912 (V_PERM_B32_e64 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub1), 2913 (S_MOV_B32 (i32 0x00010203))), 2914 sub0, 2915 (V_PERM_B32_e64 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub0), 2916 (S_MOV_B32 (i32 0x00010203))), 2917 sub1) 2918>; 2919 2920// Magic number: 1 | (0 << 8) | (12 << 16) | (12 << 24) 2921// The 12s emit 0s. 2922def : GCNPat < 2923 (i16 (bswap i16:$a)), 2924 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001))) 2925>; 2926 2927def : GCNPat < 2928 (i32 (zext (bswap i16:$a))), 2929 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001))) 2930>; 2931 2932// Magic number: 1 | (0 << 8) | (3 << 16) | (2 << 24) 2933def : GCNPat < 2934 (v2i16 (bswap v2i16:$a)), 2935 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x02030001))) 2936>; 2937 2938} 2939 2940def : GCNPat< 2941 (i64 (DivergentUnaryFrag<bitreverse> i64:$a)), 2942 (REG_SEQUENCE VReg_64, 2943 (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1))), sub0, 2944 (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0))), sub1)>; 2945 2946// Prefer selecting to max when legal, but using mul is always valid. 2947let AddedComplexity = -5 in { 2948 2949let OtherPredicates = [NotHasTrue16BitInsts] in { 2950def : GCNPat< 2951 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 2952 (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src) 2953>; 2954 2955def : GCNPat< 2956 (fcanonicalize (f16 (fneg (VOP3Mods f16:$src, i32:$src_mods)))), 2957 (V_MUL_F16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src) 2958>; 2959} // End OtherPredicates 2960 2961let OtherPredicates = [HasTrue16BitInsts] in { 2962def : GCNPat< 2963 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 2964 (V_MUL_F16_fake16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src) 2965>; 2966 2967def : GCNPat< 2968 (fcanonicalize (f16 (fneg (VOP3Mods f16:$src, i32:$src_mods)))), 2969 (V_MUL_F16_fake16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src) 2970>; 2971} // End OtherPredicates 2972 2973def : GCNPat< 2974 (fcanonicalize (v2f16 (VOP3PMods v2f16:$src, i32:$src_mods))), 2975 (V_PK_MUL_F16 0, (i32 CONST.FP16_ONE), $src_mods, $src, DSTCLAMP.NONE) 2976>; 2977 2978def : GCNPat< 2979 (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))), 2980 (V_MUL_F32_e64 0, (i32 CONST.FP32_ONE), $src_mods, $src) 2981>; 2982 2983def : GCNPat< 2984 (fcanonicalize (f32 (fneg (VOP3Mods f32:$src, i32:$src_mods)))), 2985 (V_MUL_F32_e64 0, (i32 CONST.FP32_NEG_ONE), $src_mods, $src) 2986>; 2987 2988let SubtargetPredicate = HasPackedFP32Ops in { 2989def : GCNPat< 2990 (fcanonicalize (v2f32 (VOP3PMods v2f32:$src, i32:$src_mods))), 2991 (V_PK_MUL_F32 0, CONST.FP32_ONE, $src_mods, $src) 2992>; 2993} 2994 2995// TODO: Handle fneg like other types. 2996let SubtargetPredicate = isNotGFX12Plus in { 2997def : GCNPat< 2998 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))), 2999 (V_MUL_F64_e64 0, CONST.FP64_ONE, $src_mods, $src) 3000>; 3001} 3002} // End AddedComplexity = -5 3003 3004multiclass SelectCanonicalizeAsMax< 3005 list<Predicate> f32_preds = [], 3006 list<Predicate> f64_preds = [], 3007 list<Predicate> f16_preds = []> { 3008 def : GCNPat< 3009 (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))), 3010 (V_MAX_F32_e64 $src_mods, $src, $src_mods, $src)> { 3011 let OtherPredicates = f32_preds; 3012 } 3013 3014 def : GCNPat< 3015 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))), 3016 (V_MAX_F64_e64 $src_mods, $src, $src_mods, $src)> { 3017 let OtherPredicates = !listconcat(f64_preds, [isNotGFX12Plus]); 3018 } 3019 3020 def : GCNPat< 3021 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))), 3022 (V_MAX_NUM_F64_e64 $src_mods, $src, $src_mods, $src)> { 3023 let OtherPredicates = !listconcat(f64_preds, [isGFX12Plus]); 3024 } 3025 3026 def : GCNPat< 3027 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 3028 (V_MAX_F16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> { 3029 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts, NotHasTrue16BitInsts]); 3030 } 3031 3032 def : GCNPat< 3033 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 3034 (V_MAX_F16_fake16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> { 3035 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts, HasTrue16BitInsts]); 3036 } 3037 3038 def : GCNPat< 3039 (fcanonicalize (v2f16 (VOP3PMods v2f16:$src, i32:$src_mods))), 3040 (V_PK_MAX_F16 $src_mods, $src, $src_mods, $src, DSTCLAMP.NONE)> { 3041 // FIXME: Should have VOP3P subtarget predicate 3042 let OtherPredicates = f16_preds; 3043 } 3044} 3045 3046// On pre-gfx9 targets, v_max_*/v_min_* did not respect the denormal 3047// mode, and would never flush. For f64, it's faster to do implement 3048// this with a max. For f16/f32 it's a wash, but prefer max when 3049// valid. 3050// 3051// FIXME: Lowering f32/f16 with max is worse since we can use a 3052// smaller encoding if the input is fneg'd. It also adds an extra 3053// register use. 3054let SubtargetPredicate = HasMinMaxDenormModes in { 3055 defm : SelectCanonicalizeAsMax<[], [], []>; 3056} // End SubtargetPredicate = HasMinMaxDenormModes 3057 3058let SubtargetPredicate = NotHasMinMaxDenormModes in { 3059 // Use the max lowering if we don't need to flush. 3060 3061 // FIXME: We don't do use this for f32 as a workaround for the 3062 // library being compiled with the default ieee mode, but 3063 // potentially being called from flushing kernels. Really we should 3064 // not be mixing code expecting different default FP modes, but mul 3065 // works in any FP environment. 3066 defm : SelectCanonicalizeAsMax<[FalsePredicate], [FP64Denormals], [FP16Denormals]>; 3067} // End SubtargetPredicate = NotHasMinMaxDenormModes 3068 3069 3070let OtherPredicates = [HasDLInsts] in { 3071// Don't allow source modifiers. If there are any source modifiers then it's 3072// better to select fma instead of fmac. 3073def : GCNPat < 3074 (fma (f32 (VOP3NoMods f32:$src0)), 3075 (f32 (VOP3NoMods f32:$src1)), 3076 (f32 (VOP3NoMods f32:$src2))), 3077 (V_FMAC_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 3078 SRCMODS.NONE, $src2) 3079>; 3080} // End OtherPredicates = [HasDLInsts] 3081 3082let SubtargetPredicate = isGFX10Plus in { 3083// Don't allow source modifiers. If there are any source modifiers then it's 3084// better to select fma instead of fmac. 3085let OtherPredicates = [NotHasTrue16BitInsts] in 3086def : GCNPat < 3087 (fma (f16 (VOP3NoMods f32:$src0)), 3088 (f16 (VOP3NoMods f32:$src1)), 3089 (f16 (VOP3NoMods f32:$src2))), 3090 (V_FMAC_F16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 3091 SRCMODS.NONE, $src2) 3092>; 3093let OtherPredicates = [HasTrue16BitInsts] in 3094def : GCNPat < 3095 (fma (f16 (VOP3NoMods f32:$src0)), 3096 (f16 (VOP3NoMods f32:$src1)), 3097 (f16 (VOP3NoMods f32:$src2))), 3098 (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 3099 SRCMODS.NONE, $src2) 3100>; 3101} 3102 3103let OtherPredicates = [HasFmacF64Inst] in 3104// Don't allow source modifiers. If there are any source modifiers then it's 3105// better to select fma instead of fmac. 3106def : GCNPat < 3107 (fma (f64 (VOP3NoMods f64:$src0)), 3108 (f64 (VOP3NoMods f64:$src1)), 3109 (f64 (VOP3NoMods f64:$src2))), 3110 (V_FMAC_F64_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 3111 SRCMODS.NONE, $src2) 3112>; 3113 3114// COPY is workaround tablegen bug from multiple outputs 3115// from S_LSHL_B32's multiple outputs from implicit scc def. 3116let AddedComplexity = 1 in { 3117def : GCNPat < 3118 (v2i16 (UniformBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))), 3119 (S_LSHL_B32 SReg_32:$src1, (i16 16)) 3120>; 3121 3122def : GCNPat < 3123 (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 VGPR_32:$src1))), 3124 (v2i16 (V_LSHLREV_B32_e64 (i16 16), VGPR_32:$src1)) 3125>; 3126 3127 3128def : GCNPat < 3129 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))), 3130 (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1) 3131>; 3132 3133def : GCNPat < 3134 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src1), (i16 0))), 3135 (v2i16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1)) 3136>; 3137 3138def : GCNPat < 3139 (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))), 3140 (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1) 3141>; 3142 3143def : GCNPat < 3144 (v2f16 (DivergentBinFrag<build_vector> (f16 VGPR_32:$src1), (f16 FP_ZERO))), 3145 (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1)) 3146>; 3147 3148def : GCNPat < 3149 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 undef))), 3150 (COPY_TO_REGCLASS SReg_32:$src0, SReg_32) 3151>; 3152 3153def : GCNPat < 3154 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 undef))), 3155 (COPY_TO_REGCLASS VGPR_32:$src0, VGPR_32) 3156>; 3157 3158def : GCNPat < 3159 (v2f16 (build_vector f16:$src0, (f16 undef))), 3160 (COPY $src0) 3161>; 3162 3163def : GCNPat < 3164 (v2i16 (UniformBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))), 3165 (S_LSHL_B32 SReg_32:$src1, (i32 16)) 3166>; 3167 3168def : GCNPat < 3169 (v2i16 (DivergentBinFrag<build_vector> (i16 undef), (i16 VGPR_32:$src1))), 3170 (v2i16 (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1)) 3171>; 3172 3173 3174def : GCNPat < 3175 (v2f16 (UniformBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))), 3176 (S_LSHL_B32 SReg_32:$src1, (i32 16)) 3177>; 3178 3179def : GCNPat < 3180 (v2f16 (DivergentBinFrag<build_vector> (f16 undef), (f16 VGPR_32:$src1))), 3181 (v2f16 (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1)) 3182>; 3183} 3184 3185let SubtargetPredicate = HasVOP3PInsts in { 3186def : GCNPat < 3187 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))), 3188 (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) 3189>; 3190 3191def : GCNPat < 3192 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 VGPR_32:$src1))), 3193 (v2i16 (V_LSHL_OR_B32_e64 $src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), $src0)))) 3194>; 3195 3196// With multiple uses of the shift, this will duplicate the shift and 3197// increase register pressure. 3198def : GCNPat < 3199 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), 3200 (v2i16 (S_PACK_LH_B32_B16 SReg_32:$src0, SReg_32:$src1)) 3201>; 3202 3203def : GCNPat < 3204 (v2i16 (UniformBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), 3205 (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), 3206 (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1) 3207>; 3208 3209def : GCNPat < 3210 (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))), 3211 (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) 3212>; 3213 3214 3215 3216foreach Ty = [i16, f16] in { 3217 3218defvar vecTy = !if(!eq(Ty, i16), v2i16, v2f16); 3219defvar immzeroTy = !if(!eq(Ty, i16), immzero, fpimmzero); 3220 3221// Take the lower 16 bits from each VGPR_32 and concat them 3222def : GCNPat < 3223 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), (Ty VGPR_32:$b))), 3224 (V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x05040100))) 3225>; 3226 3227 3228// Take the lower 16 bits from V[0] and the upper 16 bits from V[1] 3229// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000) 3230def : GCNPat < 3231 (vecTy (DivergentBinFrag<build_vector> (Ty (immzeroTy)), 3232 (Ty !if(!eq(Ty, i16), 3233 (Ty (trunc (srl VGPR_32:$b, (i32 16)))), 3234 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))), 3235 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff0000)), VGPR_32:$b) 3236>; 3237 3238 3239// Take the lower 16 bits from V[0] and the upper 16 bits from V[1] 3240// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000) 3241def : GCNPat < 3242 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), 3243 (Ty !if(!eq(Ty, i16), 3244 (Ty (trunc (srl VGPR_32:$b, (i32 16)))), 3245 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))), 3246 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x0000ffff)), VGPR_32:$a, VGPR_32:$b) 3247>; 3248 3249 3250// Take the upper 16 bits from V[0] and the lower 16 bits from V[1] 3251// Special case, can use V_ALIGNBIT (always uses encoded literal) 3252def : GCNPat < 3253 (vecTy (DivergentBinFrag<build_vector> 3254 (Ty !if(!eq(Ty, i16), 3255 (Ty (trunc (srl VGPR_32:$a, (i32 16)))), 3256 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))), 3257 (Ty VGPR_32:$b))), 3258 (V_ALIGNBIT_B32_e64 VGPR_32:$b, VGPR_32:$a, (i32 16)) 3259>; 3260 3261// Take the upper 16 bits from each VGPR_32 and concat them 3262def : GCNPat < 3263 (vecTy (DivergentBinFrag<build_vector> 3264 (Ty !if(!eq(Ty, i16), 3265 (Ty (trunc (srl VGPR_32:$a, (i32 16)))), 3266 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))), 3267 (Ty !if(!eq(Ty, i16), 3268 (Ty (trunc (srl VGPR_32:$b, (i32 16)))), 3269 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))), 3270 (V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x07060302))) 3271>; 3272 3273 3274} // end foreach Ty 3275 3276 3277let AddedComplexity = 5 in { 3278def : GCNPat < 3279 (v2f16 (is_canonicalized<build_vector> (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)), 3280 (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))), 3281 (V_PACK_B32_F16_e64 $src0_mods, VGPR_32:$src0, $src1_mods, VGPR_32:$src1) 3282>; 3283} 3284} // End SubtargetPredicate = HasVOP3PInsts 3285 3286// With multiple uses of the shift, this will duplicate the shift and 3287// increase register pressure. 3288let SubtargetPredicate = isGFX11Plus in 3289def : GCNPat < 3290 (v2i16 (build_vector (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), (i16 SReg_32:$src1))), 3291 (v2i16 (S_PACK_HL_B32_B16 SReg_32:$src0, SReg_32:$src1)) 3292>; 3293 3294 3295def : GCNPat < 3296 (v2f16 (scalar_to_vector f16:$src0)), 3297 (COPY $src0) 3298>; 3299 3300def : GCNPat < 3301 (v2i16 (scalar_to_vector i16:$src0)), 3302 (COPY $src0) 3303>; 3304 3305def : GCNPat < 3306 (v4i16 (scalar_to_vector i16:$src0)), 3307 (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0) 3308>; 3309 3310def : GCNPat < 3311 (v4f16 (scalar_to_vector f16:$src0)), 3312 (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0) 3313>; 3314 3315def : GCNPat < 3316 (i64 (int_amdgcn_mov_dpp i64:$src, timm:$dpp_ctrl, timm:$row_mask, 3317 timm:$bank_mask, timm:$bound_ctrl)), 3318 (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$src, VReg_64_Align2:$src, 3319 (as_i32timm $dpp_ctrl), (as_i32timm $row_mask), 3320 (as_i32timm $bank_mask), 3321 (as_i1timm $bound_ctrl)) 3322>; 3323 3324def : GCNPat < 3325 (i64 (int_amdgcn_update_dpp i64:$old, i64:$src, timm:$dpp_ctrl, timm:$row_mask, 3326 timm:$bank_mask, timm:$bound_ctrl)), 3327 (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$old, VReg_64_Align2:$src, (as_i32timm $dpp_ctrl), 3328 (as_i32timm $row_mask), (as_i32timm $bank_mask), 3329 (as_i1timm $bound_ctrl)) 3330>; 3331 3332//===----------------------------------------------------------------------===// 3333// Fract Patterns 3334//===----------------------------------------------------------------------===// 3335 3336let SubtargetPredicate = isGFX6 in { 3337 3338// V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x)) is 3339// used instead. However, SI doesn't have V_FLOOR_F64, so the most efficient 3340// way to implement it is using V_FRACT_F64. 3341// The workaround for the V_FRACT bug is: 3342// fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999) 3343 3344// Convert floor(x) to (x - fract(x)) 3345 3346// Don't bother handling this for GlobalISel, it's handled during 3347// lowering. 3348// 3349// FIXME: DAG should also custom lower this. 3350def : GCNPat < 3351 (f64 (ffloor (f64 (VOP3Mods f64:$x, i32:$mods)))), 3352 (V_ADD_F64_e64 3353 $mods, 3354 $x, 3355 SRCMODS.NEG, 3356 (V_CNDMASK_B64_PSEUDO 3357 (V_MIN_F64_e64 3358 SRCMODS.NONE, 3359 (V_FRACT_F64_e64 $mods, $x), 3360 SRCMODS.NONE, 3361 (V_MOV_B64_PSEUDO 0x3fefffffffffffff)), 3362 $x, 3363 (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, (i32 3 /*NaN*/)))) 3364>; 3365 3366} // End SubtargetPredicates = isGFX6 3367 3368//============================================================================// 3369// Miscellaneous Optimization Patterns 3370//============================================================================// 3371 3372// Undo sub x, c -> add x, -c canonicalization since c is more likely 3373// an inline immediate than -c. 3374// TODO: Also do for 64-bit. 3375def : GCNPat< 3376 (UniformBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)), 3377 (S_SUB_I32 SReg_32:$src0, NegSubInlineConst32:$src1) 3378>; 3379 3380def : GCNPat< 3381 (DivergentBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)), 3382 (V_SUB_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> { 3383 let SubtargetPredicate = HasAddNoCarryInsts; 3384} 3385 3386def : GCNPat< 3387 (DivergentBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)), 3388 (V_SUB_CO_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> { 3389 let SubtargetPredicate = NotHasAddNoCarryInsts; 3390} 3391 3392 3393// Avoid pointlessly materializing a constant in VGPR. 3394// FIXME: Should also do this for readlane, but tablegen crashes on 3395// the ignored src1. 3396def : GCNPat< 3397 (int_amdgcn_readfirstlane (i32 imm:$src)), 3398 (S_MOV_B32 SReg_32:$src) 3399>; 3400 3401multiclass BFMPatterns <ValueType vt, PatFrag SHL, PatFrag ADD, InstSI BFM> { 3402 def : GCNPat < 3403 (vt (SHL (vt (add (vt (shl 1, vt:$a)), -1)), vt:$b)), 3404 (BFM $a, $b) 3405 >; 3406 3407 def : GCNPat < 3408 (vt (ADD (vt (shl 1, vt:$a)), -1)), 3409 (BFM $a, (i32 0)) 3410 >; 3411} 3412 3413defm : BFMPatterns <i32, UniformBinFrag<shl>, UniformBinFrag<add>, S_BFM_B32>; 3414// FIXME: defm : BFMPatterns <i64, UniformBinFrag<shl>, UniformBinFrag<add>, S_BFM_B64>; 3415defm : BFMPatterns <i32, DivergentBinFrag<shl>, DivergentBinFrag<add>, V_BFM_B32_e64>; 3416 3417// Bitfield extract patterns 3418 3419def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{ 3420 return isMask_32(Imm); 3421}]>; 3422 3423def IMMPopCount : SDNodeXForm<imm, [{ 3424 return CurDAG->getTargetConstant(llvm::popcount(N->getZExtValue()), SDLoc(N), 3425 MVT::i32); 3426}]>; 3427 3428def : AMDGPUPat < 3429 (DivergentBinFrag<and> (i32 (srl i32:$src, i32:$rshift)), 3430 IMMZeroBasedBitfieldMask:$mask), 3431 (V_BFE_U32_e64 $src, $rshift, (i32 (IMMPopCount $mask))) 3432>; 3433 3434// x & ((1 << y) - 1) 3435def : AMDGPUPat < 3436 (DivergentBinFrag<and> i32:$src, (add_oneuse (shl_oneuse 1, i32:$width), -1)), 3437 (V_BFE_U32_e64 $src, (i32 0), $width) 3438>; 3439 3440// x & ~(-1 << y) 3441def : AMDGPUPat < 3442 (DivergentBinFrag<and> i32:$src, 3443 (xor_oneuse (shl_oneuse -1, i32:$width), -1)), 3444 (V_BFE_U32_e64 $src, (i32 0), $width) 3445>; 3446 3447// x & (-1 >> (bitwidth - y)) 3448def : AMDGPUPat < 3449 (DivergentBinFrag<and> i32:$src, (srl_oneuse -1, (sub 32, i32:$width))), 3450 (V_BFE_U32_e64 $src, (i32 0), $width) 3451>; 3452 3453// x << (bitwidth - y) >> (bitwidth - y) 3454def : AMDGPUPat < 3455 (DivergentBinFrag<srl> (shl_oneuse i32:$src, (sub 32, i32:$width)), 3456 (sub 32, i32:$width)), 3457 (V_BFE_U32_e64 $src, (i32 0), $width) 3458>; 3459 3460def : AMDGPUPat < 3461 (DivergentBinFrag<sra> (shl_oneuse i32:$src, (sub 32, i32:$width)), 3462 (sub 32, i32:$width)), 3463 (V_BFE_I32_e64 $src, (i32 0), $width) 3464>; 3465 3466// SHA-256 Ma patterns 3467 3468// ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y 3469def : AMDGPUPatIgnoreCopies < 3470 (DivergentBinFrag<or> (and i32:$x, i32:$z), 3471 (and i32:$y, (or i32:$x, i32:$z))), 3472 (V_BFI_B32_e64 (V_XOR_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), 3473 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)), 3474 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32), 3475 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)) 3476>; 3477 3478def : AMDGPUPatIgnoreCopies < 3479 (DivergentBinFrag<or> (and i64:$x, i64:$z), 3480 (and i64:$y, (or i64:$x, i64:$z))), 3481 (REG_SEQUENCE VReg_64, 3482 (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), 3483 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), 3484 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0)), 3485 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), sub0, 3486 (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), 3487 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), 3488 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1)), 3489 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), sub1) 3490>; 3491 3492multiclass IntMed3Pat<Instruction med3Inst, 3493 SDPatternOperator min, 3494 SDPatternOperator max> { 3495 3496 // This matches 16 permutations of 3497 // min(max(a, b), max(min(a, b), c)) 3498 def : AMDGPUPat < 3499 (min (max i32:$src0, i32:$src1), 3500 (max (min i32:$src0, i32:$src1), i32:$src2)), 3501 (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) 3502>; 3503 3504 // This matches 16 permutations of 3505 // max(min(x, y), min(max(x, y), z)) 3506 def : AMDGPUPat < 3507 (max (min i32:$src0, i32:$src1), 3508 (min (max i32:$src0, i32:$src1), i32:$src2)), 3509 (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) 3510>; 3511} 3512 3513defm : IntMed3Pat<V_MED3_I32_e64, smin, smax>; 3514defm : IntMed3Pat<V_MED3_U32_e64, umin, umax>; 3515 3516multiclass FPMed3Pat<ValueType vt, 3517 Instruction med3Inst> { 3518 // This matches 16 permutations of max(min(x, y), min(max(x, y), z)) 3519 def : GCNPat< 3520 (fmaxnum_like_nnan 3521 (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3522 (VOP3Mods vt:$src1, i32:$src1_mods)), 3523 (fminnum_like (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3524 (VOP3Mods vt:$src1, i32:$src1_mods)), 3525 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))), 3526 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, 3527 DSTCLAMP.NONE, DSTOMOD.NONE)>; 3528 3529 3530 // This matches 16 permutations of min(max(x, y), max(min(x, y), z)) 3531 def : GCNPat< 3532 (fminnum_like_nnan 3533 (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3534 (VOP3Mods vt:$src1, i32:$src1_mods)), 3535 (fmaxnum_like (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3536 (VOP3Mods vt:$src1, i32:$src1_mods)), 3537 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))), 3538 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, 3539 DSTCLAMP.NONE, DSTOMOD.NONE)>; 3540} 3541 3542class FP16Med3Pat<ValueType vt, 3543 Instruction med3Inst> : GCNPat< 3544 (fmaxnum_like_nnan (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3545 (VOP3Mods vt:$src1, i32:$src1_mods)), 3546 (fminnum_like (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3547 (VOP3Mods vt:$src1, i32:$src1_mods)), 3548 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))), 3549 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, DSTCLAMP.NONE) 3550>; 3551 3552multiclass Int16Med3Pat<Instruction med3Inst, 3553 SDPatternOperator min, 3554 SDPatternOperator max> { 3555 // This matches 16 permutations of 3556 // max(min(x, y), min(max(x, y), z)) 3557 def : GCNPat < 3558 (max (min i16:$src0, i16:$src1), 3559 (min (max i16:$src0, i16:$src1), i16:$src2)), 3560 (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE) 3561>; 3562 3563 // This matches 16 permutations of 3564 // min(max(a, b), max(min(a, b), c)) 3565 def : GCNPat < 3566 (min (max i16:$src0, i16:$src1), 3567 (max (min i16:$src0, i16:$src1), i16:$src2)), 3568 (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE) 3569>; 3570} 3571 3572defm : FPMed3Pat<f32, V_MED3_F32_e64>; 3573 3574class 3575IntMinMaxPat<Instruction minmaxInst, SDPatternOperator min_or_max, 3576 SDPatternOperator max_or_min_oneuse> : AMDGPUPat < 3577 (DivergentBinFrag<min_or_max> (max_or_min_oneuse i32:$src0, i32:$src1), 3578 i32:$src2), 3579 (minmaxInst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) 3580>; 3581 3582class 3583FPMinMaxPat<Instruction minmaxInst, ValueType vt, SDPatternOperator min_or_max, 3584 SDPatternOperator max_or_min_oneuse> : GCNPat < 3585 (min_or_max (max_or_min_oneuse (VOP3Mods vt:$src0, i32:$src0_mods), 3586 (VOP3Mods vt:$src1, i32:$src1_mods)), 3587 (vt (VOP3Mods vt:$src2, i32:$src2_mods))), 3588 (minmaxInst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, 3589 DSTCLAMP.NONE, DSTOMOD.NONE) 3590>; 3591 3592let OtherPredicates = [isGFX11Plus] in { 3593def : IntMinMaxPat<V_MAXMIN_I32_e64, smin, smax_oneuse>; 3594def : IntMinMaxPat<V_MINMAX_I32_e64, smax, smin_oneuse>; 3595def : IntMinMaxPat<V_MAXMIN_U32_e64, umin, umax_oneuse>; 3596def : IntMinMaxPat<V_MINMAX_U32_e64, umax, umin_oneuse>; 3597def : FPMinMaxPat<V_MINMAX_F32_e64, f32, fmaxnum_like, fminnum_like_oneuse>; 3598def : FPMinMaxPat<V_MAXMIN_F32_e64, f32, fminnum_like, fmaxnum_like_oneuse>; 3599def : FPMinMaxPat<V_MINMAX_F16_e64, f16, fmaxnum_like, fminnum_like_oneuse>; 3600def : FPMinMaxPat<V_MAXMIN_F16_e64, f16, fminnum_like, fmaxnum_like_oneuse>; 3601} 3602 3603let OtherPredicates = [isGFX9Plus] in { 3604def : FP16Med3Pat<f16, V_MED3_F16_e64>; 3605defm : Int16Med3Pat<V_MED3_I16_e64, smin, smax>; 3606defm : Int16Med3Pat<V_MED3_U16_e64, umin, umax>; 3607} // End Predicates = [isGFX9Plus] 3608 3609let OtherPredicates = [isGFX12Plus] in { 3610def : FPMinMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>; 3611def : FPMinMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>; 3612def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>; 3613def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>; 3614} 3615 3616// Convert a floating-point power of 2 to the integer exponent. 3617def FPPow2ToExponentXForm : SDNodeXForm<fpimm, [{ 3618 const auto &APF = N->getValueAPF(); 3619 int Log2 = APF.getExactLog2Abs(); 3620 assert(Log2 != INT_MIN); 3621 return CurDAG->getTargetConstant(Log2, SDLoc(N), MVT::i32); 3622}]>; 3623 3624// Check if a floating point value is a power of 2 floating-point 3625// immediate where it's preferable to emit a multiply by as an 3626// ldexp. We skip over 0.5 to 4.0 as those are inline immediates 3627// anyway. 3628def fpimm_pos_pow2_prefer_ldexp_f64 : FPImmLeaf<f64, [{ 3629 if (Imm.isNegative()) 3630 return false; 3631 3632 int Exp = Imm.getExactLog2Abs(); 3633 // Prefer leaving the FP inline immediates as they are. 3634 // 0.5, 1.0, 2.0, 4.0 3635 3636 // For f64 ldexp is always better than materializing a 64-bit 3637 // constant. 3638 return Exp != INT_MIN && (Exp < -1 || Exp > 2); 3639 }], FPPow2ToExponentXForm 3640>; 3641 3642def fpimm_neg_pow2_prefer_ldexp_f64 : FPImmLeaf<f64, [{ 3643 if (!Imm.isNegative()) 3644 return false; 3645 int Exp = Imm.getExactLog2Abs(); 3646 // Prefer leaving the FP inline immediates as they are. 3647 // 0.5, 1.0, 2.0, 4.0 3648 3649 // For f64 ldexp is always better than materializing a 64-bit 3650 // constant. 3651 return Exp != INT_MIN && (Exp < -1 || Exp > 2); 3652 }], FPPow2ToExponentXForm 3653>; 3654 3655// f64 is different because we also want to handle cases that may 3656// require materialization of the exponent. 3657// TODO: If we know f64 ops are fast, prefer add (ldexp x, N), y over fma 3658// TODO: For f32/f16, it's not a clear win on code size to use ldexp 3659// in place of mul since we have to use the vop3 form. Are there power 3660// savings or some other reason to prefer ldexp over mul? 3661def : GCNPat< 3662 (any_fmul (f64 (VOP3Mods f64:$src0, i32:$src0_mods)), 3663 fpimm_pos_pow2_prefer_ldexp_f64:$src1), 3664 (V_LDEXP_F64_e64 i32:$src0_mods, VSrc_b64:$src0, 3665 0, (S_MOV_B32 (i32 (FPPow2ToExponentXForm $src1)))) 3666>; 3667 3668def : GCNPat< 3669 (any_fmul f64:$src0, fpimm_neg_pow2_prefer_ldexp_f64:$src1), 3670 (V_LDEXP_F64_e64 SRCMODS.NEG, VSrc_b64:$src0, 3671 0, (S_MOV_B32 (i32 (FPPow2ToExponentXForm $src1)))) 3672>; 3673 3674// We want to avoid using VOP3Mods which could pull in another fneg 3675// which we would need to be re-negated (which should never happen in 3676// practice). I don't see a way to apply an SDNodeXForm that accounts 3677// for a second operand. 3678def : GCNPat< 3679 (any_fmul (fabs f64:$src0), fpimm_neg_pow2_prefer_ldexp_f64:$src1), 3680 (V_LDEXP_F64_e64 SRCMODS.NEG_ABS, VSrc_b64:$src0, 3681 0, (S_MOV_B32 (i32 (FPPow2ToExponentXForm $src1)))) 3682>; 3683 3684class AMDGPUGenericInstruction : GenericInstruction { 3685 let Namespace = "AMDGPU"; 3686} 3687 3688// Convert a wave address to a swizzled vector address (i.e. this is 3689// for copying the stack pointer to a vector address appropriate to 3690// use in the offset field of mubuf instructions). 3691def G_AMDGPU_WAVE_ADDRESS : AMDGPUGenericInstruction { 3692 let OutOperandList = (outs type0:$dst); 3693 let InOperandList = (ins type0:$src); 3694 let hasSideEffects = 0; 3695} 3696 3697// Returns -1 if the input is zero. 3698def G_AMDGPU_FFBH_U32 : AMDGPUGenericInstruction { 3699 let OutOperandList = (outs type0:$dst); 3700 let InOperandList = (ins type1:$src); 3701 let hasSideEffects = 0; 3702} 3703 3704// Returns -1 if the input is zero. 3705def G_AMDGPU_FFBL_B32 : AMDGPUGenericInstruction { 3706 let OutOperandList = (outs type0:$dst); 3707 let InOperandList = (ins type1:$src); 3708 let hasSideEffects = 0; 3709} 3710 3711def G_AMDGPU_RCP_IFLAG : AMDGPUGenericInstruction { 3712 let OutOperandList = (outs type0:$dst); 3713 let InOperandList = (ins type1:$src); 3714 let hasSideEffects = 0; 3715} 3716 3717class BufferLoadGenericInstruction : AMDGPUGenericInstruction { 3718 let OutOperandList = (outs type0:$dst); 3719 let InOperandList = (ins type1:$rsrc, type2:$vindex, type2:$voffset, 3720 type2:$soffset, untyped_imm_0:$offset, 3721 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3722 let hasSideEffects = 0; 3723 let mayLoad = 1; 3724} 3725 3726class TBufferLoadGenericInstruction : AMDGPUGenericInstruction { 3727 let OutOperandList = (outs type0:$dst); 3728 let InOperandList = (ins type1:$rsrc, type2:$vindex, type2:$voffset, 3729 type2:$soffset, untyped_imm_0:$offset, untyped_imm_0:$format, 3730 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3731 let hasSideEffects = 0; 3732 let mayLoad = 1; 3733} 3734 3735def G_AMDGPU_BUFFER_LOAD_UBYTE : BufferLoadGenericInstruction; 3736def G_AMDGPU_BUFFER_LOAD_SBYTE : BufferLoadGenericInstruction; 3737def G_AMDGPU_BUFFER_LOAD_USHORT : BufferLoadGenericInstruction; 3738def G_AMDGPU_BUFFER_LOAD_SSHORT : BufferLoadGenericInstruction; 3739def G_AMDGPU_BUFFER_LOAD : BufferLoadGenericInstruction; 3740def G_AMDGPU_BUFFER_LOAD_FORMAT : BufferLoadGenericInstruction; 3741def G_AMDGPU_BUFFER_LOAD_FORMAT_TFE : BufferLoadGenericInstruction; 3742def G_AMDGPU_BUFFER_LOAD_FORMAT_D16 : BufferLoadGenericInstruction; 3743def G_AMDGPU_TBUFFER_LOAD_FORMAT : TBufferLoadGenericInstruction; 3744def G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 : TBufferLoadGenericInstruction; 3745 3746class BufferStoreGenericInstruction : AMDGPUGenericInstruction { 3747 let OutOperandList = (outs); 3748 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, 3749 type2:$soffset, untyped_imm_0:$offset, 3750 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3751 let hasSideEffects = 0; 3752 let mayStore = 1; 3753} 3754 3755class TBufferStoreGenericInstruction : AMDGPUGenericInstruction { 3756 let OutOperandList = (outs); 3757 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, 3758 type2:$soffset, untyped_imm_0:$offset, 3759 untyped_imm_0:$format, 3760 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3761 let hasSideEffects = 0; 3762 let mayStore = 1; 3763} 3764 3765def G_AMDGPU_BUFFER_STORE : BufferStoreGenericInstruction; 3766def G_AMDGPU_BUFFER_STORE_BYTE : BufferStoreGenericInstruction; 3767def G_AMDGPU_BUFFER_STORE_SHORT : BufferStoreGenericInstruction; 3768def G_AMDGPU_BUFFER_STORE_FORMAT : BufferStoreGenericInstruction; 3769def G_AMDGPU_BUFFER_STORE_FORMAT_D16 : BufferStoreGenericInstruction; 3770def G_AMDGPU_TBUFFER_STORE_FORMAT : TBufferStoreGenericInstruction; 3771def G_AMDGPU_TBUFFER_STORE_FORMAT_D16 : TBufferStoreGenericInstruction; 3772 3773def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction { 3774 let OutOperandList = (outs type0:$dst); 3775 let InOperandList = (ins type0:$src0, type0:$src1); 3776 let hasSideEffects = 0; 3777} 3778 3779def G_AMDGPU_FMAX_LEGACY : AMDGPUGenericInstruction { 3780 let OutOperandList = (outs type0:$dst); 3781 let InOperandList = (ins type0:$src0, type0:$src1); 3782 let hasSideEffects = 0; 3783} 3784 3785foreach N = 0-3 in { 3786def G_AMDGPU_CVT_F32_UBYTE#N : AMDGPUGenericInstruction { 3787 let OutOperandList = (outs type0:$dst); 3788 let InOperandList = (ins type0:$src0); 3789 let hasSideEffects = 0; 3790} 3791} 3792 3793def G_AMDGPU_CVT_PK_I16_I32 : AMDGPUGenericInstruction { 3794 let OutOperandList = (outs type0:$dst); 3795 let InOperandList = (ins type0:$src0, type0:$src1); 3796 let hasSideEffects = 0; 3797} 3798 3799def G_AMDGPU_SMED3 : AMDGPUGenericInstruction { 3800 let OutOperandList = (outs type0:$dst); 3801 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); 3802 let hasSideEffects = 0; 3803} 3804 3805def G_AMDGPU_UMED3 : AMDGPUGenericInstruction { 3806 let OutOperandList = (outs type0:$dst); 3807 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); 3808 let hasSideEffects = 0; 3809} 3810 3811def G_AMDGPU_FMED3 : AMDGPUGenericInstruction { 3812 let OutOperandList = (outs type0:$dst); 3813 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); 3814 let hasSideEffects = 0; 3815} 3816 3817def G_AMDGPU_CLAMP : AMDGPUGenericInstruction { 3818 let OutOperandList = (outs type0:$dst); 3819 let InOperandList = (ins type0:$src); 3820 let hasSideEffects = 0; 3821} 3822 3823// Integer multiply-add: arg0 * arg1 + arg2. 3824// 3825// arg0 and arg1 are 32-bit integers (interpreted as signed or unsigned), 3826// arg2 is a 64-bit integer. Result is a 64-bit integer and a 1-bit carry-out. 3827class G_AMDGPU_MAD_64_32 : AMDGPUGenericInstruction { 3828 let OutOperandList = (outs type0:$dst, type1:$carry_out); 3829 let InOperandList = (ins type2:$arg0, type2:$arg1, type0:$arg2); 3830 let hasSideEffects = 0; 3831} 3832 3833def G_AMDGPU_MAD_U64_U32 : G_AMDGPU_MAD_64_32; 3834def G_AMDGPU_MAD_I64_I32 : G_AMDGPU_MAD_64_32; 3835 3836// Atomic cmpxchg. $cmpval ad $newval are packed in a single vector 3837// operand Expects a MachineMemOperand in addition to explicit 3838// operands. 3839def G_AMDGPU_ATOMIC_CMPXCHG : AMDGPUGenericInstruction { 3840 let OutOperandList = (outs type0:$oldval); 3841 let InOperandList = (ins ptype1:$addr, type0:$cmpval_newval); 3842 let hasSideEffects = 0; 3843 let mayLoad = 1; 3844 let mayStore = 1; 3845} 3846 3847let Namespace = "AMDGPU" in { 3848def G_AMDGPU_ATOMIC_FMIN : G_ATOMICRMW_OP; 3849def G_AMDGPU_ATOMIC_FMAX : G_ATOMICRMW_OP; 3850} 3851 3852class BufferAtomicGenericInstruction : AMDGPUGenericInstruction { 3853 let OutOperandList = (outs type0:$dst); 3854 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, 3855 type2:$soffset, untyped_imm_0:$offset, 3856 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3857 let hasSideEffects = 0; 3858 let mayLoad = 1; 3859 let mayStore = 1; 3860} 3861 3862def G_AMDGPU_BUFFER_ATOMIC_SWAP : BufferAtomicGenericInstruction; 3863def G_AMDGPU_BUFFER_ATOMIC_ADD : BufferAtomicGenericInstruction; 3864def G_AMDGPU_BUFFER_ATOMIC_SUB : BufferAtomicGenericInstruction; 3865def G_AMDGPU_BUFFER_ATOMIC_SMIN : BufferAtomicGenericInstruction; 3866def G_AMDGPU_BUFFER_ATOMIC_UMIN : BufferAtomicGenericInstruction; 3867def G_AMDGPU_BUFFER_ATOMIC_SMAX : BufferAtomicGenericInstruction; 3868def G_AMDGPU_BUFFER_ATOMIC_UMAX : BufferAtomicGenericInstruction; 3869def G_AMDGPU_BUFFER_ATOMIC_AND : BufferAtomicGenericInstruction; 3870def G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 : BufferAtomicGenericInstruction; 3871def G_AMDGPU_BUFFER_ATOMIC_OR : BufferAtomicGenericInstruction; 3872def G_AMDGPU_BUFFER_ATOMIC_XOR : BufferAtomicGenericInstruction; 3873def G_AMDGPU_BUFFER_ATOMIC_INC : BufferAtomicGenericInstruction; 3874def G_AMDGPU_BUFFER_ATOMIC_DEC : BufferAtomicGenericInstruction; 3875def G_AMDGPU_BUFFER_ATOMIC_FADD : BufferAtomicGenericInstruction; 3876def G_AMDGPU_BUFFER_ATOMIC_FADD_BF16 : BufferAtomicGenericInstruction; 3877def G_AMDGPU_BUFFER_ATOMIC_FMIN : BufferAtomicGenericInstruction; 3878def G_AMDGPU_BUFFER_ATOMIC_FMAX : BufferAtomicGenericInstruction; 3879 3880def G_AMDGPU_BUFFER_ATOMIC_CMPSWAP : AMDGPUGenericInstruction { 3881 let OutOperandList = (outs type0:$dst); 3882 let InOperandList = (ins type0:$vdata, type0:$cmp, type1:$rsrc, type2:$vindex, 3883 type2:$voffset, type2:$soffset, untyped_imm_0:$offset, 3884 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3885 let hasSideEffects = 0; 3886 let mayLoad = 1; 3887 let mayStore = 1; 3888} 3889 3890// Wrapper around llvm.amdgcn.s.buffer.load. This is mostly needed as 3891// a workaround for the intrinsic being defined as readnone, but 3892// really needs a memory operand. 3893 3894class SBufferLoadInstruction : AMDGPUGenericInstruction { 3895 let OutOperandList = (outs type0:$dst); 3896 let InOperandList = (ins type1:$rsrc, type2:$offset, untyped_imm_0:$cachepolicy); 3897 let hasSideEffects = 0; 3898 let mayLoad = 1; 3899 let mayStore = 0; 3900} 3901 3902def G_AMDGPU_S_BUFFER_LOAD : SBufferLoadInstruction; 3903def G_AMDGPU_S_BUFFER_LOAD_SBYTE : SBufferLoadInstruction; 3904def G_AMDGPU_S_BUFFER_LOAD_UBYTE : SBufferLoadInstruction; 3905def G_AMDGPU_S_BUFFER_LOAD_SSHORT : SBufferLoadInstruction; 3906def G_AMDGPU_S_BUFFER_LOAD_USHORT : SBufferLoadInstruction; 3907 3908def G_AMDGPU_S_MUL_U64_U32 : AMDGPUGenericInstruction { 3909 let OutOperandList = (outs type0:$dst); 3910 let InOperandList = (ins type0:$src0, type0:$src1); 3911 let hasSideEffects = 0; 3912} 3913 3914def G_AMDGPU_S_MUL_I64_I32 : AMDGPUGenericInstruction { 3915 let OutOperandList = (outs type0:$dst); 3916 let InOperandList = (ins type0:$src0, type0:$src1); 3917 let hasSideEffects = 0; 3918} 3919 3920// This is equivalent to the G_INTRINSIC*, but the operands may have 3921// been legalized depending on the subtarget requirements. 3922def G_AMDGPU_INTRIN_IMAGE_LOAD : AMDGPUGenericInstruction { 3923 let OutOperandList = (outs type0:$dst); 3924 let InOperandList = (ins unknown:$intrin, variable_ops); 3925 let hasSideEffects = 0; 3926 let mayLoad = 1; 3927 3928 // FIXME: Use separate opcode for atomics. 3929 let mayStore = 1; 3930} 3931 3932def G_AMDGPU_INTRIN_IMAGE_LOAD_D16 : AMDGPUGenericInstruction { 3933 let OutOperandList = (outs type0:$dst); 3934 let InOperandList = (ins unknown:$intrin, variable_ops); 3935 let hasSideEffects = 0; 3936 let mayLoad = 1; 3937 3938 // FIXME: Use separate opcode for atomics. 3939 let mayStore = 1; 3940} 3941 3942// This is equivalent to the G_INTRINSIC*, but the operands may have 3943// been legalized depending on the subtarget requirements. 3944def G_AMDGPU_INTRIN_IMAGE_STORE : AMDGPUGenericInstruction { 3945 let OutOperandList = (outs); 3946 let InOperandList = (ins unknown:$intrin, variable_ops); 3947 let hasSideEffects = 0; 3948 let mayStore = 1; 3949} 3950 3951def G_AMDGPU_INTRIN_IMAGE_STORE_D16 : AMDGPUGenericInstruction { 3952 let OutOperandList = (outs); 3953 let InOperandList = (ins unknown:$intrin, variable_ops); 3954 let hasSideEffects = 0; 3955 let mayStore = 1; 3956} 3957 3958def G_AMDGPU_INTRIN_BVH_INTERSECT_RAY : AMDGPUGenericInstruction { 3959 let OutOperandList = (outs type0:$dst); 3960 let InOperandList = (ins unknown:$intrin, variable_ops); 3961 let hasSideEffects = 0; 3962 let mayLoad = 1; 3963 let mayStore = 0; 3964} 3965 3966// Generic instruction for SI_CALL, so we can select the register bank and insert a waterfall loop 3967// if necessary. 3968def G_SI_CALL : AMDGPUGenericInstruction { 3969 let OutOperandList = (outs SReg_64:$dst); 3970 let InOperandList = (ins type0:$src0, unknown:$callee); 3971 let Size = 4; 3972 let isCall = 1; 3973 let UseNamedOperandTable = 1; 3974 let SchedRW = [WriteBranch]; 3975 // TODO: Should really base this on the call target 3976 let isConvergent = 1; 3977} 3978 3979def G_FPTRUNC_ROUND_UPWARD : AMDGPUGenericInstruction { 3980 let OutOperandList = (outs type0:$vdst); 3981 let InOperandList = (ins type1:$src0); 3982 let hasSideEffects = 0; 3983} 3984 3985def G_FPTRUNC_ROUND_DOWNWARD : AMDGPUGenericInstruction { 3986 let OutOperandList = (outs type0:$vdst); 3987 let InOperandList = (ins type1:$src0); 3988 let hasSideEffects = 0; 3989} 3990 3991//============================================================================// 3992// Dummy Instructions 3993//============================================================================// 3994 3995def V_ILLEGAL : Enc32, InstSI<(outs), (ins), "v_illegal"> { 3996 let Inst{31-0} = 0x00000000; 3997 let FixedSize = 1; 3998 let Size = 4; 3999 let Uses = [EXEC]; 4000 let hasSideEffects = 1; 4001 let SubtargetPredicate = isGFX10Plus; 4002} 4003