1//===-- SIInstructions.td - SI Instruction Definitions --------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// This file was originally auto-generated from a GPU register header file and 9// all the instruction definitions were originally commented out. Instructions 10// that are not yet supported remain commented out. 11//===----------------------------------------------------------------------===// 12 13class GCNPat<dag pattern, dag result> : Pat<pattern, result>, GCNPredicateControl { 14 15} 16 17class UniformSextInreg<ValueType VT> : PatFrag< 18 (ops node:$src), 19 (sext_inreg $src, VT), 20 [{ return !N->isDivergent(); }]>; 21 22class DivergentSextInreg<ValueType VT> : PatFrag< 23 (ops node:$src), 24 (sext_inreg $src, VT), 25 [{ return N->isDivergent(); }]>; 26 27include "SOPInstructions.td" 28include "VOPInstructions.td" 29include "SMInstructions.td" 30include "FLATInstructions.td" 31include "BUFInstructions.td" 32include "EXPInstructions.td" 33include "LDSDIRInstructions.td" 34include "VINTERPInstructions.td" 35 36//===----------------------------------------------------------------------===// 37// VINTRP Instructions 38//===----------------------------------------------------------------------===// 39 40// Used to inject printing of "_e32" suffix for VI (there are "_e64" variants for VI) 41def VINTRPDst : VINTRPDstOperand <VGPR_32>; 42 43let Uses = [MODE, M0, EXEC] in { 44 45// FIXME: Specify SchedRW for VINTRP instructions. 46 47multiclass V_INTERP_P1_F32_m : VINTRP_m < 48 0x00000000, 49 (outs VINTRPDst:$vdst), 50 (ins VGPR_32:$vsrc, InterpAttr:$attr, InterpAttrChan:$attrchan), 51 "v_interp_p1_f32$vdst, $vsrc, $attr$attrchan", 52 [(set f32:$vdst, (int_amdgcn_interp_p1 f32:$vsrc, 53 (i32 timm:$attrchan), (i32 timm:$attr), M0))] 54>; 55 56let OtherPredicates = [has32BankLDS, isNotGFX90APlus] in { 57 58defm V_INTERP_P1_F32 : V_INTERP_P1_F32_m; 59 60} // End OtherPredicates = [has32BankLDS, isNotGFX90APlus] 61 62let OtherPredicates = [has16BankLDS, isNotGFX90APlus], 63 Constraints = "@earlyclobber $vdst", isAsmParserOnly=1 in { 64 65defm V_INTERP_P1_F32_16bank : V_INTERP_P1_F32_m; 66 67} // End OtherPredicates = [has32BankLDS, isNotGFX90APlus], 68 // Constraints = "@earlyclobber $vdst", isAsmParserOnly=1 69 70let OtherPredicates = [isNotGFX90APlus] in { 71let DisableEncoding = "$src0", Constraints = "$src0 = $vdst" in { 72 73defm V_INTERP_P2_F32 : VINTRP_m < 74 0x00000001, 75 (outs VINTRPDst:$vdst), 76 (ins VGPR_32:$src0, VGPR_32:$vsrc, InterpAttr:$attr, 77 InterpAttrChan:$attrchan), 78 "v_interp_p2_f32$vdst, $vsrc, $attr$attrchan", 79 [(set f32:$vdst, (int_amdgcn_interp_p2 f32:$src0, f32:$vsrc, 80 (i32 timm:$attrchan), (i32 timm:$attr), M0))]>; 81 82} // End DisableEncoding = "$src0", Constraints = "$src0 = $vdst" 83 84defm V_INTERP_MOV_F32 : VINTRP_m < 85 0x00000002, 86 (outs VINTRPDst:$vdst), 87 (ins InterpSlot:$vsrc, InterpAttr:$attr, InterpAttrChan:$attrchan), 88 "v_interp_mov_f32$vdst, $vsrc, $attr$attrchan", 89 [(set f32:$vdst, (int_amdgcn_interp_mov (i32 timm:$vsrc), 90 (i32 timm:$attrchan), (i32 timm:$attr), M0))]>; 91 92} // End OtherPredicates = [isNotGFX90APlus] 93 94} // End Uses = [MODE, M0, EXEC] 95 96//===----------------------------------------------------------------------===// 97// Pseudo Instructions 98//===----------------------------------------------------------------------===// 99 100// Insert a branch to an endpgm block to use as a fallback trap. 101def ENDPGM_TRAP : SPseudoInstSI< 102 (outs), (ins), 103 [(AMDGPUendpgm_trap)], 104 "ENDPGM_TRAP"> { 105 let hasSideEffects = 1; 106 let usesCustomInserter = 1; 107} 108 109def ATOMIC_FENCE : SPseudoInstSI< 110 (outs), (ins i32imm:$ordering, i32imm:$scope), 111 [(atomic_fence (i32 timm:$ordering), (i32 timm:$scope))], 112 "ATOMIC_FENCE $ordering, $scope"> { 113 let hasSideEffects = 1; 114 let maybeAtomic = 1; 115} 116 117let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in { 118 119// For use in patterns 120def V_CNDMASK_B64_PSEUDO : VOP3Common <(outs VReg_64:$vdst), 121 (ins VSrc_b64:$src0, VSrc_b64:$src1, SSrc_b64:$src2), "", []> { 122 let isPseudo = 1; 123 let isCodeGenOnly = 1; 124 let usesCustomInserter = 1; 125} 126 127// 64-bit vector move instruction. This is mainly used by the 128// SIFoldOperands pass to enable folding of inline immediates. 129def V_MOV_B64_PSEUDO : VPseudoInstSI <(outs VReg_64:$vdst), 130 (ins VSrc_b64:$src0)> { 131 let isReMaterializable = 1; 132 let isAsCheapAsAMove = 1; 133 let isMoveImm = 1; 134 let SchedRW = [Write64Bit]; 135 let Size = 16; // Needs maximum 2 v_mov_b32 instructions 8 byte long each. 136 let UseNamedOperandTable = 1; 137} 138 139// 64-bit vector move with dpp. Expanded post-RA. 140def V_MOV_B64_DPP_PSEUDO : VOP_DPP_Pseudo <"v_mov_b64_dpp", VOP_I64_I64> { 141 let Size = 16; // Requires two 8-byte v_mov_b32_dpp to complete. 142} 143 144// 64-bit scalar move immediate instruction. This is used to avoid subregs 145// initialization and allow rematerialization. 146def S_MOV_B64_IMM_PSEUDO : SPseudoInstSI <(outs SReg_64:$sdst), 147 (ins i64imm:$src0)> { 148 let isReMaterializable = 1; 149 let isAsCheapAsAMove = 1; 150 let isMoveImm = 1; 151 let SchedRW = [WriteSALU, Write64Bit]; 152 let Size = 16; // Needs maximum 2 s_mov_b32 instructions 8 byte long each. 153 let Uses = []; 154} 155 156// Pseudoinstruction for @llvm.amdgcn.wqm. It is turned into a copy after the 157// WQM pass processes it. 158def WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 159 160// Pseudoinstruction for @llvm.amdgcn.softwqm. Like @llvm.amdgcn.wqm it is 161// turned into a copy by WQM pass, but does not seed WQM requirements. 162def SOFT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 163 164// Pseudoinstruction for @llvm.amdgcn.strict.wwm. It is turned into a copy post-RA, so 165// that the @earlyclobber is respected. The @earlyclobber is to make sure that 166// the instruction that defines $src0 (which is run in Whole Wave Mode) doesn't 167// accidentally clobber inactive channels of $vdst. 168let Constraints = "@earlyclobber $vdst" in { 169def STRICT_WWM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 170def STRICT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>; 171} 172 173} // End let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] 174 175def ENTER_STRICT_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> { 176 let Uses = [EXEC]; 177 let Defs = [EXEC, SCC]; 178 let hasSideEffects = 0; 179 let mayLoad = 0; 180 let mayStore = 0; 181} 182 183def EXIT_STRICT_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> { 184 let hasSideEffects = 0; 185 let mayLoad = 0; 186 let mayStore = 0; 187} 188 189def ENTER_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> { 190 let Uses = [EXEC]; 191 let Defs = [EXEC, SCC]; 192 let hasSideEffects = 0; 193 let mayLoad = 0; 194 let mayStore = 0; 195} 196 197def EXIT_STRICT_WQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> { 198 let hasSideEffects = 0; 199 let mayLoad = 0; 200 let mayStore = 0; 201} 202 203let usesCustomInserter = 1 in { 204def S_INVERSE_BALLOT_U32 : SPseudoInstSI <(outs SReg_32:$sdst), (ins SSrc_b32:$mask)>; 205 206def S_INVERSE_BALLOT_U64 : SPseudoInstSI <(outs SReg_64:$sdst), (ins SSrc_b64:$mask)>; 207} // End usesCustomInserter = 1 208 209// PSEUDO_WM is treated like STRICT_WWM/STRICT_WQM without exec changes. 210def ENTER_PSEUDO_WM : SPseudoInstSI <(outs), (ins)> { 211 let Uses = [EXEC]; 212 let Defs = [EXEC]; 213 let hasSideEffects = 0; 214 let mayLoad = 0; 215 let mayStore = 0; 216} 217 218def EXIT_PSEUDO_WM : SPseudoInstSI <(outs), (ins)> { 219 let hasSideEffects = 0; 220 let mayLoad = 0; 221 let mayStore = 0; 222} 223 224// Pseudo instructions used for @llvm.fptrunc.round upward 225// and @llvm.fptrunc.round downward. 226// These intrinsics will be legalized to G_FPTRUNC_ROUND_UPWARD 227// and G_FPTRUNC_ROUND_DOWNWARD before being lowered to 228// FPTRUNC_UPWARD_PSEUDO and FPTRUNC_DOWNWARD_PSEUDO. 229// The final codegen is done in the ModeRegister pass. 230let Uses = [MODE, EXEC] in { 231def FPTRUNC_UPWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst), 232 (ins VGPR_32:$src0), 233 [(set f16:$vdst, (SIfptrunc_round_upward f32:$src0))]>; 234 235def FPTRUNC_DOWNWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst), 236 (ins VGPR_32:$src0), 237 [(set f16:$vdst, (SIfptrunc_round_downward f32:$src0))]>; 238} // End Uses = [MODE, EXEC] 239 240// Invert the exec mask and overwrite the inactive lanes of dst with inactive, 241// restoring it after we're done. 242let Defs = [SCC], isConvergent = 1 in { 243def V_SET_INACTIVE_B32 : VPseudoInstSI <(outs VGPR_32:$vdst), 244 (ins VSrc_b32: $src, VSrc_b32:$inactive), 245 [(set i32:$vdst, (int_amdgcn_set_inactive i32:$src, i32:$inactive))]> { 246} 247 248def V_SET_INACTIVE_B64 : VPseudoInstSI <(outs VReg_64:$vdst), 249 (ins VSrc_b64: $src, VSrc_b64:$inactive), 250 [(set i64:$vdst, (int_amdgcn_set_inactive i64:$src, i64:$inactive))]> { 251} 252} // End Defs = [SCC] 253 254let usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in { 255 def WAVE_REDUCE_UMIN_PSEUDO_U32 : VPseudoInstSI <(outs SGPR_32:$sdst), 256 (ins VSrc_b32: $src, VSrc_b32:$strategy), 257 [(set i32:$sdst, (int_amdgcn_wave_reduce_umin i32:$src, i32:$strategy))]> { 258 } 259 260 def WAVE_REDUCE_UMAX_PSEUDO_U32 : VPseudoInstSI <(outs SGPR_32:$sdst), 261 (ins VSrc_b32: $src, VSrc_b32:$strategy), 262 [(set i32:$sdst, (int_amdgcn_wave_reduce_umax i32:$src, i32:$strategy))]> { 263 } 264} 265 266let usesCustomInserter = 1, Defs = [VCC, EXEC] in { 267def V_ADD_U64_PSEUDO : VPseudoInstSI < 268 (outs VReg_64:$vdst), (ins VSrc_b64:$src0, VSrc_b64:$src1), 269 [(set VReg_64:$vdst, (DivergentBinFrag<add> i64:$src0, i64:$src1))] 270>; 271 272def V_SUB_U64_PSEUDO : VPseudoInstSI < 273 (outs VReg_64:$vdst), (ins VSrc_b64:$src0, VSrc_b64:$src1), 274 [(set VReg_64:$vdst, (DivergentBinFrag<sub> i64:$src0, i64:$src1))] 275>; 276} // End usesCustomInserter = 1, Defs = [VCC, EXEC] 277 278let usesCustomInserter = 1, Defs = [SCC] in { 279def S_ADD_U64_PSEUDO : SPseudoInstSI < 280 (outs SReg_64:$sdst), (ins SSrc_b64:$src0, SSrc_b64:$src1), 281 [(set SReg_64:$sdst, (UniformBinFrag<add> i64:$src0, i64:$src1))] 282>; 283 284def S_SUB_U64_PSEUDO : SPseudoInstSI < 285 (outs SReg_64:$sdst), (ins SSrc_b64:$src0, SSrc_b64:$src1), 286 [(set SReg_64:$sdst, (UniformBinFrag<sub> i64:$src0, i64:$src1))] 287>; 288 289def S_ADD_CO_PSEUDO : SPseudoInstSI < 290 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1, SSrc_i1:$scc_in) 291>; 292 293def S_SUB_CO_PSEUDO : SPseudoInstSI < 294 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1, SSrc_i1:$scc_in) 295>; 296 297def S_UADDO_PSEUDO : SPseudoInstSI < 298 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1) 299>; 300 301def S_USUBO_PSEUDO : SPseudoInstSI < 302 (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1) 303>; 304 305} // End usesCustomInserter = 1, Defs = [SCC] 306 307let usesCustomInserter = 1 in { 308def GET_GROUPSTATICSIZE : SPseudoInstSI <(outs SReg_32:$sdst), (ins), 309 [(set SReg_32:$sdst, (int_amdgcn_groupstaticsize))]>; 310} // End let usesCustomInserter = 1, SALU = 1 311 312// Wrap an instruction by duplicating it, except for setting isTerminator. 313class WrapTerminatorInst<SOP_Pseudo base_inst> : SPseudoInstSI< 314 base_inst.OutOperandList, 315 base_inst.InOperandList> { 316 let Uses = base_inst.Uses; 317 let Defs = base_inst.Defs; 318 let isTerminator = 1; 319 let isAsCheapAsAMove = base_inst.isAsCheapAsAMove; 320 let hasSideEffects = base_inst.hasSideEffects; 321 let UseNamedOperandTable = base_inst.UseNamedOperandTable; 322 let CodeSize = base_inst.CodeSize; 323 let SchedRW = base_inst.SchedRW; 324} 325 326let WaveSizePredicate = isWave64 in { 327def S_MOV_B64_term : WrapTerminatorInst<S_MOV_B64>; 328def S_XOR_B64_term : WrapTerminatorInst<S_XOR_B64>; 329def S_OR_B64_term : WrapTerminatorInst<S_OR_B64>; 330def S_ANDN2_B64_term : WrapTerminatorInst<S_ANDN2_B64>; 331def S_AND_B64_term : WrapTerminatorInst<S_AND_B64>; 332def S_AND_SAVEEXEC_B64_term : WrapTerminatorInst<S_AND_SAVEEXEC_B64>; 333} 334 335let WaveSizePredicate = isWave32 in { 336def S_MOV_B32_term : WrapTerminatorInst<S_MOV_B32>; 337def S_XOR_B32_term : WrapTerminatorInst<S_XOR_B32>; 338def S_OR_B32_term : WrapTerminatorInst<S_OR_B32>; 339def S_ANDN2_B32_term : WrapTerminatorInst<S_ANDN2_B32>; 340def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>; 341def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>; 342} 343 344 345def WAVE_BARRIER : SPseudoInstSI<(outs), (ins), 346 [(int_amdgcn_wave_barrier)]> { 347 let SchedRW = []; 348 let hasNoSchedulingInfo = 1; 349 let hasSideEffects = 1; 350 let mayLoad = 0; 351 let mayStore = 0; 352 let isConvergent = 1; 353 let FixedSize = 1; 354 let Size = 0; 355 let isMeta = 1; 356} 357 358def SCHED_BARRIER : SPseudoInstSI<(outs), (ins i32imm:$mask), 359 [(int_amdgcn_sched_barrier (i32 timm:$mask))]> { 360 let SchedRW = []; 361 let hasNoSchedulingInfo = 1; 362 let hasSideEffects = 1; 363 let mayLoad = 0; 364 let mayStore = 0; 365 let isConvergent = 1; 366 let FixedSize = 1; 367 let Size = 0; 368 let isMeta = 1; 369} 370 371def SCHED_GROUP_BARRIER : SPseudoInstSI< 372 (outs), 373 (ins i32imm:$mask, i32imm:$size, i32imm:$syncid), 374 [(int_amdgcn_sched_group_barrier (i32 timm:$mask), (i32 timm:$size), (i32 timm:$syncid))]> { 375 let SchedRW = []; 376 let hasNoSchedulingInfo = 1; 377 let hasSideEffects = 1; 378 let mayLoad = 0; 379 let mayStore = 0; 380 let isConvergent = 1; 381 let FixedSize = 1; 382 let Size = 0; 383 let isMeta = 1; 384} 385 386def IGLP_OPT : SPseudoInstSI<(outs), (ins i32imm:$mask), 387 [(int_amdgcn_iglp_opt (i32 timm:$mask))]> { 388 let SchedRW = []; 389 let hasNoSchedulingInfo = 1; 390 let hasSideEffects = 1; 391 let mayLoad = 0; 392 let mayStore = 0; 393 let isConvergent = 1; 394 let FixedSize = 1; 395 let Size = 0; 396 let isMeta = 1; 397} 398 399// SI pseudo instructions. These are used by the CFG structurizer pass 400// and should be lowered to ISA instructions prior to codegen. 401 402// As we have enhanced control flow intrinsics to work under unstructured CFG, 403// duplicating such intrinsics can be actually treated as legal. On the contrary, 404// by making them non-duplicable, we are observing better code generation result. 405// So we choose to mark them non-duplicable in hope of getting better code 406// generation as well as simplied CFG during Machine IR optimization stage. 407 408let isTerminator = 1, isNotDuplicable = 1 in { 409 410let OtherPredicates = [EnableLateCFGStructurize] in { 411 def SI_NON_UNIFORM_BRCOND_PSEUDO : CFPseudoInstSI < 412 (outs), 413 (ins SReg_1:$vcc, brtarget:$target), 414 [(brcond i1:$vcc, bb:$target)]> { 415 let Size = 12; 416} 417} 418 419def SI_IF: CFPseudoInstSI < 420 (outs SReg_1:$dst), (ins SReg_1:$vcc, brtarget:$target), 421 [(set i1:$dst, (AMDGPUif i1:$vcc, bb:$target))], 1, 1> { 422 let Constraints = ""; 423 let Size = 12; 424 let hasSideEffects = 1; 425 let IsNeverUniform = 1; 426} 427 428def SI_ELSE : CFPseudoInstSI < 429 (outs SReg_1:$dst), 430 (ins SReg_1:$src, brtarget:$target), [], 1, 1> { 431 let Size = 12; 432 let hasSideEffects = 1; 433 let IsNeverUniform = 1; 434} 435 436def SI_WATERFALL_LOOP : CFPseudoInstSI < 437 (outs), 438 (ins brtarget:$target), [], 1> { 439 let Size = 8; 440 let isBranch = 1; 441 let Defs = []; 442} 443 444def SI_LOOP : CFPseudoInstSI < 445 (outs), (ins SReg_1:$saved, brtarget:$target), 446 [(AMDGPUloop i1:$saved, bb:$target)], 1, 1> { 447 let Size = 8; 448 let isBranch = 1; 449 let hasSideEffects = 1; 450 let IsNeverUniform = 1; 451} 452 453} // End isTerminator = 1 454 455def SI_END_CF : CFPseudoInstSI < 456 (outs), (ins SReg_1:$saved), [], 1, 1> { 457 let Size = 4; 458 let isAsCheapAsAMove = 1; 459 let isReMaterializable = 1; 460 let hasSideEffects = 1; 461 let isNotDuplicable = 1; // Not a hard requirement, see long comments above for details. 462 let mayLoad = 1; // FIXME: Should not need memory flags 463 let mayStore = 1; 464} 465 466def SI_IF_BREAK : CFPseudoInstSI < 467 (outs SReg_1:$dst), (ins SReg_1:$vcc, SReg_1:$src), []> { 468 let Size = 4; 469 let isNotDuplicable = 1; // Not a hard requirement, see long comments above for details. 470 let isAsCheapAsAMove = 1; 471 let isReMaterializable = 1; 472} 473 474// Branch to the early termination block of the shader if SCC is 0. 475// This uses SCC from a previous SALU operation, i.e. the update of 476// a mask of live lanes after a kill/demote operation. 477// Only valid in pixel shaders. 478def SI_EARLY_TERMINATE_SCC0 : SPseudoInstSI <(outs), (ins)> { 479 let Uses = [EXEC,SCC]; 480} 481 482let Uses = [EXEC] in { 483 484multiclass PseudoInstKill <dag ins> { 485 // Even though this pseudo can usually be expanded without an SCC def, we 486 // conservatively assume that it has an SCC def, both because it is sometimes 487 // required in degenerate cases (when V_CMPX cannot be used due to constant 488 // bus limitations) and because it allows us to avoid having to track SCC 489 // liveness across basic blocks. 490 let Defs = [EXEC,SCC] in 491 def _PSEUDO : PseudoInstSI <(outs), ins> { 492 let isConvergent = 1; 493 let usesCustomInserter = 1; 494 } 495 496 let Defs = [EXEC,SCC] in 497 def _TERMINATOR : SPseudoInstSI <(outs), ins> { 498 let isTerminator = 1; 499 } 500} 501 502defm SI_KILL_I1 : PseudoInstKill <(ins SCSrc_i1:$src, i1imm:$killvalue)>; 503let Defs = [VCC] in 504defm SI_KILL_F32_COND_IMM : PseudoInstKill <(ins VSrc_b32:$src0, i32imm:$src1, i32imm:$cond)>; 505 506let Defs = [EXEC,VCC] in 507def SI_ILLEGAL_COPY : SPseudoInstSI < 508 (outs unknown:$dst), (ins unknown:$src), 509 [], " ; illegal copy $src to $dst">; 510 511} // End Uses = [EXEC], Defs = [EXEC,VCC] 512 513// Branch on undef scc. Used to avoid intermediate copy from 514// IMPLICIT_DEF to SCC. 515def SI_BR_UNDEF : SPseudoInstSI <(outs), (ins SOPPBrTarget:$simm16)> { 516 let isTerminator = 1; 517 let usesCustomInserter = 1; 518 let isBranch = 1; 519} 520 521def SI_PS_LIVE : PseudoInstSI < 522 (outs SReg_1:$dst), (ins), 523 [(set i1:$dst, (int_amdgcn_ps_live))]> { 524 let SALU = 1; 525} 526 527let Uses = [EXEC] in { 528def SI_LIVE_MASK : PseudoInstSI < 529 (outs SReg_1:$dst), (ins), 530 [(set i1:$dst, (int_amdgcn_live_mask))]> { 531 let SALU = 1; 532} 533let Defs = [EXEC,SCC] in { 534// Demote: Turn a pixel shader thread into a helper lane. 535def SI_DEMOTE_I1 : SPseudoInstSI <(outs), (ins SCSrc_i1:$src, i1imm:$killvalue)>; 536} // End Defs = [EXEC,SCC] 537} // End Uses = [EXEC] 538 539def SI_MASKED_UNREACHABLE : SPseudoInstSI <(outs), (ins), 540 [(int_amdgcn_unreachable)], 541 "; divergent unreachable"> { 542 let Size = 0; 543 let hasNoSchedulingInfo = 1; 544 let FixedSize = 1; 545 let isMeta = 1; 546} 547 548// Used as an isel pseudo to directly emit initialization with an 549// s_mov_b32 rather than a copy of another initialized 550// register. MachineCSE skips copies, and we don't want to have to 551// fold operands before it runs. 552def SI_INIT_M0 : SPseudoInstSI <(outs), (ins SSrc_b32:$src)> { 553 let Defs = [M0]; 554 let usesCustomInserter = 1; 555 let isAsCheapAsAMove = 1; 556 let isReMaterializable = 1; 557} 558 559def SI_INIT_EXEC : SPseudoInstSI < 560 (outs), (ins i64imm:$src), 561 [(int_amdgcn_init_exec (i64 timm:$src))]> { 562 let Defs = [EXEC]; 563 let isAsCheapAsAMove = 1; 564} 565 566def SI_INIT_EXEC_FROM_INPUT : SPseudoInstSI < 567 (outs), (ins SSrc_b32:$input, i32imm:$shift), 568 [(int_amdgcn_init_exec_from_input i32:$input, (i32 timm:$shift))]> { 569 let Defs = [EXEC]; 570} 571 572// Return for returning shaders to a shader variant epilog. 573def SI_RETURN_TO_EPILOG : SPseudoInstSI < 574 (outs), (ins variable_ops), [(AMDGPUreturn_to_epilog)]> { 575 let isTerminator = 1; 576 let isBarrier = 1; 577 let isReturn = 1; 578 let hasNoSchedulingInfo = 1; 579 let DisableWQM = 1; 580 let FixedSize = 1; 581 582 // TODO: Should this be true? 583 let isMeta = 0; 584} 585 586// Return for returning function calls. 587def SI_RETURN : SPseudoInstSI < 588 (outs), (ins), [(AMDGPUret_glue)], 589 "; return"> { 590 let isTerminator = 1; 591 let isBarrier = 1; 592 let isReturn = 1; 593 let SchedRW = [WriteBranch]; 594} 595 596// Return for returning function calls without output register. 597// 598// This version is only needed so we can fill in the output register 599// in the custom inserter. 600def SI_CALL_ISEL : SPseudoInstSI < 601 (outs), (ins SSrc_b64:$src0, unknown:$callee), 602 [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> { 603 let Size = 4; 604 let isCall = 1; 605 let SchedRW = [WriteBranch]; 606 let usesCustomInserter = 1; 607 // TODO: Should really base this on the call target 608 let isConvergent = 1; 609} 610 611def : GCNPat< 612 (AMDGPUcall i64:$src0, (i64 0)), 613 (SI_CALL_ISEL $src0, (i64 0)) 614>; 615 616// Wrapper around s_swappc_b64 with extra $callee parameter to track 617// the called function after regalloc. 618def SI_CALL : SPseudoInstSI < 619 (outs SReg_64:$dst), (ins SSrc_b64:$src0, unknown:$callee)> { 620 let Size = 4; 621 let FixedSize = 1; 622 let isCall = 1; 623 let UseNamedOperandTable = 1; 624 let SchedRW = [WriteBranch]; 625 // TODO: Should really base this on the call target 626 let isConvergent = 1; 627} 628 629class SI_TCRETURN_Pseudo<RegisterClass rc, SDNode sd> : SPseudoInstSI <(outs), 630 (ins rc:$src0, unknown:$callee, i32imm:$fpdiff), 631 [(sd i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> { 632 let Size = 4; 633 let FixedSize = 1; 634 let isCall = 1; 635 let isTerminator = 1; 636 let isReturn = 1; 637 let isBarrier = 1; 638 let UseNamedOperandTable = 1; 639 let SchedRW = [WriteBranch]; 640 // TODO: Should really base this on the call target 641 let isConvergent = 1; 642} 643 644// Tail call handling pseudo 645def SI_TCRETURN : SI_TCRETURN_Pseudo<CCR_SGPR_64, AMDGPUtc_return>; 646def SI_TCRETURN_GFX : SI_TCRETURN_Pseudo<Gfx_CCR_SGPR_64, AMDGPUtc_return_gfx>; 647 648// Handle selecting indirect tail calls 649def : GCNPat< 650 (AMDGPUtc_return i64:$src0, (i64 0), (i32 timm:$fpdiff)), 651 (SI_TCRETURN CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff) 652>; 653 654// Handle selecting indirect tail calls for AMDGPU_gfx 655def : GCNPat< 656 (AMDGPUtc_return_gfx i64:$src0, (i64 0), (i32 timm:$fpdiff)), 657 (SI_TCRETURN_GFX Gfx_CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff) 658>; 659 660def ADJCALLSTACKUP : SPseudoInstSI< 661 (outs), (ins i32imm:$amt0, i32imm:$amt1), 662 [(callseq_start timm:$amt0, timm:$amt1)], 663 "; adjcallstackup $amt0 $amt1"> { 664 let Size = 8; // Worst case. (s_add_u32 + constant) 665 let FixedSize = 1; 666 let hasSideEffects = 1; 667 let usesCustomInserter = 1; 668 let SchedRW = [WriteSALU]; 669 let Defs = [SCC]; 670} 671 672def ADJCALLSTACKDOWN : SPseudoInstSI< 673 (outs), (ins i32imm:$amt1, i32imm:$amt2), 674 [(callseq_end timm:$amt1, timm:$amt2)], 675 "; adjcallstackdown $amt1"> { 676 let Size = 8; // Worst case. (s_add_u32 + constant) 677 let hasSideEffects = 1; 678 let usesCustomInserter = 1; 679 let SchedRW = [WriteSALU]; 680 let Defs = [SCC]; 681} 682 683let Defs = [M0, EXEC, SCC], 684 UseNamedOperandTable = 1 in { 685 686// SI_INDIRECT_SRC/DST are only used by legacy SelectionDAG indirect 687// addressing implementation. 688class SI_INDIRECT_SRC<RegisterClass rc> : VPseudoInstSI < 689 (outs VGPR_32:$vdst), 690 (ins rc:$src, VS_32:$idx, i32imm:$offset)> { 691 let usesCustomInserter = 1; 692} 693 694class SI_INDIRECT_DST<RegisterClass rc> : VPseudoInstSI < 695 (outs rc:$vdst), 696 (ins rc:$src, VS_32:$idx, i32imm:$offset, VGPR_32:$val)> { 697 let Constraints = "$src = $vdst"; 698 let usesCustomInserter = 1; 699} 700 701def SI_INDIRECT_SRC_V1 : SI_INDIRECT_SRC<VGPR_32>; 702def SI_INDIRECT_SRC_V2 : SI_INDIRECT_SRC<VReg_64>; 703def SI_INDIRECT_SRC_V4 : SI_INDIRECT_SRC<VReg_128>; 704def SI_INDIRECT_SRC_V8 : SI_INDIRECT_SRC<VReg_256>; 705def SI_INDIRECT_SRC_V9 : SI_INDIRECT_SRC<VReg_288>; 706def SI_INDIRECT_SRC_V10 : SI_INDIRECT_SRC<VReg_320>; 707def SI_INDIRECT_SRC_V11 : SI_INDIRECT_SRC<VReg_352>; 708def SI_INDIRECT_SRC_V12 : SI_INDIRECT_SRC<VReg_384>; 709def SI_INDIRECT_SRC_V16 : SI_INDIRECT_SRC<VReg_512>; 710def SI_INDIRECT_SRC_V32 : SI_INDIRECT_SRC<VReg_1024>; 711 712def SI_INDIRECT_DST_V1 : SI_INDIRECT_DST<VGPR_32>; 713def SI_INDIRECT_DST_V2 : SI_INDIRECT_DST<VReg_64>; 714def SI_INDIRECT_DST_V4 : SI_INDIRECT_DST<VReg_128>; 715def SI_INDIRECT_DST_V8 : SI_INDIRECT_DST<VReg_256>; 716def SI_INDIRECT_DST_V9 : SI_INDIRECT_DST<VReg_288>; 717def SI_INDIRECT_DST_V10 : SI_INDIRECT_DST<VReg_320>; 718def SI_INDIRECT_DST_V11 : SI_INDIRECT_DST<VReg_352>; 719def SI_INDIRECT_DST_V12 : SI_INDIRECT_DST<VReg_384>; 720def SI_INDIRECT_DST_V16 : SI_INDIRECT_DST<VReg_512>; 721def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>; 722 723} // End Uses = [EXEC], Defs = [M0, EXEC] 724 725// This is a pseudo variant of the v_movreld_b32 instruction in which the 726// vector operand appears only twice, once as def and once as use. Using this 727// pseudo avoids problems with the Two Address instructions pass. 728class INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc, 729 RegisterOperand val_ty> : PseudoInstSI < 730 (outs rc:$vdst), (ins rc:$vsrc, val_ty:$val, i32imm:$subreg)> { 731 let Constraints = "$vsrc = $vdst"; 732 let Uses = [M0]; 733} 734 735class V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> : 736 INDIRECT_REG_WRITE_MOVREL_pseudo<rc, VSrc_b32> { 737 let VALU = 1; 738 let VOP1 = 1; 739 let Uses = [M0, EXEC]; 740} 741 742class S_INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc, 743 RegisterOperand val_ty> : 744 INDIRECT_REG_WRITE_MOVREL_pseudo<rc, val_ty> { 745 let SALU = 1; 746 let SOP1 = 1; 747 let Uses = [M0]; 748} 749 750class S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> : 751 S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b32>; 752class S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<RegisterClass rc> : 753 S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b64>; 754 755def V_INDIRECT_REG_WRITE_MOVREL_B32_V1 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VGPR_32>; 756def V_INDIRECT_REG_WRITE_MOVREL_B32_V2 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_64>; 757def V_INDIRECT_REG_WRITE_MOVREL_B32_V3 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_96>; 758def V_INDIRECT_REG_WRITE_MOVREL_B32_V4 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_128>; 759def V_INDIRECT_REG_WRITE_MOVREL_B32_V5 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_160>; 760def V_INDIRECT_REG_WRITE_MOVREL_B32_V8 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_256>; 761def V_INDIRECT_REG_WRITE_MOVREL_B32_V9 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_288>; 762def V_INDIRECT_REG_WRITE_MOVREL_B32_V10 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_320>; 763def V_INDIRECT_REG_WRITE_MOVREL_B32_V11 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_352>; 764def V_INDIRECT_REG_WRITE_MOVREL_B32_V12 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_384>; 765def V_INDIRECT_REG_WRITE_MOVREL_B32_V16 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_512>; 766def V_INDIRECT_REG_WRITE_MOVREL_B32_V32 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_1024>; 767 768def S_INDIRECT_REG_WRITE_MOVREL_B32_V1 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_32>; 769def S_INDIRECT_REG_WRITE_MOVREL_B32_V2 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_64>; 770def S_INDIRECT_REG_WRITE_MOVREL_B32_V3 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_96>; 771def S_INDIRECT_REG_WRITE_MOVREL_B32_V4 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_128>; 772def S_INDIRECT_REG_WRITE_MOVREL_B32_V5 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_160>; 773def S_INDIRECT_REG_WRITE_MOVREL_B32_V8 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_256>; 774def S_INDIRECT_REG_WRITE_MOVREL_B32_V9 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_288>; 775def S_INDIRECT_REG_WRITE_MOVREL_B32_V10 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_320>; 776def S_INDIRECT_REG_WRITE_MOVREL_B32_V11 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_352>; 777def S_INDIRECT_REG_WRITE_MOVREL_B32_V12 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_384>; 778def S_INDIRECT_REG_WRITE_MOVREL_B32_V16 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_512>; 779def S_INDIRECT_REG_WRITE_MOVREL_B32_V32 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_1024>; 780 781def S_INDIRECT_REG_WRITE_MOVREL_B64_V1 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_64>; 782def S_INDIRECT_REG_WRITE_MOVREL_B64_V2 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_128>; 783def S_INDIRECT_REG_WRITE_MOVREL_B64_V4 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_256>; 784def S_INDIRECT_REG_WRITE_MOVREL_B64_V8 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_512>; 785def S_INDIRECT_REG_WRITE_MOVREL_B64_V16 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_1024>; 786 787// These variants of V_INDIRECT_REG_READ/WRITE use VGPR indexing. By using these 788// pseudos we avoid spills or copies being inserted within indirect sequences 789// that switch the VGPR indexing mode. Spills to accvgprs could be effected by 790// this mode switching. 791 792class V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI < 793 (outs rc:$vdst), (ins rc:$vsrc, VSrc_b32:$val, SSrc_b32:$idx, i32imm:$subreg)> { 794 let Constraints = "$vsrc = $vdst"; 795 let VALU = 1; 796 let Uses = [M0, EXEC]; 797 let Defs = [M0]; 798} 799 800def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VGPR_32>; 801def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_64>; 802def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_96>; 803def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_128>; 804def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_160>; 805def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_256>; 806def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_288>; 807def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_320>; 808def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_352>; 809def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_384>; 810def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_512>; 811def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_1024>; 812 813class V_INDIRECT_REG_READ_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI < 814 (outs VGPR_32:$vdst), (ins rc:$vsrc, SSrc_b32:$idx, i32imm:$subreg)> { 815 let VALU = 1; 816 let Uses = [M0, EXEC]; 817 let Defs = [M0]; 818} 819 820def V_INDIRECT_REG_READ_GPR_IDX_B32_V1 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VGPR_32>; 821def V_INDIRECT_REG_READ_GPR_IDX_B32_V2 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_64>; 822def V_INDIRECT_REG_READ_GPR_IDX_B32_V3 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_96>; 823def V_INDIRECT_REG_READ_GPR_IDX_B32_V4 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_128>; 824def V_INDIRECT_REG_READ_GPR_IDX_B32_V5 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_160>; 825def V_INDIRECT_REG_READ_GPR_IDX_B32_V8 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_256>; 826def V_INDIRECT_REG_READ_GPR_IDX_B32_V9 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_288>; 827def V_INDIRECT_REG_READ_GPR_IDX_B32_V10 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_320>; 828def V_INDIRECT_REG_READ_GPR_IDX_B32_V11 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_352>; 829def V_INDIRECT_REG_READ_GPR_IDX_B32_V12 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_384>; 830def V_INDIRECT_REG_READ_GPR_IDX_B32_V16 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_512>; 831def V_INDIRECT_REG_READ_GPR_IDX_B32_V32 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_1024>; 832 833multiclass SI_SPILL_SGPR <RegisterClass sgpr_class> { 834 let UseNamedOperandTable = 1, SGPRSpill = 1, Uses = [EXEC] in { 835 def _SAVE : PseudoInstSI < 836 (outs), 837 (ins sgpr_class:$data, i32imm:$addr)> { 838 let mayStore = 1; 839 let mayLoad = 0; 840 } 841 842 def _RESTORE : PseudoInstSI < 843 (outs sgpr_class:$data), 844 (ins i32imm:$addr)> { 845 let mayStore = 0; 846 let mayLoad = 1; 847 } 848 } // End UseNamedOperandTable = 1 849} 850 851// You cannot use M0 as the output of v_readlane_b32 instructions or 852// use it in the sdata operand of SMEM instructions. We still need to 853// be able to spill the physical register m0, so allow it for 854// SI_SPILL_32_* instructions. 855defm SI_SPILL_S32 : SI_SPILL_SGPR <SReg_32>; 856defm SI_SPILL_S64 : SI_SPILL_SGPR <SReg_64>; 857defm SI_SPILL_S96 : SI_SPILL_SGPR <SReg_96>; 858defm SI_SPILL_S128 : SI_SPILL_SGPR <SReg_128>; 859defm SI_SPILL_S160 : SI_SPILL_SGPR <SReg_160>; 860defm SI_SPILL_S192 : SI_SPILL_SGPR <SReg_192>; 861defm SI_SPILL_S224 : SI_SPILL_SGPR <SReg_224>; 862defm SI_SPILL_S256 : SI_SPILL_SGPR <SReg_256>; 863defm SI_SPILL_S288 : SI_SPILL_SGPR <SReg_288>; 864defm SI_SPILL_S320 : SI_SPILL_SGPR <SReg_320>; 865defm SI_SPILL_S352 : SI_SPILL_SGPR <SReg_352>; 866defm SI_SPILL_S384 : SI_SPILL_SGPR <SReg_384>; 867defm SI_SPILL_S512 : SI_SPILL_SGPR <SReg_512>; 868defm SI_SPILL_S1024 : SI_SPILL_SGPR <SReg_1024>; 869 870// VGPR or AGPR spill instructions. In case of AGPR spilling a temp register 871// needs to be used and an extra instruction to move between VGPR and AGPR. 872// UsesTmp adds to the total size of an expanded spill in this case. 873multiclass SI_SPILL_VGPR <RegisterClass vgpr_class, bit UsesTmp = 0> { 874 let UseNamedOperandTable = 1, VGPRSpill = 1, 875 SchedRW = [WriteVMEM] in { 876 def _SAVE : VPseudoInstSI < 877 (outs), 878 (ins vgpr_class:$vdata, i32imm:$vaddr, 879 SReg_32:$soffset, i32imm:$offset)> { 880 let mayStore = 1; 881 let mayLoad = 0; 882 // (2 * 4) + (8 * num_subregs) bytes maximum 883 int MaxSize = !add(!shl(!srl(vgpr_class.Size, 5), !add(UsesTmp, 3)), 8); 884 // Size field is unsigned char and cannot fit more. 885 let Size = !if(!le(MaxSize, 256), MaxSize, 252); 886 } 887 888 def _RESTORE : VPseudoInstSI < 889 (outs vgpr_class:$vdata), 890 (ins i32imm:$vaddr, 891 SReg_32:$soffset, i32imm:$offset)> { 892 let mayStore = 0; 893 let mayLoad = 1; 894 895 // (2 * 4) + (8 * num_subregs) bytes maximum 896 int MaxSize = !add(!shl(!srl(vgpr_class.Size, 5), !add(UsesTmp, 3)), 8); 897 // Size field is unsigned char and cannot fit more. 898 let Size = !if(!le(MaxSize, 256), MaxSize, 252); 899 } 900 } // End UseNamedOperandTable = 1, VGPRSpill = 1, SchedRW = [WriteVMEM] 901} 902 903defm SI_SPILL_V32 : SI_SPILL_VGPR <VGPR_32>; 904defm SI_SPILL_V64 : SI_SPILL_VGPR <VReg_64>; 905defm SI_SPILL_V96 : SI_SPILL_VGPR <VReg_96>; 906defm SI_SPILL_V128 : SI_SPILL_VGPR <VReg_128>; 907defm SI_SPILL_V160 : SI_SPILL_VGPR <VReg_160>; 908defm SI_SPILL_V192 : SI_SPILL_VGPR <VReg_192>; 909defm SI_SPILL_V224 : SI_SPILL_VGPR <VReg_224>; 910defm SI_SPILL_V256 : SI_SPILL_VGPR <VReg_256>; 911defm SI_SPILL_V288 : SI_SPILL_VGPR <VReg_288>; 912defm SI_SPILL_V320 : SI_SPILL_VGPR <VReg_320>; 913defm SI_SPILL_V352 : SI_SPILL_VGPR <VReg_352>; 914defm SI_SPILL_V384 : SI_SPILL_VGPR <VReg_384>; 915defm SI_SPILL_V512 : SI_SPILL_VGPR <VReg_512>; 916defm SI_SPILL_V1024 : SI_SPILL_VGPR <VReg_1024>; 917 918defm SI_SPILL_A32 : SI_SPILL_VGPR <AGPR_32, 1>; 919defm SI_SPILL_A64 : SI_SPILL_VGPR <AReg_64, 1>; 920defm SI_SPILL_A96 : SI_SPILL_VGPR <AReg_96, 1>; 921defm SI_SPILL_A128 : SI_SPILL_VGPR <AReg_128, 1>; 922defm SI_SPILL_A160 : SI_SPILL_VGPR <AReg_160, 1>; 923defm SI_SPILL_A192 : SI_SPILL_VGPR <AReg_192, 1>; 924defm SI_SPILL_A224 : SI_SPILL_VGPR <AReg_224, 1>; 925defm SI_SPILL_A256 : SI_SPILL_VGPR <AReg_256, 1>; 926defm SI_SPILL_A288 : SI_SPILL_VGPR <AReg_288, 1>; 927defm SI_SPILL_A320 : SI_SPILL_VGPR <AReg_320, 1>; 928defm SI_SPILL_A352 : SI_SPILL_VGPR <AReg_352, 1>; 929defm SI_SPILL_A384 : SI_SPILL_VGPR <AReg_384, 1>; 930defm SI_SPILL_A512 : SI_SPILL_VGPR <AReg_512, 1>; 931defm SI_SPILL_A1024 : SI_SPILL_VGPR <AReg_1024, 1>; 932 933defm SI_SPILL_AV32 : SI_SPILL_VGPR <AV_32, 1>; 934defm SI_SPILL_AV64 : SI_SPILL_VGPR <AV_64, 1>; 935defm SI_SPILL_AV96 : SI_SPILL_VGPR <AV_96, 1>; 936defm SI_SPILL_AV128 : SI_SPILL_VGPR <AV_128, 1>; 937defm SI_SPILL_AV160 : SI_SPILL_VGPR <AV_160, 1>; 938defm SI_SPILL_AV192 : SI_SPILL_VGPR <AV_192, 1>; 939defm SI_SPILL_AV224 : SI_SPILL_VGPR <AV_224, 1>; 940defm SI_SPILL_AV256 : SI_SPILL_VGPR <AV_256, 1>; 941defm SI_SPILL_AV288 : SI_SPILL_VGPR <AV_288, 1>; 942defm SI_SPILL_AV320 : SI_SPILL_VGPR <AV_320, 1>; 943defm SI_SPILL_AV352 : SI_SPILL_VGPR <AV_352, 1>; 944defm SI_SPILL_AV384 : SI_SPILL_VGPR <AV_384, 1>; 945defm SI_SPILL_AV512 : SI_SPILL_VGPR <AV_512, 1>; 946defm SI_SPILL_AV1024 : SI_SPILL_VGPR <AV_1024, 1>; 947 948let isConvergent = 1 in 949defm SI_SPILL_WWM_V32 : SI_SPILL_VGPR <VGPR_32>; 950 951def SI_PC_ADD_REL_OFFSET : SPseudoInstSI < 952 (outs SReg_64:$dst), 953 (ins si_ga:$ptr_lo, si_ga:$ptr_hi), 954 [(set SReg_64:$dst, 955 (i64 (SIpc_add_rel_offset tglobaladdr:$ptr_lo, tglobaladdr:$ptr_hi)))]> { 956 let Defs = [SCC]; 957} 958 959def : GCNPat < 960 (SIpc_add_rel_offset tglobaladdr:$ptr_lo, 0), 961 (SI_PC_ADD_REL_OFFSET $ptr_lo, (i32 0)) 962>; 963 964def : GCNPat< 965 (AMDGPUtrap timm:$trapid), 966 (S_TRAP $trapid) 967>; 968 969def : GCNPat< 970 (AMDGPUelse i1:$src, bb:$target), 971 (SI_ELSE $src, $target) 972>; 973 974def : Pat < 975 (int_amdgcn_kill i1:$src), 976 (SI_KILL_I1_PSEUDO SCSrc_i1:$src, 0) 977>; 978 979def : Pat < 980 (int_amdgcn_kill (i1 (not i1:$src))), 981 (SI_KILL_I1_PSEUDO SCSrc_i1:$src, -1) 982>; 983 984def : Pat < 985 (int_amdgcn_kill (i1 (setcc f32:$src, InlineImmFP32:$imm, cond:$cond))), 986 (SI_KILL_F32_COND_IMM_PSEUDO VSrc_b32:$src, (bitcast_fpimm_to_i32 $imm), (cond_as_i32imm $cond)) 987>; 988 989def : Pat < 990 (int_amdgcn_wqm_demote i1:$src), 991 (SI_DEMOTE_I1 SCSrc_i1:$src, 0) 992>; 993 994def : Pat < 995 (int_amdgcn_wqm_demote (i1 (not i1:$src))), 996 (SI_DEMOTE_I1 SCSrc_i1:$src, -1) 997>; 998 999 // TODO: we could add more variants for other types of conditionals 1000 1001def : Pat < 1002 (i64 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))), 1003 (COPY $src) // Return the SGPRs representing i1 src 1004>; 1005 1006def : Pat < 1007 (i32 (int_amdgcn_icmp i1:$src, (i1 0), (i32 33))), 1008 (COPY $src) // Return the SGPRs representing i1 src 1009>; 1010 1011//===----------------------------------------------------------------------===// 1012// VOP1 Patterns 1013//===----------------------------------------------------------------------===// 1014 1015multiclass f16_fp_Pats<Instruction cvt_f16_f32_inst_e64, Instruction cvt_f32_f16_inst_e64> { 1016 // f16_to_fp patterns 1017 def : GCNPat < 1018 (f32 (f16_to_fp i32:$src0)), 1019 (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src0) 1020 >; 1021 1022 def : GCNPat < 1023 (f32 (f16_to_fp (and_oneuse i32:$src0, 0x7fff))), 1024 (cvt_f32_f16_inst_e64 SRCMODS.ABS, $src0) 1025 >; 1026 1027 def : GCNPat < 1028 (f32 (f16_to_fp (i32 (srl_oneuse (and_oneuse i32:$src0, 0x7fff0000), (i32 16))))), 1029 (cvt_f32_f16_inst_e64 SRCMODS.ABS, (i32 (V_LSHRREV_B32_e64 (i32 16), i32:$src0))) 1030 >; 1031 1032 def : GCNPat < 1033 (f32 (f16_to_fp (or_oneuse i32:$src0, 0x8000))), 1034 (cvt_f32_f16_inst_e64 SRCMODS.NEG_ABS, $src0) 1035 >; 1036 1037 def : GCNPat < 1038 (f32 (f16_to_fp (xor_oneuse i32:$src0, 0x8000))), 1039 (cvt_f32_f16_inst_e64 SRCMODS.NEG, $src0) 1040 >; 1041 1042 def : GCNPat < 1043 (f64 (fpextend f16:$src)), 1044 (V_CVT_F64_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src)) 1045 >; 1046 1047 // fp_to_fp16 patterns 1048 def : GCNPat < 1049 (i32 (AMDGPUfp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), 1050 (cvt_f16_f32_inst_e64 $src0_modifiers, f32:$src0) 1051 >; 1052 1053 def : GCNPat < 1054 (i32 (fp_to_sint f16:$src)), 1055 (V_CVT_I32_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, VSrc_b32:$src)) 1056 >; 1057 1058 def : GCNPat < 1059 (i32 (fp_to_uint f16:$src)), 1060 (V_CVT_U32_F32_e32 (cvt_f32_f16_inst_e64 SRCMODS.NONE, VSrc_b32:$src)) 1061 >; 1062 1063 def : GCNPat < 1064 (f16 (sint_to_fp i32:$src)), 1065 (cvt_f16_f32_inst_e64 SRCMODS.NONE, (V_CVT_F32_I32_e32 VSrc_b32:$src)) 1066 >; 1067 1068 def : GCNPat < 1069 (f16 (uint_to_fp i32:$src)), 1070 (cvt_f16_f32_inst_e64 SRCMODS.NONE, (V_CVT_F32_U32_e32 VSrc_b32:$src)) 1071 >; 1072} 1073 1074let SubtargetPredicate = NotHasTrue16BitInsts in 1075defm : f16_fp_Pats<V_CVT_F16_F32_e64, V_CVT_F32_F16_e64>; 1076 1077let SubtargetPredicate = HasTrue16BitInsts in 1078defm : f16_fp_Pats<V_CVT_F16_F32_t16_e64, V_CVT_F32_F16_t16_e64>; 1079 1080//===----------------------------------------------------------------------===// 1081// VOP2 Patterns 1082//===----------------------------------------------------------------------===// 1083 1084// NoMods pattern used for mac. If there are any source modifiers then it's 1085// better to select mad instead of mac. 1086class FMADPat <ValueType vt, Instruction inst> 1087 : GCNPat <(vt (any_fmad (vt (VOP3NoMods vt:$src0)), 1088 (vt (VOP3NoMods vt:$src1)), 1089 (vt (VOP3NoMods vt:$src2)))), 1090 (inst SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 1091 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1092>; 1093 1094// Prefer mac form when there are no modifiers. 1095let AddedComplexity = 9 in { 1096let OtherPredicates = [HasMadMacF32Insts] in 1097def : FMADPat <f32, V_MAC_F32_e64>; 1098 1099// Don't allow source modifiers. If there are any source modifiers then it's 1100// better to select mad instead of mac. 1101let SubtargetPredicate = isGFX6GFX7GFX10, 1102 OtherPredicates = [HasMadMacF32Insts, NoFP32Denormals] in 1103def : GCNPat < 1104 (f32 (fadd (AMDGPUfmul_legacy (VOP3NoMods f32:$src0), 1105 (VOP3NoMods f32:$src1)), 1106 (VOP3NoMods f32:$src2))), 1107 (V_MAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 1108 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1109>; 1110 1111// Don't allow source modifiers. If there are any source modifiers then it's 1112// better to select fma instead of fmac. 1113let SubtargetPredicate = HasFmaLegacy32 in 1114def : GCNPat < 1115 (f32 (int_amdgcn_fma_legacy (VOP3NoMods f32:$src0), 1116 (VOP3NoMods f32:$src1), 1117 (VOP3NoMods f32:$src2))), 1118 (V_FMAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 1119 SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1120>; 1121 1122let SubtargetPredicate = Has16BitInsts in 1123def : FMADPat <f16, V_MAC_F16_e64>; 1124} // AddedComplexity = 9 1125 1126let OtherPredicates = [HasMadMacF32Insts, NoFP32Denormals] in 1127def : GCNPat < 1128 (f32 (fadd (AMDGPUfmul_legacy (VOP3Mods f32:$src0, i32:$src0_mod), 1129 (VOP3Mods f32:$src1, i32:$src1_mod)), 1130 (VOP3Mods f32:$src2, i32:$src2_mod))), 1131 (V_MAD_LEGACY_F32_e64 $src0_mod, $src0, $src1_mod, $src1, 1132 $src2_mod, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) 1133>; 1134 1135class VOPSelectModsPat <ValueType vt> : GCNPat < 1136 (vt (select i1:$src0, (VOP3ModsNonCanonicalizing vt:$src1, i32:$src1_mods), 1137 (VOP3ModsNonCanonicalizing vt:$src2, i32:$src2_mods))), 1138 (V_CNDMASK_B32_e64 FP32InputMods:$src2_mods, VSrc_b32:$src2, 1139 FP32InputMods:$src1_mods, VSrc_b32:$src1, SSrc_i1:$src0) 1140>; 1141 1142class VOPSelectPat <ValueType vt> : GCNPat < 1143 (vt (select i1:$src0, vt:$src1, vt:$src2)), 1144 (V_CNDMASK_B32_e64 0, VSrc_b32:$src2, 0, VSrc_b32:$src1, SSrc_i1:$src0) 1145>; 1146 1147def : VOPSelectModsPat <i32>; 1148def : VOPSelectModsPat <f32>; 1149def : VOPSelectPat <f16>; 1150def : VOPSelectPat <i16>; 1151 1152let AddedComplexity = 1 in { 1153def : GCNPat < 1154 (i32 (add (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)), i32:$val)), 1155 (V_BCNT_U32_B32_e64 $popcnt, $val) 1156>; 1157} 1158 1159def : GCNPat < 1160 (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)), 1161 (V_BCNT_U32_B32_e64 VSrc_b32:$popcnt, (i32 0)) 1162>; 1163 1164def : GCNPat < 1165 (i16 (add (i16 (trunc (i32 (DivergentUnaryFrag<ctpop> i32:$popcnt)))), i16:$val)), 1166 (V_BCNT_U32_B32_e64 $popcnt, $val) 1167>; 1168 1169def : GCNPat < 1170 (i64 (DivergentUnaryFrag<ctpop> i64:$src)), 1171 (REG_SEQUENCE VReg_64, 1172 (V_BCNT_U32_B32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub1)), 1173 (i32 (V_BCNT_U32_B32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0)))), sub0, 1174 (i32 (V_MOV_B32_e32 (i32 0))), sub1) 1175>; 1176 1177/********** ============================================ **********/ 1178/********** Extraction, Insertion, Building and Casting **********/ 1179/********** ============================================ **********/ 1180 1181// Special case for 2 element vectors. REQ_SEQUENCE produces better code 1182// than an INSERT_SUBREG. 1183multiclass Insert_Element_V2<RegisterClass RC, ValueType elem_type, ValueType vec_type> { 1184 def : GCNPat < 1185 (insertelt vec_type:$vec, elem_type:$elem, 0), 1186 (REG_SEQUENCE RC, $elem, sub0, (elem_type (EXTRACT_SUBREG $vec, sub1)), sub1) 1187 >; 1188 1189 def : GCNPat < 1190 (insertelt vec_type:$vec, elem_type:$elem, 1), 1191 (REG_SEQUENCE RC, (elem_type (EXTRACT_SUBREG $vec, sub0)), sub0, $elem, sub1) 1192 >; 1193} 1194 1195foreach Index = 0-1 in { 1196 def Extract_Element_v2i32_#Index : Extract_Element < 1197 i32, v2i32, Index, !cast<SubRegIndex>(sub#Index) 1198 >; 1199 1200 def Extract_Element_v2f32_#Index : Extract_Element < 1201 f32, v2f32, Index, !cast<SubRegIndex>(sub#Index) 1202 >; 1203} 1204 1205defm : Insert_Element_V2 <SReg_64, i32, v2i32>; 1206defm : Insert_Element_V2 <SReg_64, f32, v2f32>; 1207 1208foreach Index = 0-2 in { 1209 def Extract_Element_v3i32_#Index : Extract_Element < 1210 i32, v3i32, Index, !cast<SubRegIndex>(sub#Index) 1211 >; 1212 def Insert_Element_v3i32_#Index : Insert_Element < 1213 i32, v3i32, Index, !cast<SubRegIndex>(sub#Index) 1214 >; 1215 1216 def Extract_Element_v3f32_#Index : Extract_Element < 1217 f32, v3f32, Index, !cast<SubRegIndex>(sub#Index) 1218 >; 1219 def Insert_Element_v3f32_#Index : Insert_Element < 1220 f32, v3f32, Index, !cast<SubRegIndex>(sub#Index) 1221 >; 1222} 1223 1224foreach Index = 0-3 in { 1225 def Extract_Element_v4i32_#Index : Extract_Element < 1226 i32, v4i32, Index, !cast<SubRegIndex>(sub#Index) 1227 >; 1228 def Insert_Element_v4i32_#Index : Insert_Element < 1229 i32, v4i32, Index, !cast<SubRegIndex>(sub#Index) 1230 >; 1231 1232 def Extract_Element_v4f32_#Index : Extract_Element < 1233 f32, v4f32, Index, !cast<SubRegIndex>(sub#Index) 1234 >; 1235 def Insert_Element_v4f32_#Index : Insert_Element < 1236 f32, v4f32, Index, !cast<SubRegIndex>(sub#Index) 1237 >; 1238} 1239 1240foreach Index = 0-4 in { 1241 def Extract_Element_v5i32_#Index : Extract_Element < 1242 i32, v5i32, Index, !cast<SubRegIndex>(sub#Index) 1243 >; 1244 def Insert_Element_v5i32_#Index : Insert_Element < 1245 i32, v5i32, Index, !cast<SubRegIndex>(sub#Index) 1246 >; 1247 1248 def Extract_Element_v5f32_#Index : Extract_Element < 1249 f32, v5f32, Index, !cast<SubRegIndex>(sub#Index) 1250 >; 1251 def Insert_Element_v5f32_#Index : Insert_Element < 1252 f32, v5f32, Index, !cast<SubRegIndex>(sub#Index) 1253 >; 1254} 1255 1256foreach Index = 0-5 in { 1257 def Extract_Element_v6i32_#Index : Extract_Element < 1258 i32, v6i32, Index, !cast<SubRegIndex>(sub#Index) 1259 >; 1260 def Insert_Element_v6i32_#Index : Insert_Element < 1261 i32, v6i32, Index, !cast<SubRegIndex>(sub#Index) 1262 >; 1263 1264 def Extract_Element_v6f32_#Index : Extract_Element < 1265 f32, v6f32, Index, !cast<SubRegIndex>(sub#Index) 1266 >; 1267 def Insert_Element_v6f32_#Index : Insert_Element < 1268 f32, v6f32, Index, !cast<SubRegIndex>(sub#Index) 1269 >; 1270} 1271 1272foreach Index = 0-6 in { 1273 def Extract_Element_v7i32_#Index : Extract_Element < 1274 i32, v7i32, Index, !cast<SubRegIndex>(sub#Index) 1275 >; 1276 def Insert_Element_v7i32_#Index : Insert_Element < 1277 i32, v7i32, Index, !cast<SubRegIndex>(sub#Index) 1278 >; 1279 1280 def Extract_Element_v7f32_#Index : Extract_Element < 1281 f32, v7f32, Index, !cast<SubRegIndex>(sub#Index) 1282 >; 1283 def Insert_Element_v7f32_#Index : Insert_Element < 1284 f32, v7f32, Index, !cast<SubRegIndex>(sub#Index) 1285 >; 1286} 1287 1288foreach Index = 0-7 in { 1289 def Extract_Element_v8i32_#Index : Extract_Element < 1290 i32, v8i32, Index, !cast<SubRegIndex>(sub#Index) 1291 >; 1292 def Insert_Element_v8i32_#Index : Insert_Element < 1293 i32, v8i32, Index, !cast<SubRegIndex>(sub#Index) 1294 >; 1295 1296 def Extract_Element_v8f32_#Index : Extract_Element < 1297 f32, v8f32, Index, !cast<SubRegIndex>(sub#Index) 1298 >; 1299 def Insert_Element_v8f32_#Index : Insert_Element < 1300 f32, v8f32, Index, !cast<SubRegIndex>(sub#Index) 1301 >; 1302} 1303 1304foreach Index = 0-8 in { 1305 def Extract_Element_v9i32_#Index : Extract_Element < 1306 i32, v9i32, Index, !cast<SubRegIndex>(sub#Index) 1307 >; 1308 def Insert_Element_v9i32_#Index : Insert_Element < 1309 i32, v9i32, Index, !cast<SubRegIndex>(sub#Index) 1310 >; 1311 1312 def Extract_Element_v9f32_#Index : Extract_Element < 1313 f32, v9f32, Index, !cast<SubRegIndex>(sub#Index) 1314 >; 1315 def Insert_Element_v9f32_#Index : Insert_Element < 1316 f32, v9f32, Index, !cast<SubRegIndex>(sub#Index) 1317 >; 1318} 1319 1320foreach Index = 0-9 in { 1321 def Extract_Element_v10i32_#Index : Extract_Element < 1322 i32, v10i32, Index, !cast<SubRegIndex>(sub#Index) 1323 >; 1324 def Insert_Element_v10i32_#Index : Insert_Element < 1325 i32, v10i32, Index, !cast<SubRegIndex>(sub#Index) 1326 >; 1327 1328 def Extract_Element_v10f32_#Index : Extract_Element < 1329 f32, v10f32, Index, !cast<SubRegIndex>(sub#Index) 1330 >; 1331 def Insert_Element_v10f32_#Index : Insert_Element < 1332 f32, v10f32, Index, !cast<SubRegIndex>(sub#Index) 1333 >; 1334} 1335 1336foreach Index = 0-10 in { 1337 def Extract_Element_v11i32_#Index : Extract_Element < 1338 i32, v11i32, Index, !cast<SubRegIndex>(sub#Index) 1339 >; 1340 def Insert_Element_v11i32_#Index : Insert_Element < 1341 i32, v11i32, Index, !cast<SubRegIndex>(sub#Index) 1342 >; 1343 1344 def Extract_Element_v11f32_#Index : Extract_Element < 1345 f32, v11f32, Index, !cast<SubRegIndex>(sub#Index) 1346 >; 1347 def Insert_Element_v11f32_#Index : Insert_Element < 1348 f32, v11f32, Index, !cast<SubRegIndex>(sub#Index) 1349 >; 1350} 1351 1352foreach Index = 0-11 in { 1353 def Extract_Element_v12i32_#Index : Extract_Element < 1354 i32, v12i32, Index, !cast<SubRegIndex>(sub#Index) 1355 >; 1356 def Insert_Element_v12i32_#Index : Insert_Element < 1357 i32, v12i32, Index, !cast<SubRegIndex>(sub#Index) 1358 >; 1359 1360 def Extract_Element_v12f32_#Index : Extract_Element < 1361 f32, v12f32, Index, !cast<SubRegIndex>(sub#Index) 1362 >; 1363 def Insert_Element_v12f32_#Index : Insert_Element < 1364 f32, v12f32, Index, !cast<SubRegIndex>(sub#Index) 1365 >; 1366} 1367 1368foreach Index = 0-15 in { 1369 def Extract_Element_v16i32_#Index : Extract_Element < 1370 i32, v16i32, Index, !cast<SubRegIndex>(sub#Index) 1371 >; 1372 def Insert_Element_v16i32_#Index : Insert_Element < 1373 i32, v16i32, Index, !cast<SubRegIndex>(sub#Index) 1374 >; 1375 1376 def Extract_Element_v16f32_#Index : Extract_Element < 1377 f32, v16f32, Index, !cast<SubRegIndex>(sub#Index) 1378 >; 1379 def Insert_Element_v16f32_#Index : Insert_Element < 1380 f32, v16f32, Index, !cast<SubRegIndex>(sub#Index) 1381 >; 1382} 1383 1384 1385foreach Index = 0-31 in { 1386 def Extract_Element_v32i32_#Index : Extract_Element < 1387 i32, v32i32, Index, !cast<SubRegIndex>(sub#Index) 1388 >; 1389 1390 def Insert_Element_v32i32_#Index : Insert_Element < 1391 i32, v32i32, Index, !cast<SubRegIndex>(sub#Index) 1392 >; 1393 1394 def Extract_Element_v32f32_#Index : Extract_Element < 1395 f32, v32f32, Index, !cast<SubRegIndex>(sub#Index) 1396 >; 1397 1398 def Insert_Element_v32f32_#Index : Insert_Element < 1399 f32, v32f32, Index, !cast<SubRegIndex>(sub#Index) 1400 >; 1401} 1402 1403// FIXME: Why do only some of these type combinations for SReg and 1404// VReg? 1405// 16-bit bitcast 1406def : BitConvert <i16, f16, VGPR_32>; 1407def : BitConvert <f16, i16, VGPR_32>; 1408def : BitConvert <i16, f16, SReg_32>; 1409def : BitConvert <f16, i16, SReg_32>; 1410 1411// 32-bit bitcast 1412def : BitConvert <i32, f32, VGPR_32>; 1413def : BitConvert <f32, i32, VGPR_32>; 1414def : BitConvert <i32, f32, SReg_32>; 1415def : BitConvert <f32, i32, SReg_32>; 1416def : BitConvert <v2i16, i32, SReg_32>; 1417def : BitConvert <i32, v2i16, SReg_32>; 1418def : BitConvert <v2f16, i32, SReg_32>; 1419def : BitConvert <i32, v2f16, SReg_32>; 1420def : BitConvert <v2i16, v2f16, SReg_32>; 1421def : BitConvert <v2f16, v2i16, SReg_32>; 1422def : BitConvert <v2f16, f32, SReg_32>; 1423def : BitConvert <f32, v2f16, SReg_32>; 1424def : BitConvert <v2i16, f32, SReg_32>; 1425def : BitConvert <f32, v2i16, SReg_32>; 1426 1427// 64-bit bitcast 1428def : BitConvert <i64, f64, VReg_64>; 1429def : BitConvert <f64, i64, VReg_64>; 1430def : BitConvert <v2i32, v2f32, VReg_64>; 1431def : BitConvert <v2f32, v2i32, VReg_64>; 1432def : BitConvert <i64, v2i32, VReg_64>; 1433def : BitConvert <v2i32, i64, VReg_64>; 1434def : BitConvert <i64, v2f32, VReg_64>; 1435def : BitConvert <v2f32, i64, VReg_64>; 1436def : BitConvert <f64, v2f32, VReg_64>; 1437def : BitConvert <v2f32, f64, VReg_64>; 1438def : BitConvert <f64, v2i32, VReg_64>; 1439def : BitConvert <v2i32, f64, VReg_64>; 1440def : BitConvert <v4i16, v4f16, VReg_64>; 1441def : BitConvert <v4f16, v4i16, VReg_64>; 1442 1443// FIXME: Make SGPR 1444def : BitConvert <v2i32, v4f16, VReg_64>; 1445def : BitConvert <v4f16, v2i32, VReg_64>; 1446def : BitConvert <v2i32, v4f16, VReg_64>; 1447def : BitConvert <v2i32, v4i16, VReg_64>; 1448def : BitConvert <v4i16, v2i32, VReg_64>; 1449def : BitConvert <v2f32, v4f16, VReg_64>; 1450def : BitConvert <v4f16, v2f32, VReg_64>; 1451def : BitConvert <v2f32, v4i16, VReg_64>; 1452def : BitConvert <v4i16, v2f32, VReg_64>; 1453def : BitConvert <v4i16, f64, VReg_64>; 1454def : BitConvert <v4f16, f64, VReg_64>; 1455def : BitConvert <f64, v4i16, VReg_64>; 1456def : BitConvert <f64, v4f16, VReg_64>; 1457def : BitConvert <v4i16, i64, VReg_64>; 1458def : BitConvert <v4f16, i64, VReg_64>; 1459def : BitConvert <i64, v4i16, VReg_64>; 1460def : BitConvert <i64, v4f16, VReg_64>; 1461 1462def : BitConvert <v4i32, v4f32, VReg_128>; 1463def : BitConvert <v4f32, v4i32, VReg_128>; 1464 1465// 96-bit bitcast 1466def : BitConvert <v3i32, v3f32, SGPR_96>; 1467def : BitConvert <v3f32, v3i32, SGPR_96>; 1468 1469// 128-bit bitcast 1470def : BitConvert <v2i64, v4i32, SReg_128>; 1471def : BitConvert <v4i32, v2i64, SReg_128>; 1472def : BitConvert <v2f64, v4f32, VReg_128>; 1473def : BitConvert <v2f64, v4i32, VReg_128>; 1474def : BitConvert <v4f32, v2f64, VReg_128>; 1475def : BitConvert <v4i32, v2f64, VReg_128>; 1476def : BitConvert <v2i64, v2f64, VReg_128>; 1477def : BitConvert <v2f64, v2i64, VReg_128>; 1478def : BitConvert <v4f32, v2i64, VReg_128>; 1479def : BitConvert <v2i64, v4f32, VReg_128>; 1480def : BitConvert <v8i16, v4i32, SReg_128>; 1481def : BitConvert <v4i32, v8i16, SReg_128>; 1482def : BitConvert <v8f16, v4f32, VReg_128>; 1483def : BitConvert <v8f16, v4i32, VReg_128>; 1484def : BitConvert <v4f32, v8f16, VReg_128>; 1485def : BitConvert <v4i32, v8f16, VReg_128>; 1486def : BitConvert <v8i16, v8f16, VReg_128>; 1487def : BitConvert <v8f16, v8i16, VReg_128>; 1488def : BitConvert <v4f32, v8i16, VReg_128>; 1489def : BitConvert <v8i16, v4f32, VReg_128>; 1490def : BitConvert <v8i16, v8f16, SReg_128>; 1491def : BitConvert <v8i16, v2i64, SReg_128>; 1492def : BitConvert <v8i16, v2f64, SReg_128>; 1493def : BitConvert <v8f16, v2i64, SReg_128>; 1494def : BitConvert <v8f16, v2f64, SReg_128>; 1495def : BitConvert <v8f16, v8i16, SReg_128>; 1496def : BitConvert <v2i64, v8i16, SReg_128>; 1497def : BitConvert <v2f64, v8i16, SReg_128>; 1498def : BitConvert <v2i64, v8f16, SReg_128>; 1499def : BitConvert <v2f64, v8f16, SReg_128>; 1500 1501// 160-bit bitcast 1502def : BitConvert <v5i32, v5f32, SReg_160>; 1503def : BitConvert <v5f32, v5i32, SReg_160>; 1504def : BitConvert <v5i32, v5f32, VReg_160>; 1505def : BitConvert <v5f32, v5i32, VReg_160>; 1506 1507// 192-bit bitcast 1508def : BitConvert <v6i32, v6f32, SReg_192>; 1509def : BitConvert <v6f32, v6i32, SReg_192>; 1510def : BitConvert <v6i32, v6f32, VReg_192>; 1511def : BitConvert <v6f32, v6i32, VReg_192>; 1512def : BitConvert <v3i64, v3f64, VReg_192>; 1513def : BitConvert <v3f64, v3i64, VReg_192>; 1514def : BitConvert <v3i64, v6i32, VReg_192>; 1515def : BitConvert <v3i64, v6f32, VReg_192>; 1516def : BitConvert <v3f64, v6i32, VReg_192>; 1517def : BitConvert <v3f64, v6f32, VReg_192>; 1518def : BitConvert <v6i32, v3i64, VReg_192>; 1519def : BitConvert <v6f32, v3i64, VReg_192>; 1520def : BitConvert <v6i32, v3f64, VReg_192>; 1521def : BitConvert <v6f32, v3f64, VReg_192>; 1522 1523// 224-bit bitcast 1524def : BitConvert <v7i32, v7f32, SReg_224>; 1525def : BitConvert <v7f32, v7i32, SReg_224>; 1526def : BitConvert <v7i32, v7f32, VReg_224>; 1527def : BitConvert <v7f32, v7i32, VReg_224>; 1528 1529// 256-bit bitcast 1530def : BitConvert <v8i32, v8f32, SReg_256>; 1531def : BitConvert <v8f32, v8i32, SReg_256>; 1532def : BitConvert <v8i32, v8f32, VReg_256>; 1533def : BitConvert <v8f32, v8i32, VReg_256>; 1534def : BitConvert <v4i64, v4f64, VReg_256>; 1535def : BitConvert <v4f64, v4i64, VReg_256>; 1536def : BitConvert <v4i64, v8i32, VReg_256>; 1537def : BitConvert <v4i64, v8f32, VReg_256>; 1538def : BitConvert <v4f64, v8i32, VReg_256>; 1539def : BitConvert <v4f64, v8f32, VReg_256>; 1540def : BitConvert <v8i32, v4i64, VReg_256>; 1541def : BitConvert <v8f32, v4i64, VReg_256>; 1542def : BitConvert <v8i32, v4f64, VReg_256>; 1543def : BitConvert <v8f32, v4f64, VReg_256>; 1544def : BitConvert <v16i16, v16f16, SReg_256>; 1545def : BitConvert <v16f16, v16i16, SReg_256>; 1546def : BitConvert <v16i16, v16f16, VReg_256>; 1547def : BitConvert <v16f16, v16i16, VReg_256>; 1548def : BitConvert <v16f16, v8i32, VReg_256>; 1549def : BitConvert <v16i16, v8i32, VReg_256>; 1550def : BitConvert <v16f16, v8f32, VReg_256>; 1551def : BitConvert <v16i16, v8f32, VReg_256>; 1552def : BitConvert <v8i32, v16f16, VReg_256>; 1553def : BitConvert <v8i32, v16i16, VReg_256>; 1554def : BitConvert <v8f32, v16f16, VReg_256>; 1555def : BitConvert <v8f32, v16i16, VReg_256>; 1556def : BitConvert <v16f16, v4i64, VReg_256>; 1557def : BitConvert <v16i16, v4i64, VReg_256>; 1558def : BitConvert <v16f16, v4f64, VReg_256>; 1559def : BitConvert <v16i16, v4f64, VReg_256>; 1560def : BitConvert <v4i64, v16f16, VReg_256>; 1561def : BitConvert <v4i64, v16i16, VReg_256>; 1562def : BitConvert <v4f64, v16f16, VReg_256>; 1563def : BitConvert <v4f64, v16i16, VReg_256>; 1564 1565// 288-bit bitcast 1566def : BitConvert <v9i32, v9f32, SReg_288>; 1567def : BitConvert <v9f32, v9i32, SReg_288>; 1568def : BitConvert <v9i32, v9f32, VReg_288>; 1569def : BitConvert <v9f32, v9i32, VReg_288>; 1570 1571// 320-bit bitcast 1572def : BitConvert <v10i32, v10f32, SReg_320>; 1573def : BitConvert <v10f32, v10i32, SReg_320>; 1574def : BitConvert <v10i32, v10f32, VReg_320>; 1575def : BitConvert <v10f32, v10i32, VReg_320>; 1576 1577// 320-bit bitcast 1578def : BitConvert <v11i32, v11f32, SReg_352>; 1579def : BitConvert <v11f32, v11i32, SReg_352>; 1580def : BitConvert <v11i32, v11f32, VReg_352>; 1581def : BitConvert <v11f32, v11i32, VReg_352>; 1582 1583// 384-bit bitcast 1584def : BitConvert <v12i32, v12f32, SReg_384>; 1585def : BitConvert <v12f32, v12i32, SReg_384>; 1586def : BitConvert <v12i32, v12f32, VReg_384>; 1587def : BitConvert <v12f32, v12i32, VReg_384>; 1588 1589// 512-bit bitcast 1590def : BitConvert <v16i32, v16f32, VReg_512>; 1591def : BitConvert <v16f32, v16i32, VReg_512>; 1592def : BitConvert <v8i64, v8f64, VReg_512>; 1593def : BitConvert <v8f64, v8i64, VReg_512>; 1594def : BitConvert <v8i64, v16i32, VReg_512>; 1595def : BitConvert <v8f64, v16i32, VReg_512>; 1596def : BitConvert <v16i32, v8i64, VReg_512>; 1597def : BitConvert <v16i32, v8f64, VReg_512>; 1598def : BitConvert <v8i64, v16f32, VReg_512>; 1599def : BitConvert <v8f64, v16f32, VReg_512>; 1600def : BitConvert <v16f32, v8i64, VReg_512>; 1601def : BitConvert <v16f32, v8f64, VReg_512>; 1602 1603// 1024-bit bitcast 1604def : BitConvert <v32i32, v32f32, VReg_1024>; 1605def : BitConvert <v32f32, v32i32, VReg_1024>; 1606def : BitConvert <v16i64, v16f64, VReg_1024>; 1607def : BitConvert <v16f64, v16i64, VReg_1024>; 1608def : BitConvert <v16i64, v32i32, VReg_1024>; 1609def : BitConvert <v32i32, v16i64, VReg_1024>; 1610def : BitConvert <v16f64, v32f32, VReg_1024>; 1611def : BitConvert <v32f32, v16f64, VReg_1024>; 1612def : BitConvert <v16i64, v32f32, VReg_1024>; 1613def : BitConvert <v32i32, v16f64, VReg_1024>; 1614def : BitConvert <v16f64, v32i32, VReg_1024>; 1615def : BitConvert <v32f32, v16i64, VReg_1024>; 1616 1617 1618/********** =================== **********/ 1619/********** Src & Dst modifiers **********/ 1620/********** =================== **********/ 1621 1622 1623// If denormals are not enabled, it only impacts the compare of the 1624// inputs. The output result is not flushed. 1625class ClampPat<Instruction inst, ValueType vt> : GCNPat < 1626 (vt (AMDGPUclamp (VOP3Mods vt:$src0, i32:$src0_modifiers))), 1627 (inst i32:$src0_modifiers, vt:$src0, 1628 i32:$src0_modifiers, vt:$src0, DSTCLAMP.ENABLE, DSTOMOD.NONE) 1629>; 1630 1631def : ClampPat<V_MAX_F32_e64, f32>; 1632def : ClampPat<V_MAX_F64_e64, f64>; 1633let SubtargetPredicate = NotHasTrue16BitInsts in 1634def : ClampPat<V_MAX_F16_e64, f16>; 1635let SubtargetPredicate = HasTrue16BitInsts in 1636def : ClampPat<V_MAX_F16_t16_e64, f16>; 1637 1638let SubtargetPredicate = HasVOP3PInsts in { 1639def : GCNPat < 1640 (v2f16 (AMDGPUclamp (VOP3PMods v2f16:$src0, i32:$src0_modifiers))), 1641 (V_PK_MAX_F16 $src0_modifiers, $src0, 1642 $src0_modifiers, $src0, DSTCLAMP.ENABLE) 1643>; 1644} 1645 1646 1647/********** ================================ **********/ 1648/********** Floating point absolute/negative **********/ 1649/********** ================================ **********/ 1650 1651def : GCNPat < 1652 (UniformUnaryFrag<fneg> (fabs (f32 SReg_32:$src))), 1653 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000))) // Set sign bit 1654>; 1655 1656def : GCNPat < 1657 (UniformUnaryFrag<fabs> (f32 SReg_32:$src)), 1658 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fffffff))) 1659>; 1660 1661def : GCNPat < 1662 (UniformUnaryFrag<fneg> (f32 SReg_32:$src)), 1663 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000))) 1664>; 1665 1666def : GCNPat < 1667 (UniformUnaryFrag<fneg> (f16 SReg_32:$src)), 1668 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) 1669>; 1670 1671def : GCNPat < 1672 (UniformUnaryFrag<fabs> (f16 SReg_32:$src)), 1673 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00007fff))) 1674>; 1675 1676def : GCNPat < 1677 (UniformUnaryFrag<fneg> (fabs (f16 SReg_32:$src))), 1678 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) // Set sign bit 1679>; 1680 1681def : GCNPat < 1682 (UniformUnaryFrag<fneg> (v2f16 SReg_32:$src)), 1683 (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) 1684>; 1685 1686def : GCNPat < 1687 (UniformUnaryFrag<fabs> (v2f16 SReg_32:$src)), 1688 (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x7fff7fff))) 1689>; 1690 1691// This is really (fneg (fabs v2f16:$src)) 1692// 1693// fabs is not reported as free because there is modifier for it in 1694// VOP3P instructions, so it is turned into the bit op. 1695def : GCNPat < 1696 (UniformUnaryFrag<fneg> (v2f16 (bitconvert (and_oneuse (i32 SReg_32:$src), 0x7fff7fff)))), 1697 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit 1698>; 1699 1700def : GCNPat < 1701 (UniformUnaryFrag<fneg> (v2f16 (fabs SReg_32:$src))), 1702 (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000))) // Set sign bit 1703>; 1704 1705 1706// COPY_TO_REGCLASS is needed to avoid using SCC from S_XOR_B32 instead 1707// of the real value. 1708def : GCNPat < 1709 (UniformUnaryFrag<fneg> (v2f32 SReg_64:$src)), 1710 (v2f32 (REG_SEQUENCE SReg_64, 1711 (f32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub0)), 1712 (i32 (S_MOV_B32 (i32 0x80000000)))), 1713 SReg_32)), sub0, 1714 (f32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG $src, sub1)), 1715 (i32 (S_MOV_B32 (i32 0x80000000)))), 1716 SReg_32)), sub1)) 1717>; 1718 1719def : GCNPat < 1720 (UniformUnaryFrag<fabs> (v2f32 SReg_64:$src)), 1721 (v2f32 (REG_SEQUENCE SReg_64, 1722 (f32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG $src, sub0)), 1723 (i32 (S_MOV_B32 (i32 0x7fffffff)))), 1724 SReg_32)), sub0, 1725 (f32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG $src, sub1)), 1726 (i32 (S_MOV_B32 (i32 0x7fffffff)))), 1727 SReg_32)), sub1)) 1728>; 1729 1730def : GCNPat < 1731 (UniformUnaryFrag<fneg> (fabs (v2f32 SReg_64:$src))), 1732 (v2f32 (REG_SEQUENCE SReg_64, 1733 (f32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG $src, sub0)), 1734 (i32 (S_MOV_B32 (i32 0x80000000)))), 1735 SReg_32)), sub0, 1736 (f32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG $src, sub1)), 1737 (i32 (S_MOV_B32 (i32 0x80000000)))), 1738 SReg_32)), sub1)) 1739>; 1740 1741// FIXME: Use S_BITSET0_B32/B64? 1742def : GCNPat < 1743 (UniformUnaryFrag<fabs> (f64 SReg_64:$src)), 1744 (REG_SEQUENCE SReg_64, 1745 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)), 1746 sub0, 1747 (i32 (COPY_TO_REGCLASS (S_AND_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)), 1748 (S_MOV_B32 (i32 0x7fffffff))), SReg_32)), // Set sign bit. 1749 sub1) 1750>; 1751 1752def : GCNPat < 1753 (UniformUnaryFrag<fneg> (f64 SReg_64:$src)), 1754 (REG_SEQUENCE SReg_64, 1755 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)), 1756 sub0, 1757 (i32 (COPY_TO_REGCLASS (S_XOR_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)), 1758 (i32 (S_MOV_B32 (i32 0x80000000)))), SReg_32)), 1759 sub1) 1760>; 1761 1762def : GCNPat < 1763 (UniformUnaryFrag<fneg> (fabs (f64 SReg_64:$src))), 1764 (REG_SEQUENCE SReg_64, 1765 (i32 (EXTRACT_SUBREG SReg_64:$src, sub0)), 1766 sub0, 1767 (i32 (COPY_TO_REGCLASS (S_OR_B32 (i32 (EXTRACT_SUBREG SReg_64:$src, sub1)), 1768 (S_MOV_B32 (i32 0x80000000))), SReg_32)),// Set sign bit. 1769 sub1) 1770>; 1771 1772 1773def : GCNPat < 1774 (fneg (fabs (f32 VGPR_32:$src))), 1775 (V_OR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src) // Set sign bit 1776>; 1777 1778def : GCNPat < 1779 (fabs (f32 VGPR_32:$src)), 1780 (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), VGPR_32:$src) 1781>; 1782 1783def : GCNPat < 1784 (fneg (f32 VGPR_32:$src)), 1785 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src) 1786>; 1787 1788def : GCNPat < 1789 (fabs (f16 VGPR_32:$src)), 1790 (V_AND_B32_e64 (S_MOV_B32 (i32 0x00007fff)), VGPR_32:$src) 1791>; 1792 1793def : GCNPat < 1794 (fneg (f16 VGPR_32:$src)), 1795 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) 1796>; 1797 1798def : GCNPat < 1799 (fneg (fabs (f16 VGPR_32:$src))), 1800 (V_OR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) // Set sign bit 1801>; 1802 1803def : GCNPat < 1804 (fneg (v2f16 VGPR_32:$src)), 1805 (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src) 1806>; 1807 1808def : GCNPat < 1809 (fabs (v2f16 VGPR_32:$src)), 1810 (V_AND_B32_e64 (S_MOV_B32 (i32 0x7fff7fff)), VGPR_32:$src) 1811>; 1812 1813def : GCNPat < 1814 (fneg (v2f16 (fabs VGPR_32:$src))), 1815 (V_OR_B32_e64 (S_MOV_B32 (i32 0x80008000)), VGPR_32:$src) 1816>; 1817 1818def : GCNPat < 1819 (fabs (f64 VReg_64:$src)), 1820 (REG_SEQUENCE VReg_64, 1821 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)), 1822 sub0, 1823 (V_AND_B32_e64 (i32 (S_MOV_B32 (i32 0x7fffffff))), 1824 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))), 1825 sub1) 1826>; 1827 1828def : GCNPat < 1829 (fneg (f64 VReg_64:$src)), 1830 (REG_SEQUENCE VReg_64, 1831 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)), 1832 sub0, 1833 (V_XOR_B32_e64 (i32 (S_MOV_B32 (i32 0x80000000))), 1834 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))), 1835 sub1) 1836>; 1837 1838def : GCNPat < 1839 (fneg (fabs (f64 VReg_64:$src))), 1840 (REG_SEQUENCE VReg_64, 1841 (i32 (EXTRACT_SUBREG VReg_64:$src, sub0)), 1842 sub0, 1843 (V_OR_B32_e64 (i32 (S_MOV_B32 (i32 0x80000000))), 1844 (i32 (EXTRACT_SUBREG VReg_64:$src, sub1))), 1845 sub1) 1846>; 1847 1848def : GCNPat < 1849 (DivergentUnaryFrag<fneg> (v2f32 VReg_64:$src)), 1850 (V_PK_ADD_F32 11 /* OP_SEL_1 | NEG_LO | HEG_HI */, VReg_64:$src, 1851 11 /* OP_SEL_1 | NEG_LO | HEG_HI */, 0, 1852 0, 0, 0, 0, 0) 1853> { 1854 let SubtargetPredicate = HasPackedFP32Ops; 1855} 1856 1857def : GCNPat < 1858 (fcopysign f16:$src0, f16:$src1), 1859 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, $src1) 1860>; 1861 1862def : GCNPat < 1863 (fcopysign f32:$src0, f16:$src1), 1864 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, 1865 (V_LSHLREV_B32_e64 (i32 16), $src1)) 1866>; 1867 1868def : GCNPat < 1869 (fcopysign f64:$src0, f16:$src1), 1870 (REG_SEQUENCE SReg_64, 1871 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 1872 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)), 1873 (V_LSHLREV_B32_e64 (i32 16), $src1)), sub1) 1874>; 1875 1876def : GCNPat < 1877 (fcopysign f16:$src0, f32:$src1), 1878 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, 1879 (V_LSHRREV_B32_e64 (i32 16), $src1)) 1880>; 1881 1882def : GCNPat < 1883 (fcopysign f16:$src0, f64:$src1), 1884 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, 1885 (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1))) 1886>; 1887 1888/********** ================== **********/ 1889/********** Immediate Patterns **********/ 1890/********** ================== **********/ 1891 1892def : GCNPat < 1893 (VGPRImm<(i32 imm)>:$imm), 1894 (V_MOV_B32_e32 imm:$imm) 1895>; 1896 1897def : GCNPat < 1898 (VGPRImm<(f32 fpimm)>:$imm), 1899 (V_MOV_B32_e32 (f32 (bitcast_fpimm_to_i32 $imm))) 1900>; 1901 1902def : GCNPat < 1903 (i32 imm:$imm), 1904 (S_MOV_B32 imm:$imm) 1905>; 1906 1907def : GCNPat < 1908 (VGPRImm<(SIlds tglobaladdr:$ga)>), 1909 (V_MOV_B32_e32 $ga) 1910>; 1911 1912def : GCNPat < 1913 (SIlds tglobaladdr:$ga), 1914 (S_MOV_B32 $ga) 1915>; 1916 1917// FIXME: Workaround for ordering issue with peephole optimizer where 1918// a register class copy interferes with immediate folding. Should 1919// use s_mov_b32, which can be shrunk to s_movk_i32 1920def : GCNPat < 1921 (VGPRImm<(f16 fpimm)>:$imm), 1922 (V_MOV_B32_e32 (f16 (bitcast_fpimm_to_i32 $imm))) 1923>; 1924 1925def : GCNPat < 1926 (f32 fpimm:$imm), 1927 (S_MOV_B32 (f32 (bitcast_fpimm_to_i32 $imm))) 1928>; 1929 1930def : GCNPat < 1931 (f16 fpimm:$imm), 1932 (S_MOV_B32 (i32 (bitcast_fpimm_to_i32 $imm))) 1933>; 1934 1935def : GCNPat < 1936 (p5 frameindex:$fi), 1937 (V_MOV_B32_e32 (p5 (frameindex_to_targetframeindex $fi))) 1938>; 1939 1940def : GCNPat < 1941 (p5 frameindex:$fi), 1942 (S_MOV_B32 (p5 (frameindex_to_targetframeindex $fi))) 1943>; 1944 1945def : GCNPat < 1946 (i64 InlineImm64:$imm), 1947 (S_MOV_B64 InlineImm64:$imm) 1948>; 1949 1950// XXX - Should this use a s_cmp to set SCC? 1951 1952// Set to sign-extended 64-bit value (true = -1, false = 0) 1953def : GCNPat < 1954 (i1 imm:$imm), 1955 (S_MOV_B64 (i64 (as_i64imm $imm))) 1956> { 1957 let WaveSizePredicate = isWave64; 1958} 1959 1960def : GCNPat < 1961 (i1 imm:$imm), 1962 (S_MOV_B32 (i32 (as_i32imm $imm))) 1963> { 1964 let WaveSizePredicate = isWave32; 1965} 1966 1967def : GCNPat < 1968 (f64 InlineImmFP64:$imm), 1969 (S_MOV_B64 (f64 (bitcast_fpimm_to_i64 InlineImmFP64:$imm))) 1970>; 1971 1972/********** ================== **********/ 1973/********** Intrinsic Patterns **********/ 1974/********** ================== **********/ 1975 1976def : GCNPat < 1977 (f32 (fpow (VOP3Mods f32:$src0, i32:$src0_mods), (VOP3Mods f32:$src1, i32:$src1_mods))), 1978 (V_EXP_F32_e64 SRCMODS.NONE, (V_MUL_LEGACY_F32_e64 $src1_mods, $src1, SRCMODS.NONE, (V_LOG_F32_e64 $src0_mods, $src0), 0, 0)) 1979>; 1980 1981def : GCNPat < 1982 (i32 (sext i1:$src0)), 1983 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 1984 /*src1mod*/(i32 0), /*src1*/(i32 -1), i1:$src0) 1985>; 1986 1987class Ext32Pat <SDNode ext> : GCNPat < 1988 (i32 (ext i1:$src0)), 1989 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 1990 /*src1mod*/(i32 0), /*src1*/(i32 1), i1:$src0) 1991>; 1992 1993def : Ext32Pat <zext>; 1994def : Ext32Pat <anyext>; 1995 1996// The multiplication scales from [0,1) to the unsigned integer range, 1997// rounding down a bit to avoid unwanted overflow. 1998def : GCNPat < 1999 (AMDGPUurecip i32:$src0), 2000 (V_CVT_U32_F32_e32 2001 (V_MUL_F32_e32 (i32 CONST.FP_4294966784), 2002 (V_RCP_IFLAG_F32_e32 (V_CVT_F32_U32_e32 $src0)))) 2003>; 2004 2005//===----------------------------------------------------------------------===// 2006// VOP3 Patterns 2007//===----------------------------------------------------------------------===// 2008 2009def : IMad24Pat<V_MAD_I32_I24_e64, 1>; 2010def : UMad24Pat<V_MAD_U32_U24_e64, 1>; 2011 2012// BFI patterns 2013 2014def BFIImm32 : PatFrag< 2015 (ops node:$x, node:$y, node:$z), 2016 (i32 (DivergentBinFrag<or> (and node:$y, node:$x), (and node:$z, imm))), 2017 [{ 2018 auto *X = dyn_cast<ConstantSDNode>(N->getOperand(0)->getOperand(1)); 2019 auto *NotX = dyn_cast<ConstantSDNode>(N->getOperand(1)->getOperand(1)); 2020 return X && NotX && 2021 ~(unsigned)X->getZExtValue() == (unsigned)NotX->getZExtValue(); 2022 }] 2023>; 2024 2025 2026// Definition from ISA doc: 2027// (y & x) | (z & ~x) 2028def : AMDGPUPatIgnoreCopies < 2029 (DivergentBinFrag<or> (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))), 2030 (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), 2031 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), 2032 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) 2033>; 2034 2035// (y & C) | (z & ~C) 2036def : AMDGPUPatIgnoreCopies < 2037 (BFIImm32 i32:$x, i32:$y, i32:$z), 2038 (V_BFI_B32_e64 VSrc_b32:$x, VSrc_b32:$y, VSrc_b32:$z) 2039>; 2040 2041// 64-bit version 2042def : AMDGPUPatIgnoreCopies < 2043 (DivergentBinFrag<or> (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))), 2044 (REG_SEQUENCE VReg_64, 2045 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), 2046 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), 2047 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, 2048 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), 2049 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), 2050 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) 2051>; 2052 2053// SHA-256 Ch function 2054// z ^ (x & (y ^ z)) 2055def : AMDGPUPatIgnoreCopies < 2056 (DivergentBinFrag<xor> i32:$z, (and i32:$x, (xor i32:$y, i32:$z))), 2057 (V_BFI_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), 2058 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32), 2059 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) 2060>; 2061 2062// 64-bit version 2063def : AMDGPUPatIgnoreCopies < 2064 (DivergentBinFrag<xor> i64:$z, (and i64:$x, (xor i64:$y, i64:$z))), 2065 (REG_SEQUENCE VReg_64, 2066 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), 2067 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), 2068 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0))), sub0, 2069 (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), 2070 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), 2071 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) 2072>; 2073 2074def : AMDGPUPat < 2075 (fcopysign f32:$src0, f32:$src1), 2076 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, $src1) 2077>; 2078 2079def : AMDGPUPat < 2080 (fcopysign f32:$src0, f64:$src1), 2081 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, 2082 (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1))) 2083>; 2084 2085def : AMDGPUPat < 2086 (fcopysign f64:$src0, f64:$src1), 2087 (REG_SEQUENCE SReg_64, 2088 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 2089 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), 2090 (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)), 2091 (i32 (EXTRACT_SUBREG SReg_64:$src1, sub1))), sub1) 2092>; 2093 2094def : AMDGPUPat < 2095 (fcopysign f64:$src0, f32:$src1), 2096 (REG_SEQUENCE SReg_64, 2097 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, 2098 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), 2099 (i32 (EXTRACT_SUBREG SReg_64:$src0, sub1)), 2100 $src1), sub1) 2101>; 2102 2103def : ROTRPattern <V_ALIGNBIT_B32_e64>; 2104 2105def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))), 2106 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)), 2107 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>; 2108 2109def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))), 2110 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)), 2111 (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>; 2112 2113/********** ====================== **********/ 2114/********** Indirect addressing **********/ 2115/********** ====================== **********/ 2116 2117multiclass SI_INDIRECT_Pattern <ValueType vt, ValueType eltvt, string VecSize> { 2118 // Extract with offset 2119 def : GCNPat< 2120 (eltvt (extractelt vt:$src, (MOVRELOffset i32:$idx, (i32 imm:$offset)))), 2121 (!cast<Instruction>("SI_INDIRECT_SRC_"#VecSize) $src, $idx, imm:$offset) 2122 >; 2123 2124 // Insert with offset 2125 def : GCNPat< 2126 (insertelt vt:$src, eltvt:$val, (MOVRELOffset i32:$idx, (i32 imm:$offset))), 2127 (!cast<Instruction>("SI_INDIRECT_DST_"#VecSize) $src, $idx, imm:$offset, $val) 2128 >; 2129} 2130 2131defm : SI_INDIRECT_Pattern <v2f32, f32, "V2">; 2132defm : SI_INDIRECT_Pattern <v4f32, f32, "V4">; 2133defm : SI_INDIRECT_Pattern <v8f32, f32, "V8">; 2134defm : SI_INDIRECT_Pattern <v9f32, f32, "V9">; 2135defm : SI_INDIRECT_Pattern <v10f32, f32, "V10">; 2136defm : SI_INDIRECT_Pattern <v11f32, f32, "V11">; 2137defm : SI_INDIRECT_Pattern <v12f32, f32, "V12">; 2138defm : SI_INDIRECT_Pattern <v16f32, f32, "V16">; 2139defm : SI_INDIRECT_Pattern <v32f32, f32, "V32">; 2140 2141defm : SI_INDIRECT_Pattern <v2i32, i32, "V2">; 2142defm : SI_INDIRECT_Pattern <v4i32, i32, "V4">; 2143defm : SI_INDIRECT_Pattern <v8i32, i32, "V8">; 2144defm : SI_INDIRECT_Pattern <v9i32, i32, "V9">; 2145defm : SI_INDIRECT_Pattern <v10i32, i32, "V10">; 2146defm : SI_INDIRECT_Pattern <v11i32, i32, "V11">; 2147defm : SI_INDIRECT_Pattern <v12i32, i32, "V12">; 2148defm : SI_INDIRECT_Pattern <v16i32, i32, "V16">; 2149defm : SI_INDIRECT_Pattern <v32i32, i32, "V32">; 2150 2151//===----------------------------------------------------------------------===// 2152// SAD Patterns 2153//===----------------------------------------------------------------------===// 2154 2155def : GCNPat < 2156 (add (sub_oneuse (umax i32:$src0, i32:$src1), 2157 (umin i32:$src0, i32:$src1)), 2158 i32:$src2), 2159 (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0)) 2160>; 2161 2162def : GCNPat < 2163 (add (select_oneuse (i1 (setugt i32:$src0, i32:$src1)), 2164 (sub i32:$src0, i32:$src1), 2165 (sub i32:$src1, i32:$src0)), 2166 i32:$src2), 2167 (V_SAD_U32_e64 $src0, $src1, $src2, (i1 0)) 2168>; 2169 2170//===----------------------------------------------------------------------===// 2171// Conversion Patterns 2172//===----------------------------------------------------------------------===// 2173def : GCNPat<(i32 (UniformSextInreg<i1> i32:$src)), 2174 (S_BFE_I32 i32:$src, (i32 65536))>; // 0 | 1 << 16 2175 2176// Handle sext_inreg in i64 2177def : GCNPat < 2178 (i64 (UniformSextInreg<i1> i64:$src)), 2179 (S_BFE_I64 i64:$src, (i32 0x10000)) // 0 | 1 << 16 2180>; 2181 2182def : GCNPat < 2183 (i16 (UniformSextInreg<i1> i16:$src)), 2184 (S_BFE_I32 $src, (i32 0x00010000)) // 0 | 1 << 16 2185>; 2186 2187def : GCNPat < 2188 (i16 (UniformSextInreg<i8> i16:$src)), 2189 (S_BFE_I32 $src, (i32 0x80000)) // 0 | 8 << 16 2190>; 2191 2192def : GCNPat < 2193 (i64 (UniformSextInreg<i8> i64:$src)), 2194 (S_BFE_I64 i64:$src, (i32 0x80000)) // 0 | 8 << 16 2195>; 2196 2197def : GCNPat < 2198 (i64 (UniformSextInreg<i16> i64:$src)), 2199 (S_BFE_I64 i64:$src, (i32 0x100000)) // 0 | 16 << 16 2200>; 2201 2202def : GCNPat < 2203 (i64 (UniformSextInreg<i32> i64:$src)), 2204 (S_BFE_I64 i64:$src, (i32 0x200000)) // 0 | 32 << 16 2205>; 2206 2207def : GCNPat< 2208 (i32 (DivergentSextInreg<i1> i32:$src)), 2209 (V_BFE_I32_e64 i32:$src, (i32 0), (i32 1))>; 2210 2211def : GCNPat < 2212 (i16 (DivergentSextInreg<i1> i16:$src)), 2213 (V_BFE_I32_e64 $src, (i32 0), (i32 1)) 2214>; 2215 2216def : GCNPat < 2217 (i16 (DivergentSextInreg<i8> i16:$src)), 2218 (V_BFE_I32_e64 $src, (i32 0), (i32 8)) 2219>; 2220 2221def : GCNPat< 2222 (i32 (DivergentSextInreg<i8> i32:$src)), 2223 (V_BFE_I32_e64 i32:$src, (i32 0), (i32 8)) 2224>; 2225 2226def : GCNPat < 2227 (i32 (DivergentSextInreg<i16> i32:$src)), 2228 (V_BFE_I32_e64 $src, (i32 0), (i32 16)) 2229>; 2230 2231def : GCNPat < 2232 (i64 (DivergentSextInreg<i1> i64:$src)), 2233 (REG_SEQUENCE VReg_64, 2234 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1)), sub0, 2235 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 1))), sub1) 2236>; 2237 2238def : GCNPat < 2239 (i64 (DivergentSextInreg<i8> i64:$src)), 2240 (REG_SEQUENCE VReg_64, 2241 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8)), sub0, 2242 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 8))), sub1) 2243>; 2244 2245def : GCNPat < 2246 (i64 (DivergentSextInreg<i16> i64:$src)), 2247 (REG_SEQUENCE VReg_64, 2248 (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16)), sub0, 2249 (V_ASHRREV_I32_e32 (i32 31), (V_BFE_I32_e64 (i32 (EXTRACT_SUBREG i64:$src, sub0)), (i32 0), (i32 16))), sub1) 2250>; 2251 2252def : GCNPat < 2253 (i64 (DivergentSextInreg<i32> i64:$src)), 2254 (REG_SEQUENCE VReg_64, 2255 (i32 (EXTRACT_SUBREG i64:$src, sub0)), sub0, 2256 (V_ASHRREV_I32_e32 (i32 31), (i32 (EXTRACT_SUBREG i64:$src, sub0))), sub1) 2257>; 2258 2259def : GCNPat < 2260 (i64 (zext i32:$src)), 2261 (REG_SEQUENCE SReg_64, $src, sub0, (S_MOV_B32 (i32 0)), sub1) 2262>; 2263 2264def : GCNPat < 2265 (i64 (anyext i32:$src)), 2266 (REG_SEQUENCE SReg_64, $src, sub0, (i32 (IMPLICIT_DEF)), sub1) 2267>; 2268 2269class ZExt_i64_i1_Pat <SDNode ext> : GCNPat < 2270 (i64 (ext i1:$src)), 2271 (REG_SEQUENCE VReg_64, 2272 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2273 /*src1mod*/(i32 0), /*src1*/(i32 1), $src), 2274 sub0, (S_MOV_B32 (i32 0)), sub1) 2275>; 2276 2277 2278def : ZExt_i64_i1_Pat<zext>; 2279def : ZExt_i64_i1_Pat<anyext>; 2280 2281// FIXME: We need to use COPY_TO_REGCLASS to work-around the fact that 2282// REG_SEQUENCE patterns don't support instructions with multiple outputs. 2283def : GCNPat < 2284 (i64 (UniformUnaryFrag<sext> i32:$src)), 2285 (REG_SEQUENCE SReg_64, $src, sub0, 2286 (i32 (COPY_TO_REGCLASS (S_ASHR_I32 $src, (i32 31)), SReg_32_XM0)), sub1) 2287>; 2288 2289def : GCNPat < 2290 (i64 (DivergentUnaryFrag<sext> i32:$src)), 2291 (REG_SEQUENCE VReg_64, $src, sub0, 2292 (i32 (COPY_TO_REGCLASS (V_ASHRREV_I32_e64 (i32 31), $src), VGPR_32)), sub1) 2293>; 2294 2295def : GCNPat < 2296 (i64 (sext i1:$src)), 2297 (REG_SEQUENCE VReg_64, 2298 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2299 /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub0, 2300 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2301 /*src1mod*/(i32 0), /*src1*/(i32 -1), $src), sub1) 2302>; 2303 2304class FPToI1Pat<Instruction Inst, int KOne, ValueType kone_type, ValueType vt, SDPatternOperator fp_to_int> : GCNPat < 2305 (i1 (fp_to_int (vt (VOP3Mods vt:$src0, i32:$src0_modifiers)))), 2306 (i1 (Inst 0, (kone_type KOne), $src0_modifiers, $src0, DSTCLAMP.NONE)) 2307>; 2308 2309def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_ONE, i16, f16, fp_to_uint>; 2310def : FPToI1Pat<V_CMP_EQ_F16_e64, CONST.FP16_NEG_ONE, i16, f16, fp_to_sint>; 2311def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_ONE, i32, f32, fp_to_uint>; 2312def : FPToI1Pat<V_CMP_EQ_F32_e64, CONST.FP32_NEG_ONE, i32, f32, fp_to_sint>; 2313def : FPToI1Pat<V_CMP_EQ_F64_e64, CONST.FP64_ONE, i64, f64, fp_to_uint>; 2314def : FPToI1Pat<V_CMP_EQ_F64_e64, CONST.FP64_NEG_ONE, i64, f64, fp_to_sint>; 2315 2316// If we need to perform a logical operation on i1 values, we need to 2317// use vector comparisons since there is only one SCC register. Vector 2318// comparisons may write to a pair of SGPRs or a single SGPR, so treat 2319// these as 32 or 64-bit comparisons. When legalizing SGPR copies, 2320// instructions resulting in the copies from SCC to these instructions 2321// will be moved to the VALU. 2322 2323let WaveSizePredicate = isWave64 in { 2324def : GCNPat < 2325 (i1 (and i1:$src0, i1:$src1)), 2326 (S_AND_B64 $src0, $src1) 2327>; 2328 2329def : GCNPat < 2330 (i1 (or i1:$src0, i1:$src1)), 2331 (S_OR_B64 $src0, $src1) 2332>; 2333 2334def : GCNPat < 2335 (i1 (xor i1:$src0, i1:$src1)), 2336 (S_XOR_B64 $src0, $src1) 2337>; 2338 2339def : GCNPat < 2340 (i1 (add i1:$src0, i1:$src1)), 2341 (S_XOR_B64 $src0, $src1) 2342>; 2343 2344def : GCNPat < 2345 (i1 (sub i1:$src0, i1:$src1)), 2346 (S_XOR_B64 $src0, $src1) 2347>; 2348 2349let AddedComplexity = 1 in { 2350def : GCNPat < 2351 (i1 (add i1:$src0, (i1 -1))), 2352 (S_NOT_B64 $src0) 2353>; 2354 2355def : GCNPat < 2356 (i1 (sub i1:$src0, (i1 -1))), 2357 (S_NOT_B64 $src0) 2358>; 2359} 2360} // end isWave64 2361 2362let WaveSizePredicate = isWave32 in { 2363def : GCNPat < 2364 (i1 (and i1:$src0, i1:$src1)), 2365 (S_AND_B32 $src0, $src1) 2366>; 2367 2368def : GCNPat < 2369 (i1 (or i1:$src0, i1:$src1)), 2370 (S_OR_B32 $src0, $src1) 2371>; 2372 2373def : GCNPat < 2374 (i1 (xor i1:$src0, i1:$src1)), 2375 (S_XOR_B32 $src0, $src1) 2376>; 2377 2378def : GCNPat < 2379 (i1 (add i1:$src0, i1:$src1)), 2380 (S_XOR_B32 $src0, $src1) 2381>; 2382 2383def : GCNPat < 2384 (i1 (sub i1:$src0, i1:$src1)), 2385 (S_XOR_B32 $src0, $src1) 2386>; 2387 2388let AddedComplexity = 1 in { 2389def : GCNPat < 2390 (i1 (add i1:$src0, (i1 -1))), 2391 (S_NOT_B32 $src0) 2392>; 2393 2394def : GCNPat < 2395 (i1 (sub i1:$src0, (i1 -1))), 2396 (S_NOT_B32 $src0) 2397>; 2398} 2399} // end isWave32 2400 2401def : GCNPat < 2402 (i32 (DivergentBinFrag<xor> i32:$src0, (i32 -1))), 2403 (V_NOT_B32_e32 $src0) 2404>; 2405 2406def : GCNPat < 2407 (i64 (DivergentBinFrag<xor> i64:$src0, (i64 -1))), 2408 (REG_SEQUENCE VReg_64, 2409 (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub0))), sub0, 2410 (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub1))), sub1 2411 ) 2412>; 2413 2414let SubtargetPredicate = NotHasTrue16BitInsts in 2415def : GCNPat < 2416 (f16 (sint_to_fp i1:$src)), 2417 (V_CVT_F16_F32_e32 ( 2418 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2419 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE), 2420 SSrc_i1:$src)) 2421>; 2422 2423let SubtargetPredicate = HasTrue16BitInsts in 2424def : GCNPat < 2425 (f16 (sint_to_fp i1:$src)), 2426 (V_CVT_F16_F32_t16_e32 ( 2427 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2428 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE), 2429 SSrc_i1:$src)) 2430>; 2431 2432let SubtargetPredicate = NotHasTrue16BitInsts in 2433def : GCNPat < 2434 (f16 (uint_to_fp i1:$src)), 2435 (V_CVT_F16_F32_e32 ( 2436 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2437 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE), 2438 SSrc_i1:$src)) 2439>; 2440let SubtargetPredicate = HasTrue16BitInsts in 2441def : GCNPat < 2442 (f16 (uint_to_fp i1:$src)), 2443 (V_CVT_F16_F32_t16_e32 ( 2444 V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2445 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE), 2446 SSrc_i1:$src)) 2447>; 2448 2449def : GCNPat < 2450 (f32 (sint_to_fp i1:$src)), 2451 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2452 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE), 2453 SSrc_i1:$src) 2454>; 2455 2456def : GCNPat < 2457 (f32 (uint_to_fp i1:$src)), 2458 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2459 /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE), 2460 SSrc_i1:$src) 2461>; 2462 2463def : GCNPat < 2464 (f64 (sint_to_fp i1:$src)), 2465 (V_CVT_F64_I32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2466 /*src1mod*/(i32 0), /*src1*/(i32 -1), 2467 SSrc_i1:$src)) 2468>; 2469 2470def : GCNPat < 2471 (f64 (uint_to_fp i1:$src)), 2472 (V_CVT_F64_U32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), 2473 /*src1mod*/(i32 0), /*src1*/(i32 1), 2474 SSrc_i1:$src)) 2475>; 2476 2477//===----------------------------------------------------------------------===// 2478// Miscellaneous Patterns 2479//===----------------------------------------------------------------------===// 2480 2481// Eliminate a zero extension from an fp16 operation if it already 2482// zeros the high bits of the 32-bit register. 2483// 2484// This is complicated on gfx9+. Some instructions maintain the legacy 2485// zeroing behavior, but others preserve the high bits. Some have a 2486// control bit to change the behavior. We can't simply say with 2487// certainty what the source behavior is without more context on how 2488// the src is lowered. e.g. fptrunc + fma may be lowered to a 2489// v_fma_mix* instruction which does not zero, or may not. 2490def : GCNPat< 2491 (i32 (DivergentUnaryFrag<abs> i32:$src)), 2492 (V_MAX_I32_e64 (V_SUB_CO_U32_e32 (i32 0), $src), $src)>; 2493 2494let AddedComplexity = 1 in { 2495def : GCNPat< 2496 (i32 (DivergentUnaryFrag<abs> i32:$src)), 2497 (V_MAX_I32_e64 (V_SUB_U32_e32 (i32 0), $src), $src)>{ 2498 let SubtargetPredicate = HasAddNoCarryInsts; 2499} 2500} // AddedComplexity = 1 2501 2502def : GCNPat< 2503 (i32 (DivergentUnaryFrag<zext> i16:$src)), 2504 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src) 2505>; 2506 2507def : GCNPat< 2508 (i64 (DivergentUnaryFrag<zext> i16:$src)), 2509 (REG_SEQUENCE VReg_64, 2510 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src), sub0, 2511 (S_MOV_B32 (i32 0)), sub1) 2512>; 2513 2514def : GCNPat< 2515 (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))), 2516 (COPY VSrc_b16:$src)>; 2517 2518def : GCNPat < 2519 (i32 (trunc i64:$a)), 2520 (EXTRACT_SUBREG $a, sub0) 2521>; 2522 2523def : GCNPat < 2524 (i1 (UniformUnaryFrag<trunc> i32:$a)), 2525 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), $a), (i32 1)) 2526>; 2527 2528def : GCNPat < 2529 (i1 (UniformUnaryFrag<trunc> i16:$a)), 2530 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), $a), (i32 1)) 2531>; 2532 2533def : GCNPat < 2534 (i1 (UniformUnaryFrag<trunc> i64:$a)), 2535 (S_CMP_EQ_U32 (S_AND_B32 (i32 1), 2536 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1)) 2537>; 2538 2539def : GCNPat < 2540 (i1 (DivergentUnaryFrag<trunc> i32:$a)), 2541 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) 2542>; 2543 2544def : GCNPat < 2545 (i1 (DivergentUnaryFrag<trunc> i16:$a)), 2546 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) 2547>; 2548 2549def IMMBitSelConst : SDNodeXForm<imm, [{ 2550 return CurDAG->getTargetConstant(1ULL << N->getZExtValue(), SDLoc(N), 2551 MVT::i32); 2552}]>; 2553 2554// Matching separate SRL and TRUNC instructions 2555// with dependent operands (SRL dest is source of TRUNC) 2556// generates three instructions. However, by using bit shifts, 2557// the V_LSHRREV_B32_e64 result can be directly used in the 2558// operand of the V_AND_B32_e64 instruction: 2559// (trunc i32 (srl i32 $a, i32 $b)) -> 2560// v_and_b32_e64 $a, (1 << $b), $a 2561// v_cmp_ne_u32_e64 $a, 0, $a 2562 2563// Handle the VALU case. 2564def : GCNPat < 2565 (i1 (DivergentUnaryFrag<trunc> (i32 (srl i32:$a, (i32 imm:$b))))), 2566 (V_CMP_NE_U32_e64 (V_AND_B32_e64 (i32 (IMMBitSelConst $b)), $a), 2567 (i32 0)) 2568>; 2569 2570// Handle the scalar case. 2571def : GCNPat < 2572 (i1 (UniformUnaryFrag<trunc> (i32 (srl i32:$a, (i32 imm:$b))))), 2573 (S_CMP_LG_U32 (S_AND_B32 (i32 (IMMBitSelConst $b)), $a), 2574 (i32 0)) 2575>; 2576 2577def : GCNPat < 2578 (i1 (DivergentUnaryFrag<trunc> i64:$a)), 2579 (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), 2580 (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1)) 2581>; 2582 2583def : GCNPat < 2584 (i32 (bswap i32:$a)), 2585 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)), 2586 (V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 24)), 2587 (V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 8))) 2588>; 2589 2590// FIXME: This should have been narrowed to i32 during legalization. 2591// This pattern should also be skipped for GlobalISel 2592def : GCNPat < 2593 (i64 (bswap i64:$a)), 2594 (REG_SEQUENCE VReg_64, 2595 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)), 2596 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2597 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2598 (i32 24)), 2599 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2600 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)), 2601 (i32 8))), 2602 sub0, 2603 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)), 2604 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2605 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2606 (i32 24)), 2607 (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2608 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)), 2609 (i32 8))), 2610 sub1) 2611>; 2612 2613// FIXME: The AddedComplexity should not be needed, but in GlobalISel 2614// the BFI pattern ends up taking precedence without it. 2615let SubtargetPredicate = isGFX8Plus, AddedComplexity = 1 in { 2616// Magic number: 3 | (2 << 8) | (1 << 16) | (0 << 24) 2617// 2618// My reading of the manual suggests we should be using src0 for the 2619// register value, but this is what seems to work. 2620def : GCNPat < 2621 (i32 (bswap i32:$a)), 2622 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x00010203))) 2623>; 2624 2625// FIXME: This should have been narrowed to i32 during legalization. 2626// This pattern should also be skipped for GlobalISel 2627def : GCNPat < 2628 (i64 (bswap i64:$a)), 2629 (REG_SEQUENCE VReg_64, 2630 (V_PERM_B32_e64 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub1), 2631 (S_MOV_B32 (i32 0x00010203))), 2632 sub0, 2633 (V_PERM_B32_e64 (i32 0), (EXTRACT_SUBREG VReg_64:$a, sub0), 2634 (S_MOV_B32 (i32 0x00010203))), 2635 sub1) 2636>; 2637 2638// Magic number: 1 | (0 << 8) | (12 << 16) | (12 << 24) 2639// The 12s emit 0s. 2640def : GCNPat < 2641 (i16 (bswap i16:$a)), 2642 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001))) 2643>; 2644 2645def : GCNPat < 2646 (i32 (zext (bswap i16:$a))), 2647 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x0c0c0001))) 2648>; 2649 2650// Magic number: 1 | (0 << 8) | (3 << 16) | (2 << 24) 2651def : GCNPat < 2652 (v2i16 (bswap v2i16:$a)), 2653 (V_PERM_B32_e64 (i32 0), VSrc_b32:$a, (S_MOV_B32 (i32 0x02030001))) 2654>; 2655 2656} 2657 2658def : GCNPat< 2659 (i64 (DivergentUnaryFrag<bitreverse> i64:$a)), 2660 (REG_SEQUENCE VReg_64, 2661 (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1))), sub0, 2662 (V_BFREV_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0))), sub1)>; 2663 2664// Prefer selecting to max when legal, but using mul is always valid. 2665let AddedComplexity = -5 in { 2666 2667let OtherPredicates = [NotHasTrue16BitInsts] in { 2668def : GCNPat< 2669 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 2670 (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src) 2671>; 2672 2673def : GCNPat< 2674 (fcanonicalize (f16 (fneg (VOP3Mods f16:$src, i32:$src_mods)))), 2675 (V_MUL_F16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src) 2676>; 2677} // End OtherPredicates 2678 2679let OtherPredicates = [HasTrue16BitInsts] in { 2680def : GCNPat< 2681 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 2682 (V_MUL_F16_t16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src) 2683>; 2684 2685def : GCNPat< 2686 (fcanonicalize (f16 (fneg (VOP3Mods f16:$src, i32:$src_mods)))), 2687 (V_MUL_F16_t16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src) 2688>; 2689} // End OtherPredicates 2690 2691def : GCNPat< 2692 (fcanonicalize (v2f16 (VOP3PMods v2f16:$src, i32:$src_mods))), 2693 (V_PK_MUL_F16 0, (i32 CONST.FP16_ONE), $src_mods, $src, DSTCLAMP.NONE) 2694>; 2695 2696def : GCNPat< 2697 (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))), 2698 (V_MUL_F32_e64 0, (i32 CONST.FP32_ONE), $src_mods, $src) 2699>; 2700 2701def : GCNPat< 2702 (fcanonicalize (f32 (fneg (VOP3Mods f32:$src, i32:$src_mods)))), 2703 (V_MUL_F32_e64 0, (i32 CONST.FP32_NEG_ONE), $src_mods, $src) 2704>; 2705 2706// TODO: Handle fneg like other types. 2707def : GCNPat< 2708 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))), 2709 (V_MUL_F64_e64 0, CONST.FP64_ONE, $src_mods, $src) 2710>; 2711} // End AddedComplexity = -5 2712 2713multiclass SelectCanonicalizeAsMax< 2714 list<Predicate> f32_preds = [], 2715 list<Predicate> f64_preds = [], 2716 list<Predicate> f16_preds = []> { 2717 def : GCNPat< 2718 (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))), 2719 (V_MAX_F32_e64 $src_mods, $src, $src_mods, $src)> { 2720 let OtherPredicates = f32_preds; 2721 } 2722 2723 def : GCNPat< 2724 (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))), 2725 (V_MAX_F64_e64 $src_mods, $src, $src_mods, $src)> { 2726 let OtherPredicates = f64_preds; 2727 } 2728 2729 def : GCNPat< 2730 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 2731 (V_MAX_F16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> { 2732 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts, NotHasTrue16BitInsts]); 2733 } 2734 2735 def : GCNPat< 2736 (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))), 2737 (V_MAX_F16_t16_e64 $src_mods, $src, $src_mods, $src, 0, 0)> { 2738 let OtherPredicates = !listconcat(f16_preds, [Has16BitInsts, HasTrue16BitInsts]); 2739 } 2740 2741 def : GCNPat< 2742 (fcanonicalize (v2f16 (VOP3PMods v2f16:$src, i32:$src_mods))), 2743 (V_PK_MAX_F16 $src_mods, $src, $src_mods, $src, DSTCLAMP.NONE)> { 2744 // FIXME: Should have VOP3P subtarget predicate 2745 let OtherPredicates = f16_preds; 2746 } 2747} 2748 2749// On pre-gfx9 targets, v_max_*/v_min_* did not respect the denormal 2750// mode, and would never flush. For f64, it's faster to do implement 2751// this with a max. For f16/f32 it's a wash, but prefer max when 2752// valid. 2753// 2754// FIXME: Lowering f32/f16 with max is worse since we can use a 2755// smaller encoding if the input is fneg'd. It also adds an extra 2756// register use. 2757let SubtargetPredicate = HasMinMaxDenormModes in { 2758 defm : SelectCanonicalizeAsMax<[], [], []>; 2759} // End SubtargetPredicate = HasMinMaxDenormModes 2760 2761let SubtargetPredicate = NotHasMinMaxDenormModes in { 2762 // Use the max lowering if we don't need to flush. 2763 2764 // FIXME: We don't do use this for f32 as a workaround for the 2765 // library being compiled with the default ieee mode, but 2766 // potentially being called from flushing kernels. Really we should 2767 // not be mixing code expecting different default FP modes, but mul 2768 // works in any FP environment. 2769 defm : SelectCanonicalizeAsMax<[FalsePredicate], [FP64Denormals], [FP16Denormals]>; 2770} // End SubtargetPredicate = NotHasMinMaxDenormModes 2771 2772 2773let OtherPredicates = [HasDLInsts] in { 2774// Don't allow source modifiers. If there are any source modifiers then it's 2775// better to select fma instead of fmac. 2776def : GCNPat < 2777 (fma (f32 (VOP3NoMods f32:$src0)), 2778 (f32 (VOP3NoMods f32:$src1)), 2779 (f32 (VOP3NoMods f32:$src2))), 2780 (V_FMAC_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 2781 SRCMODS.NONE, $src2) 2782>; 2783} // End OtherPredicates = [HasDLInsts] 2784 2785let SubtargetPredicate = isGFX10Plus in { 2786// Don't allow source modifiers. If there are any source modifiers then it's 2787// better to select fma instead of fmac. 2788let OtherPredicates = [NotHasTrue16BitInsts] in 2789def : GCNPat < 2790 (fma (f16 (VOP3NoMods f32:$src0)), 2791 (f16 (VOP3NoMods f32:$src1)), 2792 (f16 (VOP3NoMods f32:$src2))), 2793 (V_FMAC_F16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 2794 SRCMODS.NONE, $src2) 2795>; 2796let OtherPredicates = [HasTrue16BitInsts] in 2797def : GCNPat < 2798 (fma (f16 (VOP3NoMods f32:$src0)), 2799 (f16 (VOP3NoMods f32:$src1)), 2800 (f16 (VOP3NoMods f32:$src2))), 2801 (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 2802 SRCMODS.NONE, $src2) 2803>; 2804} 2805 2806let OtherPredicates = [HasFmacF64Inst] in 2807// Don't allow source modifiers. If there are any source modifiers then it's 2808// better to select fma instead of fmac. 2809def : GCNPat < 2810 (fma (f64 (VOP3NoMods f64:$src0)), 2811 (f64 (VOP3NoMods f64:$src1)), 2812 (f64 (VOP3NoMods f64:$src2))), 2813 (V_FMAC_F64_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, 2814 SRCMODS.NONE, $src2) 2815>; 2816 2817// COPY is workaround tablegen bug from multiple outputs 2818// from S_LSHL_B32's multiple outputs from implicit scc def. 2819let AddedComplexity = 1 in { 2820def : GCNPat < 2821 (v2i16 (UniformBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))), 2822 (S_LSHL_B32 SReg_32:$src1, (i16 16)) 2823>; 2824 2825def : GCNPat < 2826 (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 VGPR_32:$src1))), 2827 (v2i16 (V_LSHLREV_B32_e64 (i16 16), VGPR_32:$src1)) 2828>; 2829 2830 2831def : GCNPat < 2832 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))), 2833 (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1) 2834>; 2835 2836def : GCNPat < 2837 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src1), (i16 0))), 2838 (v2i16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1)) 2839>; 2840 2841def : GCNPat < 2842 (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))), 2843 (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1) 2844>; 2845 2846def : GCNPat < 2847 (v2f16 (DivergentBinFrag<build_vector> (f16 VGPR_32:$src1), (f16 FP_ZERO))), 2848 (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1)) 2849>; 2850 2851def : GCNPat < 2852 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 undef))), 2853 (COPY_TO_REGCLASS SReg_32:$src0, SReg_32) 2854>; 2855 2856def : GCNPat < 2857 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 undef))), 2858 (COPY_TO_REGCLASS VGPR_32:$src0, VGPR_32) 2859>; 2860 2861def : GCNPat < 2862 (v2f16 (build_vector f16:$src0, (f16 undef))), 2863 (COPY $src0) 2864>; 2865 2866def : GCNPat < 2867 (v2i16 (UniformBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))), 2868 (S_LSHL_B32 SReg_32:$src1, (i32 16)) 2869>; 2870 2871def : GCNPat < 2872 (v2i16 (DivergentBinFrag<build_vector> (i16 undef), (i16 VGPR_32:$src1))), 2873 (v2i16 (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1)) 2874>; 2875 2876 2877def : GCNPat < 2878 (v2f16 (UniformBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))), 2879 (S_LSHL_B32 SReg_32:$src1, (i32 16)) 2880>; 2881 2882def : GCNPat < 2883 (v2f16 (DivergentBinFrag<build_vector> (f16 undef), (f16 VGPR_32:$src1))), 2884 (v2f16 (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1)) 2885>; 2886} 2887 2888let SubtargetPredicate = HasVOP3PInsts in { 2889def : GCNPat < 2890 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))), 2891 (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) 2892>; 2893 2894def : GCNPat < 2895 (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 VGPR_32:$src1))), 2896 (v2i16 (V_LSHL_OR_B32_e64 $src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), $src0)))) 2897>; 2898 2899// With multiple uses of the shift, this will duplicate the shift and 2900// increase register pressure. 2901def : GCNPat < 2902 (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), 2903 (v2i16 (S_PACK_LH_B32_B16 SReg_32:$src0, SReg_32:$src1)) 2904>; 2905 2906def : GCNPat < 2907 (v2i16 (UniformBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), 2908 (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), 2909 (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1) 2910>; 2911 2912def : GCNPat < 2913 (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))), 2914 (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) 2915>; 2916 2917 2918 2919foreach Ty = [i16, f16] in { 2920 2921defvar vecTy = !if(!eq(Ty, i16), v2i16, v2f16); 2922defvar immzeroTy = !if(!eq(Ty, i16), immzero, fpimmzero); 2923 2924// Take the lower 16 bits from each VGPR_32 and concat them 2925def : GCNPat < 2926 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), (Ty VGPR_32:$b))), 2927 (V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x05040100))) 2928>; 2929 2930 2931// Take the lower 16 bits from V[0] and the upper 16 bits from V[1] 2932// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000) 2933def : GCNPat < 2934 (vecTy (DivergentBinFrag<build_vector> (Ty (immzeroTy)), 2935 (Ty !if(!eq(Ty, i16), 2936 (Ty (trunc (srl VGPR_32:$b, (i32 16)))), 2937 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))), 2938 (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff0000)), VGPR_32:$b) 2939>; 2940 2941 2942// Take the lower 16 bits from V[0] and the upper 16 bits from V[1] 2943// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000) 2944def : GCNPat < 2945 (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), 2946 (Ty !if(!eq(Ty, i16), 2947 (Ty (trunc (srl VGPR_32:$b, (i32 16)))), 2948 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))), 2949 (V_BFI_B32_e64 (S_MOV_B32 (i32 0x0000ffff)), VGPR_32:$a, VGPR_32:$b) 2950>; 2951 2952 2953// Take the upper 16 bits from V[0] and the lower 16 bits from V[1] 2954// Special case, can use V_ALIGNBIT (always uses encoded literal) 2955def : GCNPat < 2956 (vecTy (DivergentBinFrag<build_vector> 2957 (Ty !if(!eq(Ty, i16), 2958 (Ty (trunc (srl VGPR_32:$a, (i32 16)))), 2959 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))), 2960 (Ty VGPR_32:$b))), 2961 (V_ALIGNBIT_B32_e64 VGPR_32:$b, VGPR_32:$a, (i32 16)) 2962>; 2963 2964// Take the upper 16 bits from each VGPR_32 and concat them 2965def : GCNPat < 2966 (vecTy (DivergentBinFrag<build_vector> 2967 (Ty !if(!eq(Ty, i16), 2968 (Ty (trunc (srl VGPR_32:$a, (i32 16)))), 2969 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))), 2970 (Ty !if(!eq(Ty, i16), 2971 (Ty (trunc (srl VGPR_32:$b, (i32 16)))), 2972 (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))), 2973 (V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x07060302))) 2974>; 2975 2976 2977} // end foreach Ty 2978 2979 2980let AddedComplexity = 5 in { 2981def : GCNPat < 2982 (v2f16 (is_canonicalized<build_vector> (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)), 2983 (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))), 2984 (V_PACK_B32_F16_e64 $src0_mods, VGPR_32:$src0, $src1_mods, VGPR_32:$src1) 2985>; 2986} 2987} // End SubtargetPredicate = HasVOP3PInsts 2988 2989// With multiple uses of the shift, this will duplicate the shift and 2990// increase register pressure. 2991let SubtargetPredicate = isGFX11Plus in 2992def : GCNPat < 2993 (v2i16 (build_vector (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), (i16 SReg_32:$src1))), 2994 (v2i16 (S_PACK_HL_B32_B16 SReg_32:$src0, SReg_32:$src1)) 2995>; 2996 2997 2998def : GCNPat < 2999 (v2f16 (scalar_to_vector f16:$src0)), 3000 (COPY $src0) 3001>; 3002 3003def : GCNPat < 3004 (v2i16 (scalar_to_vector i16:$src0)), 3005 (COPY $src0) 3006>; 3007 3008def : GCNPat < 3009 (v4i16 (scalar_to_vector i16:$src0)), 3010 (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0) 3011>; 3012 3013def : GCNPat < 3014 (v4f16 (scalar_to_vector f16:$src0)), 3015 (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0) 3016>; 3017 3018def : GCNPat < 3019 (i64 (int_amdgcn_mov_dpp i64:$src, timm:$dpp_ctrl, timm:$row_mask, 3020 timm:$bank_mask, timm:$bound_ctrl)), 3021 (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$src, VReg_64_Align2:$src, 3022 (as_i32timm $dpp_ctrl), (as_i32timm $row_mask), 3023 (as_i32timm $bank_mask), 3024 (as_i1timm $bound_ctrl)) 3025>; 3026 3027def : GCNPat < 3028 (i64 (int_amdgcn_update_dpp i64:$old, i64:$src, timm:$dpp_ctrl, timm:$row_mask, 3029 timm:$bank_mask, timm:$bound_ctrl)), 3030 (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$old, VReg_64_Align2:$src, (as_i32timm $dpp_ctrl), 3031 (as_i32timm $row_mask), (as_i32timm $bank_mask), 3032 (as_i1timm $bound_ctrl)) 3033>; 3034 3035//===----------------------------------------------------------------------===// 3036// Fract Patterns 3037//===----------------------------------------------------------------------===// 3038 3039let SubtargetPredicate = isGFX6 in { 3040 3041// V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x)) is 3042// used instead. However, SI doesn't have V_FLOOR_F64, so the most efficient 3043// way to implement it is using V_FRACT_F64. 3044// The workaround for the V_FRACT bug is: 3045// fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999) 3046 3047// Convert floor(x) to (x - fract(x)) 3048 3049// Don't bother handling this for GlobalISel, it's handled during 3050// lowering. 3051// 3052// FIXME: DAG should also custom lower this. 3053def : GCNPat < 3054 (f64 (ffloor (f64 (VOP3Mods f64:$x, i32:$mods)))), 3055 (V_ADD_F64_e64 3056 $mods, 3057 $x, 3058 SRCMODS.NEG, 3059 (V_CNDMASK_B64_PSEUDO 3060 (V_MIN_F64_e64 3061 SRCMODS.NONE, 3062 (V_FRACT_F64_e64 $mods, $x), 3063 SRCMODS.NONE, 3064 (V_MOV_B64_PSEUDO 0x3fefffffffffffff)), 3065 $x, 3066 (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, (i32 3 /*NaN*/)))) 3067>; 3068 3069} // End SubtargetPredicates = isGFX6 3070 3071//============================================================================// 3072// Miscellaneous Optimization Patterns 3073//============================================================================// 3074 3075// Undo sub x, c -> add x, -c canonicalization since c is more likely 3076// an inline immediate than -c. 3077// TODO: Also do for 64-bit. 3078def : GCNPat< 3079 (UniformBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)), 3080 (S_SUB_I32 SReg_32:$src0, NegSubInlineConst32:$src1) 3081>; 3082 3083def : GCNPat< 3084 (DivergentBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)), 3085 (V_SUB_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> { 3086 let SubtargetPredicate = HasAddNoCarryInsts; 3087} 3088 3089def : GCNPat< 3090 (DivergentBinFrag<add> i32:$src0, (i32 NegSubInlineConst32:$src1)), 3091 (V_SUB_CO_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> { 3092 let SubtargetPredicate = NotHasAddNoCarryInsts; 3093} 3094 3095 3096// Avoid pointlessly materializing a constant in VGPR. 3097// FIXME: Should also do this for readlane, but tablegen crashes on 3098// the ignored src1. 3099def : GCNPat< 3100 (int_amdgcn_readfirstlane (i32 imm:$src)), 3101 (S_MOV_B32 SReg_32:$src) 3102>; 3103 3104multiclass BFMPatterns <ValueType vt, PatFrag SHL, PatFrag ADD, InstSI BFM> { 3105 def : GCNPat < 3106 (vt (SHL (vt (add (vt (shl 1, vt:$a)), -1)), vt:$b)), 3107 (BFM $a, $b) 3108 >; 3109 3110 def : GCNPat < 3111 (vt (ADD (vt (shl 1, vt:$a)), -1)), 3112 (BFM $a, (i32 0)) 3113 >; 3114} 3115 3116defm : BFMPatterns <i32, UniformBinFrag<shl>, UniformBinFrag<add>, S_BFM_B32>; 3117// FIXME: defm : BFMPatterns <i64, UniformBinFrag<shl>, UniformBinFrag<add>, S_BFM_B64>; 3118defm : BFMPatterns <i32, DivergentBinFrag<shl>, DivergentBinFrag<add>, V_BFM_B32_e64>; 3119 3120// Bitfield extract patterns 3121 3122def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{ 3123 return isMask_32(Imm); 3124}]>; 3125 3126def IMMPopCount : SDNodeXForm<imm, [{ 3127 return CurDAG->getTargetConstant(llvm::popcount(N->getZExtValue()), SDLoc(N), 3128 MVT::i32); 3129}]>; 3130 3131def : AMDGPUPat < 3132 (DivergentBinFrag<and> (i32 (srl i32:$src, i32:$rshift)), 3133 IMMZeroBasedBitfieldMask:$mask), 3134 (V_BFE_U32_e64 $src, $rshift, (i32 (IMMPopCount $mask))) 3135>; 3136 3137// x & ((1 << y) - 1) 3138def : AMDGPUPat < 3139 (DivergentBinFrag<and> i32:$src, (add_oneuse (shl_oneuse 1, i32:$width), -1)), 3140 (V_BFE_U32_e64 $src, (i32 0), $width) 3141>; 3142 3143// x & ~(-1 << y) 3144def : AMDGPUPat < 3145 (DivergentBinFrag<and> i32:$src, 3146 (xor_oneuse (shl_oneuse -1, i32:$width), -1)), 3147 (V_BFE_U32_e64 $src, (i32 0), $width) 3148>; 3149 3150// x & (-1 >> (bitwidth - y)) 3151def : AMDGPUPat < 3152 (DivergentBinFrag<and> i32:$src, (srl_oneuse -1, (sub 32, i32:$width))), 3153 (V_BFE_U32_e64 $src, (i32 0), $width) 3154>; 3155 3156// x << (bitwidth - y) >> (bitwidth - y) 3157def : AMDGPUPat < 3158 (DivergentBinFrag<srl> (shl_oneuse i32:$src, (sub 32, i32:$width)), 3159 (sub 32, i32:$width)), 3160 (V_BFE_U32_e64 $src, (i32 0), $width) 3161>; 3162 3163def : AMDGPUPat < 3164 (DivergentBinFrag<sra> (shl_oneuse i32:$src, (sub 32, i32:$width)), 3165 (sub 32, i32:$width)), 3166 (V_BFE_I32_e64 $src, (i32 0), $width) 3167>; 3168 3169// SHA-256 Ma patterns 3170 3171// ((x & z) | (y & (x | z))) -> BFI (XOR x, y), z, y 3172def : AMDGPUPatIgnoreCopies < 3173 (DivergentBinFrag<or> (and i32:$x, i32:$z), 3174 (and i32:$y, (or i32:$x, i32:$z))), 3175 (V_BFI_B32_e64 (V_XOR_B32_e64 (COPY_TO_REGCLASS VSrc_b32:$x, VGPR_32), 3176 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)), 3177 (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32), 3178 (COPY_TO_REGCLASS VSrc_b32:$y, VGPR_32)) 3179>; 3180 3181def : AMDGPUPatIgnoreCopies < 3182 (DivergentBinFrag<or> (and i64:$x, i64:$z), 3183 (and i64:$y, (or i64:$x, i64:$z))), 3184 (REG_SEQUENCE VReg_64, 3185 (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), 3186 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), 3187 (i32 (EXTRACT_SUBREG VReg_64:$z, sub0)), 3188 (i32 (EXTRACT_SUBREG VReg_64:$y, sub0))), sub0, 3189 (V_BFI_B32_e64 (V_XOR_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub1)), 3190 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), 3191 (i32 (EXTRACT_SUBREG VReg_64:$z, sub1)), 3192 (i32 (EXTRACT_SUBREG VReg_64:$y, sub1))), sub1) 3193>; 3194 3195multiclass IntMed3Pat<Instruction med3Inst, 3196 SDPatternOperator min, 3197 SDPatternOperator max> { 3198 3199 // This matches 16 permutations of 3200 // min(max(a, b), max(min(a, b), c)) 3201 def : AMDGPUPat < 3202 (min (max i32:$src0, i32:$src1), 3203 (max (min i32:$src0, i32:$src1), i32:$src2)), 3204 (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) 3205>; 3206 3207 // This matches 16 permutations of 3208 // max(min(x, y), min(max(x, y), z)) 3209 def : AMDGPUPat < 3210 (max (min i32:$src0, i32:$src1), 3211 (min (max i32:$src0, i32:$src1), i32:$src2)), 3212 (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) 3213>; 3214} 3215 3216defm : IntMed3Pat<V_MED3_I32_e64, smin, smax>; 3217defm : IntMed3Pat<V_MED3_U32_e64, umin, umax>; 3218 3219multiclass FPMed3Pat<ValueType vt, 3220 Instruction med3Inst> { 3221 // This matches 16 permutations of max(min(x, y), min(max(x, y), z)) 3222 def : GCNPat< 3223 (fmaxnum_like_nnan 3224 (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3225 (VOP3Mods vt:$src1, i32:$src1_mods)), 3226 (fminnum_like (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3227 (VOP3Mods vt:$src1, i32:$src1_mods)), 3228 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))), 3229 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, 3230 DSTCLAMP.NONE, DSTOMOD.NONE)>; 3231 3232 3233 // This matches 16 permutations of min(max(x, y), max(min(x, y), z)) 3234 def : GCNPat< 3235 (fminnum_like_nnan 3236 (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3237 (VOP3Mods vt:$src1, i32:$src1_mods)), 3238 (fmaxnum_like (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3239 (VOP3Mods vt:$src1, i32:$src1_mods)), 3240 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))), 3241 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, 3242 DSTCLAMP.NONE, DSTOMOD.NONE)>; 3243} 3244 3245class FP16Med3Pat<ValueType vt, 3246 Instruction med3Inst> : GCNPat< 3247 (fmaxnum_like_nnan (fminnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3248 (VOP3Mods vt:$src1, i32:$src1_mods)), 3249 (fminnum_like (fmaxnum_like (VOP3Mods vt:$src0, i32:$src0_mods), 3250 (VOP3Mods vt:$src1, i32:$src1_mods)), 3251 (vt (VOP3Mods vt:$src2, i32:$src2_mods)))), 3252 (med3Inst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, DSTCLAMP.NONE) 3253>; 3254 3255multiclass Int16Med3Pat<Instruction med3Inst, 3256 SDPatternOperator min, 3257 SDPatternOperator max> { 3258 // This matches 16 permutations of 3259 // max(min(x, y), min(max(x, y), z)) 3260 def : GCNPat < 3261 (max (min i16:$src0, i16:$src1), 3262 (min (max i16:$src0, i16:$src1), i16:$src2)), 3263 (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE) 3264>; 3265 3266 // This matches 16 permutations of 3267 // min(max(a, b), max(min(a, b), c)) 3268 def : GCNPat < 3269 (min (max i16:$src0, i16:$src1), 3270 (max (min i16:$src0, i16:$src1), i16:$src2)), 3271 (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE) 3272>; 3273} 3274 3275defm : FPMed3Pat<f32, V_MED3_F32_e64>; 3276 3277class 3278IntMinMaxPat<Instruction minmaxInst, SDPatternOperator min_or_max, 3279 SDPatternOperator max_or_min_oneuse> : AMDGPUPat < 3280 (DivergentBinFrag<min_or_max> (max_or_min_oneuse i32:$src0, i32:$src1), 3281 i32:$src2), 3282 (minmaxInst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) 3283>; 3284 3285class 3286FPMinMaxPat<Instruction minmaxInst, ValueType vt, SDPatternOperator min_or_max, 3287 SDPatternOperator max_or_min_oneuse> : GCNPat < 3288 (min_or_max (max_or_min_oneuse (VOP3Mods vt:$src0, i32:$src0_mods), 3289 (VOP3Mods vt:$src1, i32:$src1_mods)), 3290 (vt (VOP3Mods vt:$src2, i32:$src2_mods))), 3291 (minmaxInst $src0_mods, $src0, $src1_mods, $src1, $src2_mods, $src2, 3292 DSTCLAMP.NONE, DSTOMOD.NONE) 3293>; 3294 3295let OtherPredicates = [isGFX11Plus] in { 3296def : IntMinMaxPat<V_MAXMIN_I32_e64, smin, smax_oneuse>; 3297def : IntMinMaxPat<V_MINMAX_I32_e64, smax, smin_oneuse>; 3298def : IntMinMaxPat<V_MAXMIN_U32_e64, umin, umax_oneuse>; 3299def : IntMinMaxPat<V_MINMAX_U32_e64, umax, umin_oneuse>; 3300def : FPMinMaxPat<V_MINMAX_F32_e64, f32, fmaxnum_like, fminnum_like_oneuse>; 3301def : FPMinMaxPat<V_MAXMIN_F32_e64, f32, fminnum_like, fmaxnum_like_oneuse>; 3302def : FPMinMaxPat<V_MINMAX_F16_e64, f16, fmaxnum_like, fminnum_like_oneuse>; 3303def : FPMinMaxPat<V_MAXMIN_F16_e64, f16, fminnum_like, fmaxnum_like_oneuse>; 3304} 3305 3306let OtherPredicates = [isGFX9Plus] in { 3307def : FP16Med3Pat<f16, V_MED3_F16_e64>; 3308defm : Int16Med3Pat<V_MED3_I16_e64, smin, smax>; 3309defm : Int16Med3Pat<V_MED3_U16_e64, umin, umax>; 3310} // End Predicates = [isGFX9Plus] 3311 3312class AMDGPUGenericInstruction : GenericInstruction { 3313 let Namespace = "AMDGPU"; 3314} 3315 3316// Convert a wave address to a swizzled vector address (i.e. this is 3317// for copying the stack pointer to a vector address appropriate to 3318// use in the offset field of mubuf instructions). 3319def G_AMDGPU_WAVE_ADDRESS : AMDGPUGenericInstruction { 3320 let OutOperandList = (outs type0:$dst); 3321 let InOperandList = (ins type0:$src); 3322 let hasSideEffects = 0; 3323} 3324 3325// Returns -1 if the input is zero. 3326def G_AMDGPU_FFBH_U32 : AMDGPUGenericInstruction { 3327 let OutOperandList = (outs type0:$dst); 3328 let InOperandList = (ins type1:$src); 3329 let hasSideEffects = 0; 3330} 3331 3332// Returns -1 if the input is zero. 3333def G_AMDGPU_FFBL_B32 : AMDGPUGenericInstruction { 3334 let OutOperandList = (outs type0:$dst); 3335 let InOperandList = (ins type1:$src); 3336 let hasSideEffects = 0; 3337} 3338 3339def G_AMDGPU_RCP_IFLAG : AMDGPUGenericInstruction { 3340 let OutOperandList = (outs type0:$dst); 3341 let InOperandList = (ins type1:$src); 3342 let hasSideEffects = 0; 3343} 3344 3345class BufferLoadGenericInstruction : AMDGPUGenericInstruction { 3346 let OutOperandList = (outs type0:$dst); 3347 let InOperandList = (ins type1:$rsrc, type2:$vindex, type2:$voffset, 3348 type2:$soffset, untyped_imm_0:$offset, 3349 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3350 let hasSideEffects = 0; 3351 let mayLoad = 1; 3352} 3353 3354class TBufferLoadGenericInstruction : AMDGPUGenericInstruction { 3355 let OutOperandList = (outs type0:$dst); 3356 let InOperandList = (ins type1:$rsrc, type2:$vindex, type2:$voffset, 3357 type2:$soffset, untyped_imm_0:$offset, untyped_imm_0:$format, 3358 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3359 let hasSideEffects = 0; 3360 let mayLoad = 1; 3361} 3362 3363def G_AMDGPU_BUFFER_LOAD_UBYTE : BufferLoadGenericInstruction; 3364def G_AMDGPU_BUFFER_LOAD_SBYTE : BufferLoadGenericInstruction; 3365def G_AMDGPU_BUFFER_LOAD_USHORT : BufferLoadGenericInstruction; 3366def G_AMDGPU_BUFFER_LOAD_SSHORT : BufferLoadGenericInstruction; 3367def G_AMDGPU_BUFFER_LOAD : BufferLoadGenericInstruction; 3368def G_AMDGPU_BUFFER_LOAD_FORMAT : BufferLoadGenericInstruction; 3369def G_AMDGPU_BUFFER_LOAD_FORMAT_TFE : BufferLoadGenericInstruction; 3370def G_AMDGPU_BUFFER_LOAD_FORMAT_D16 : BufferLoadGenericInstruction; 3371def G_AMDGPU_TBUFFER_LOAD_FORMAT : TBufferLoadGenericInstruction; 3372def G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 : TBufferLoadGenericInstruction; 3373 3374class BufferStoreGenericInstruction : AMDGPUGenericInstruction { 3375 let OutOperandList = (outs); 3376 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, 3377 type2:$soffset, untyped_imm_0:$offset, 3378 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3379 let hasSideEffects = 0; 3380 let mayStore = 1; 3381} 3382 3383class TBufferStoreGenericInstruction : AMDGPUGenericInstruction { 3384 let OutOperandList = (outs); 3385 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, 3386 type2:$soffset, untyped_imm_0:$offset, 3387 untyped_imm_0:$format, 3388 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3389 let hasSideEffects = 0; 3390 let mayStore = 1; 3391} 3392 3393def G_AMDGPU_BUFFER_STORE : BufferStoreGenericInstruction; 3394def G_AMDGPU_BUFFER_STORE_BYTE : BufferStoreGenericInstruction; 3395def G_AMDGPU_BUFFER_STORE_SHORT : BufferStoreGenericInstruction; 3396def G_AMDGPU_BUFFER_STORE_FORMAT : BufferStoreGenericInstruction; 3397def G_AMDGPU_BUFFER_STORE_FORMAT_D16 : BufferStoreGenericInstruction; 3398def G_AMDGPU_TBUFFER_STORE_FORMAT : TBufferStoreGenericInstruction; 3399def G_AMDGPU_TBUFFER_STORE_FORMAT_D16 : TBufferStoreGenericInstruction; 3400 3401def G_AMDGPU_FMIN_LEGACY : AMDGPUGenericInstruction { 3402 let OutOperandList = (outs type0:$dst); 3403 let InOperandList = (ins type0:$src0, type0:$src1); 3404 let hasSideEffects = 0; 3405} 3406 3407def G_AMDGPU_FMAX_LEGACY : AMDGPUGenericInstruction { 3408 let OutOperandList = (outs type0:$dst); 3409 let InOperandList = (ins type0:$src0, type0:$src1); 3410 let hasSideEffects = 0; 3411} 3412 3413foreach N = 0-3 in { 3414def G_AMDGPU_CVT_F32_UBYTE#N : AMDGPUGenericInstruction { 3415 let OutOperandList = (outs type0:$dst); 3416 let InOperandList = (ins type0:$src0); 3417 let hasSideEffects = 0; 3418} 3419} 3420 3421def G_AMDGPU_CVT_PK_I16_I32 : AMDGPUGenericInstruction { 3422 let OutOperandList = (outs type0:$dst); 3423 let InOperandList = (ins type0:$src0, type0:$src1); 3424 let hasSideEffects = 0; 3425} 3426 3427def G_AMDGPU_SMED3 : AMDGPUGenericInstruction { 3428 let OutOperandList = (outs type0:$dst); 3429 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); 3430 let hasSideEffects = 0; 3431} 3432 3433def G_AMDGPU_UMED3 : AMDGPUGenericInstruction { 3434 let OutOperandList = (outs type0:$dst); 3435 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); 3436 let hasSideEffects = 0; 3437} 3438 3439def G_AMDGPU_FMED3 : AMDGPUGenericInstruction { 3440 let OutOperandList = (outs type0:$dst); 3441 let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); 3442 let hasSideEffects = 0; 3443} 3444 3445def G_AMDGPU_CLAMP : AMDGPUGenericInstruction { 3446 let OutOperandList = (outs type0:$dst); 3447 let InOperandList = (ins type0:$src); 3448 let hasSideEffects = 0; 3449} 3450 3451// Integer multiply-add: arg0 * arg1 + arg2. 3452// 3453// arg0 and arg1 are 32-bit integers (interpreted as signed or unsigned), 3454// arg2 is a 64-bit integer. Result is a 64-bit integer and a 1-bit carry-out. 3455class G_AMDGPU_MAD_64_32 : AMDGPUGenericInstruction { 3456 let OutOperandList = (outs type0:$dst, type1:$carry_out); 3457 let InOperandList = (ins type2:$arg0, type2:$arg1, type0:$arg2); 3458 let hasSideEffects = 0; 3459} 3460 3461def G_AMDGPU_MAD_U64_U32 : G_AMDGPU_MAD_64_32; 3462def G_AMDGPU_MAD_I64_I32 : G_AMDGPU_MAD_64_32; 3463 3464// Atomic cmpxchg. $cmpval ad $newval are packed in a single vector 3465// operand Expects a MachineMemOperand in addition to explicit 3466// operands. 3467def G_AMDGPU_ATOMIC_CMPXCHG : AMDGPUGenericInstruction { 3468 let OutOperandList = (outs type0:$oldval); 3469 let InOperandList = (ins ptype1:$addr, type0:$cmpval_newval); 3470 let hasSideEffects = 0; 3471 let mayLoad = 1; 3472 let mayStore = 1; 3473} 3474 3475let Namespace = "AMDGPU" in { 3476def G_AMDGPU_ATOMIC_FMIN : G_ATOMICRMW_OP; 3477def G_AMDGPU_ATOMIC_FMAX : G_ATOMICRMW_OP; 3478} 3479 3480class BufferAtomicGenericInstruction<bit NoRtn = 0> : AMDGPUGenericInstruction { 3481 let OutOperandList = !if(NoRtn, (outs), (outs type0:$dst)); 3482 let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, 3483 type2:$soffset, untyped_imm_0:$offset, 3484 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3485 let hasSideEffects = 0; 3486 let mayLoad = 1; 3487 let mayStore = 1; 3488} 3489 3490def G_AMDGPU_BUFFER_ATOMIC_SWAP : BufferAtomicGenericInstruction; 3491def G_AMDGPU_BUFFER_ATOMIC_ADD : BufferAtomicGenericInstruction; 3492def G_AMDGPU_BUFFER_ATOMIC_SUB : BufferAtomicGenericInstruction; 3493def G_AMDGPU_BUFFER_ATOMIC_SMIN : BufferAtomicGenericInstruction; 3494def G_AMDGPU_BUFFER_ATOMIC_UMIN : BufferAtomicGenericInstruction; 3495def G_AMDGPU_BUFFER_ATOMIC_SMAX : BufferAtomicGenericInstruction; 3496def G_AMDGPU_BUFFER_ATOMIC_UMAX : BufferAtomicGenericInstruction; 3497def G_AMDGPU_BUFFER_ATOMIC_AND : BufferAtomicGenericInstruction; 3498def G_AMDGPU_BUFFER_ATOMIC_OR : BufferAtomicGenericInstruction; 3499def G_AMDGPU_BUFFER_ATOMIC_XOR : BufferAtomicGenericInstruction; 3500def G_AMDGPU_BUFFER_ATOMIC_INC : BufferAtomicGenericInstruction; 3501def G_AMDGPU_BUFFER_ATOMIC_DEC : BufferAtomicGenericInstruction; 3502def G_AMDGPU_BUFFER_ATOMIC_FADD : BufferAtomicGenericInstruction; 3503def G_AMDGPU_BUFFER_ATOMIC_FMIN : BufferAtomicGenericInstruction; 3504def G_AMDGPU_BUFFER_ATOMIC_FMAX : BufferAtomicGenericInstruction; 3505 3506def G_AMDGPU_BUFFER_ATOMIC_CMPSWAP : AMDGPUGenericInstruction { 3507 let OutOperandList = (outs type0:$dst); 3508 let InOperandList = (ins type0:$vdata, type0:$cmp, type1:$rsrc, type2:$vindex, 3509 type2:$voffset, type2:$soffset, untyped_imm_0:$offset, 3510 untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen); 3511 let hasSideEffects = 0; 3512 let mayLoad = 1; 3513 let mayStore = 1; 3514} 3515 3516// Wrapper around llvm.amdgcn.s.buffer.load. This is mostly needed as 3517// a workaround for the intrinsic being defined as readnone, but 3518// really needs a memory operand. 3519def G_AMDGPU_S_BUFFER_LOAD : AMDGPUGenericInstruction { 3520 let OutOperandList = (outs type0:$dst); 3521 let InOperandList = (ins type1:$rsrc, type2:$offset, untyped_imm_0:$cachepolicy); 3522 let hasSideEffects = 0; 3523 let mayLoad = 1; 3524 let mayStore = 0; 3525} 3526 3527// This is equivalent to the G_INTRINSIC*, but the operands may have 3528// been legalized depending on the subtarget requirements. 3529def G_AMDGPU_INTRIN_IMAGE_LOAD : AMDGPUGenericInstruction { 3530 let OutOperandList = (outs type0:$dst); 3531 let InOperandList = (ins unknown:$intrin, variable_ops); 3532 let hasSideEffects = 0; 3533 let mayLoad = 1; 3534 3535 // FIXME: Use separate opcode for atomics. 3536 let mayStore = 1; 3537} 3538 3539def G_AMDGPU_INTRIN_IMAGE_LOAD_D16 : AMDGPUGenericInstruction { 3540 let OutOperandList = (outs type0:$dst); 3541 let InOperandList = (ins unknown:$intrin, variable_ops); 3542 let hasSideEffects = 0; 3543 let mayLoad = 1; 3544 3545 // FIXME: Use separate opcode for atomics. 3546 let mayStore = 1; 3547} 3548 3549// This is equivalent to the G_INTRINSIC*, but the operands may have 3550// been legalized depending on the subtarget requirements. 3551def G_AMDGPU_INTRIN_IMAGE_STORE : AMDGPUGenericInstruction { 3552 let OutOperandList = (outs); 3553 let InOperandList = (ins unknown:$intrin, variable_ops); 3554 let hasSideEffects = 0; 3555 let mayStore = 1; 3556} 3557 3558def G_AMDGPU_INTRIN_IMAGE_STORE_D16 : AMDGPUGenericInstruction { 3559 let OutOperandList = (outs); 3560 let InOperandList = (ins unknown:$intrin, variable_ops); 3561 let hasSideEffects = 0; 3562 let mayStore = 1; 3563} 3564 3565def G_AMDGPU_INTRIN_BVH_INTERSECT_RAY : AMDGPUGenericInstruction { 3566 let OutOperandList = (outs type0:$dst); 3567 let InOperandList = (ins unknown:$intrin, variable_ops); 3568 let hasSideEffects = 0; 3569 let mayLoad = 1; 3570 let mayStore = 0; 3571} 3572 3573// Generic instruction for SI_CALL, so we can select the register bank and insert a waterfall loop 3574// if necessary. 3575def G_SI_CALL : AMDGPUGenericInstruction { 3576 let OutOperandList = (outs SReg_64:$dst); 3577 let InOperandList = (ins type0:$src0, unknown:$callee); 3578 let Size = 4; 3579 let isCall = 1; 3580 let UseNamedOperandTable = 1; 3581 let SchedRW = [WriteBranch]; 3582 // TODO: Should really base this on the call target 3583 let isConvergent = 1; 3584} 3585 3586def G_FPTRUNC_ROUND_UPWARD : AMDGPUGenericInstruction { 3587 let OutOperandList = (outs type0:$vdst); 3588 let InOperandList = (ins type1:$src0); 3589 let hasSideEffects = 0; 3590} 3591 3592def G_FPTRUNC_ROUND_DOWNWARD : AMDGPUGenericInstruction { 3593 let OutOperandList = (outs type0:$vdst); 3594 let InOperandList = (ins type1:$src0); 3595 let hasSideEffects = 0; 3596} 3597 3598//============================================================================// 3599// Dummy Instructions 3600//============================================================================// 3601 3602def V_ILLEGAL : Enc32, InstSI<(outs), (ins), "v_illegal"> { 3603 let Inst{31-0} = 0x00000000; 3604 let FixedSize = 1; 3605 let Size = 4; 3606 let Uses = [EXEC]; 3607 let hasSideEffects = 1; 3608 let SubtargetPredicate = isGFX10Plus; 3609} 3610