Lines Matching +full:custom +full:- +full:temp
1 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// Custom DAG lowering for R600
12 //===----------------------------------------------------------------------===//
43 computeRegisterProperties(Subtarget->getRegisterInfo()); in R600TargetLowering()
46 setOperationAction(ISD::LOAD, {MVT::i32, MVT::v2i32, MVT::v4i32}, Custom); in R600TargetLowering()
49 // spaces, so it is custom lowered to handle those where it isn't. in R600TargetLowering()
53 setLoadExtAction(Op, VT, MVT::i8, Custom); in R600TargetLowering()
54 setLoadExtAction(Op, VT, MVT::i16, Custom); in R600TargetLowering()
65 Custom); in R600TargetLowering()
67 setTruncStoreAction(MVT::i32, MVT::i8, Custom); in R600TargetLowering()
68 setTruncStoreAction(MVT::i32, MVT::i16, Custom); in R600TargetLowering()
71 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom); in R600TargetLowering()
72 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Custom); in R600TargetLowering()
73 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Custom); in R600TargetLowering()
74 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Custom); in R600TargetLowering()
75 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Custom); in R600TargetLowering()
76 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom); in R600TargetLowering()
77 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom); in R600TargetLowering()
78 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Custom); in R600TargetLowering()
79 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Custom); in R600TargetLowering()
80 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Custom); in R600TargetLowering()
95 setOperationAction({ISD::FCOS, ISD::FSIN}, MVT::f32, Custom); in R600TargetLowering()
100 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in R600TargetLowering()
105 MVT::f64, Custom); in R600TargetLowering()
107 setOperationAction(ISD::SELECT_CC, {MVT::f32, MVT::i32}, Custom); in R600TargetLowering()
111 Custom); in R600TargetLowering()
118 if (Subtarget->hasCARRY()) in R600TargetLowering()
119 setOperationAction(ISD::UADDO, MVT::i32, Custom); in R600TargetLowering()
121 if (Subtarget->hasBORROW()) in R600TargetLowering()
122 setOperationAction(ISD::USUBO, MVT::i32, Custom); in R600TargetLowering()
125 if (!Subtarget->hasBFE()) in R600TargetLowering()
130 if (!Subtarget->hasBFE()) in R600TargetLowering()
134 if (!Subtarget->hasBFE()) in R600TargetLowering()
143 setOperationAction(ISD::FrameIndex, MVT::i32, Custom); in R600TargetLowering()
146 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Custom); in R600TargetLowering()
149 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Custom); in R600TargetLowering()
151 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32 in R600TargetLowering()
152 // to be Legal/Custom in order to avoid library calls. in R600TargetLowering()
154 Custom); in R600TargetLowering()
156 if (!Subtarget->hasFMA()) in R600TargetLowering()
162 if (!Subtarget->hasBFI()) in R600TargetLowering()
166 if (!Subtarget->hasBCNT(32)) in R600TargetLowering()
169 if (!Subtarget->hasBCNT(64)) in R600TargetLowering()
172 if (Subtarget->hasFFBH()) in R600TargetLowering()
173 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom); in R600TargetLowering()
175 if (Subtarget->hasFFBL()) in R600TargetLowering()
176 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom); in R600TargetLowering()
180 if (Subtarget->hasBFE()) in R600TargetLowering()
183 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); in R600TargetLowering()
184 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom); in R600TargetLowering()
195 // We need to custom lower some of the intrinsics in R600TargetLowering()
197 Custom); in R600TargetLowering()
206 if (std::next(I) == I->getParent()->end()) in isEOP()
208 return std::next(I)->getOpcode() == R600::RETURN; in isEOP()
214 MachineFunction *MF = BB->getParent(); in EmitInstrWithCustomInserter()
215 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitInstrWithCustomInserter()
217 const R600InstrInfo *TII = Subtarget->getInstrInfo(); in EmitInstrWithCustomInserter()
223 if (TII->isLDSRetInstr(MI.getOpcode())) { in EmitInstrWithCustomInserter()
224 int DstIdx = TII->getOperandIdx(MI.getOpcode(), R600::OpName::dst); in EmitInstrWithCustomInserter()
225 assert(DstIdx != -1); in EmitInstrWithCustomInserter()
233 NewMI = BuildMI(*BB, I, BB->findDebugLoc(I), in EmitInstrWithCustomInserter()
234 TII->get(R600::getLDSNoRetOp(MI.getOpcode()))); in EmitInstrWithCustomInserter()
243 MachineInstr *NewMI = TII->buildDefaultInstruction( in EmitInstrWithCustomInserter()
246 TII->addFlag(*NewMI, 0, MO_FLAG_ABS); in EmitInstrWithCustomInserter()
251 MachineInstr *NewMI = TII->buildDefaultInstruction( in EmitInstrWithCustomInserter()
254 TII->addFlag(*NewMI, 0, MO_FLAG_NEG); in EmitInstrWithCustomInserter()
262 TII->addFlag(*defInstr, 0, MO_FLAG_MASK); in EmitInstrWithCustomInserter()
267 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), MI.getOperand(1) in EmitInstrWithCustomInserter()
269 ->getValueAPF() in EmitInstrWithCustomInserter()
275 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), in EmitInstrWithCustomInserter()
281 auto MIB = TII->buildDefaultInstruction( in EmitInstrWithCustomInserter()
283 int Idx = TII->getOperandIdx(*MIB, R600::OpName::literal); in EmitInstrWithCustomInserter()
286 MIB->getOperand(Idx).ChangeToGA(MO.getGlobal(), MO.getOffset(), in EmitInstrWithCustomInserter()
292 MachineInstr *NewMI = TII->buildDefaultInstruction( in EmitInstrWithCustomInserter()
294 TII->setImmOperand(*NewMI, R600::OpName::src0_sel, in EmitInstrWithCustomInserter()
302 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode())) in EmitInstrWithCustomInserter()
309 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode())) in EmitInstrWithCustomInserter()
317 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP)) in EmitInstrWithCustomInserter()
323 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X), in EmitInstrWithCustomInserter()
328 TII->addFlag(*NewMI, 0, MO_FLAG_PUSH); in EmitInstrWithCustomInserter()
329 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND)) in EmitInstrWithCustomInserter()
337 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X), in EmitInstrWithCustomInserter()
342 TII->addFlag(*NewMI, 0, MO_FLAG_PUSH); in EmitInstrWithCustomInserter()
343 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND)) in EmitInstrWithCustomInserter()
355 EndBlock = BB->end(); NextExportInst != EndBlock; in EmitInstrWithCustomInserter()
357 if (NextExportInst->getOpcode() == R600::EG_ExportSwz || in EmitInstrWithCustomInserter()
358 NextExportInst->getOpcode() == R600::R600_ExportSwz) { in EmitInstrWithCustomInserter()
359 unsigned CurrentInstExportType = NextExportInst->getOperand(1) in EmitInstrWithCustomInserter()
371 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode())) in EmitInstrWithCustomInserter()
392 //===----------------------------------------------------------------------===//
393 // Custom DAG Lowering Operations
394 //===----------------------------------------------------------------------===//
415 Result.getNode()->getNumValues() == 2) && in LowerOperation()
582 switch (N->getOpcode()) { in ReplaceNodeResults()
587 if (N->getValueType(0) == MVT::i1) { in ReplaceNodeResults()
588 Results.push_back(lowerFP_TO_UINT(N->getOperand(0), DAG)); in ReplaceNodeResults()
596 if (N->getValueType(0) == MVT::i1) { in ReplaceNodeResults()
597 Results.push_back(lowerFP_TO_SINT(N->getOperand(0), DAG)); in ReplaceNodeResults()
672 if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS) in LowerGlobalAddress()
676 const GlobalValue *GV = GSD->getGlobal(); in LowerGlobalAddress()
684 // On hw >= R700, COS/SIN input must be between -1. and 1. in LowerTrig()
685 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5) in LowerTrig()
690 // TODO: Should this propagate fast-math-flags? in LowerTrig()
709 DAG.getConstantFP(-0.5, DL, MVT::f32))); in LowerTrig()
712 // On R600 hw, COS/SIN input must be between -Pi and Pi. in LowerTrig()
758 Op, DAG.getConstantFP(-1.0f, DL, MVT::f32), in lowerFP_TO_SINT()
769 // We shouldn't be using an offset wider than 16-bits for implicit parameters. in LowerImplicitParameter()
779 return Cst->isZero(); in isZero()
781 return CstFP->isZero(); in isZero()
787 return CFP->isExactlyValue(1.0); in isHWTrueValue()
794 return CFP->getValueAPF().isZero(); in isHWFalseValue()
808 SDValue Temp; in LowerSELECT_CC() local
826 // select_cc f32, f32, -1, 0, cc_supported in LowerSELECT_CC()
828 // select_cc i32, i32, -1, 0, cc_supported in LowerSELECT_CC()
833 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerSELECT_CC()
866 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerSELECT_CC()
886 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerSELECT_CC()
901 Temp = True; in LowerSELECT_CC()
903 False = Temp; in LowerSELECT_CC()
923 HWTrue = DAG.getConstant(-1, DL, CompareVT); in LowerSELECT_CC()
949 unsigned SrcAS = ASC->getSrcAddressSpace(); in lowerADDRSPACECAST()
950 unsigned DestAS = ASC->getDestAddressSpace(); in lowerADDRSPACECAST()
958 /// LLVM generates byte-addressed pointers. For indirect addressing, we need to
960 /// 16 bytes, (4 x 32bit sub-register), but we need to take into account the
961 /// \p StackWidth, which tells us how many of the 4 sub-registers will be used
1018 assert(Store->isTruncatingStore() in lowerPrivateTruncStore()
1019 || Store->getValue().getValueType() == MVT::i8); in lowerPrivateTruncStore()
1020 assert(Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS); in lowerPrivateTruncStore()
1023 if (Store->getMemoryVT() == MVT::i8) { in lowerPrivateTruncStore()
1024 assert(Store->getAlign() >= 1); in lowerPrivateTruncStore()
1026 } else if (Store->getMemoryVT() == MVT::i16) { in lowerPrivateTruncStore()
1027 assert(Store->getAlign() >= 2); in lowerPrivateTruncStore()
1033 SDValue OldChain = Store->getChain(); in lowerPrivateTruncStore()
1036 SDValue Chain = VectorTrunc ? OldChain->getOperand(0) : OldChain; in lowerPrivateTruncStore()
1037 SDValue BasePtr = Store->getBasePtr(); in lowerPrivateTruncStore()
1038 SDValue Offset = Store->getOffset(); in lowerPrivateTruncStore()
1039 EVT MemVT = Store->getMemoryVT(); in lowerPrivateTruncStore()
1067 // it also handles sub i32 non-truncating stores (like i1) in lowerPrivateTruncStore()
1069 Store->getValue()); in lowerPrivateTruncStore()
1106 unsigned AS = StoreNode->getAddressSpace(); in LowerSTORE()
1108 SDValue Chain = StoreNode->getChain(); in LowerSTORE()
1109 SDValue Ptr = StoreNode->getBasePtr(); in LowerSTORE()
1110 SDValue Value = StoreNode->getValue(); in LowerSTORE()
1113 EVT MemVT = StoreNode->getMemoryVT(); in LowerSTORE()
1118 const bool TruncatingStore = StoreNode->isTruncatingStore(); in LowerSTORE()
1129 NewChain, DL, Value, Ptr, StoreNode->getPointerInfo(), MemVT, in LowerSTORE()
1130 StoreNode->getAlign(), StoreNode->getMemOperand()->getFlags(), in LowerSTORE()
1131 StoreNode->getAAInfo()); in LowerSTORE()
1138 Align Alignment = StoreNode->getAlign(); in LowerSTORE()
1141 StoreNode->getMemOperand()->getFlags(), in LowerSTORE()
1159 assert(StoreNode->getAlign() >= 2); in LowerSTORE()
1175 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32 in LowerSTORE()
1186 Op->getVTList(), Args, MemVT, in LowerSTORE()
1187 StoreNode->getMemOperand()); in LowerSTORE()
1189 if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(MVT::i32)) { in LowerSTORE()
1193 if (StoreNode->isIndexed()) { in LowerSTORE()
1196 Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); in LowerSTORE()
1213 return DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand()); in LowerSTORE()
1257 return -1; in ConstantAddressBlock()
1265 ISD::LoadExtType ExtType = Load->getExtensionType(); in lowerPrivateExtLoad()
1266 EVT MemVT = Load->getMemoryVT(); in lowerPrivateExtLoad()
1267 assert(Load->getAlign() >= MemVT.getStoreSize()); in lowerPrivateExtLoad()
1269 SDValue BasePtr = Load->getBasePtr(); in lowerPrivateExtLoad()
1270 SDValue Chain = Load->getChain(); in lowerPrivateExtLoad()
1271 SDValue Offset = Load->getOffset(); in lowerPrivateExtLoad()
1319 unsigned AS = LoadNode->getAddressSpace(); in LowerLOAD()
1320 EVT MemVT = LoadNode->getMemoryVT(); in LowerLOAD()
1321 ISD::LoadExtType ExtType = LoadNode->getExtensionType(); in LowerLOAD()
1330 SDValue Chain = LoadNode->getChain(); in LowerLOAD()
1331 SDValue Ptr = LoadNode->getBasePtr(); in LowerLOAD()
1333 if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || in LowerLOAD()
1334 LoadNode->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) && in LowerLOAD()
1342 int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace()); in LowerLOAD()
1343 if (ConstantBlock > -1 && in LowerLOAD()
1344 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) || in LowerLOAD()
1345 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) { in LowerLOAD()
1347 if (isa<Constant>(LoadNode->getMemOperand()->getValue()) || in LowerLOAD()
1349 return constBufferLoad(LoadNode, LoadNode->getAddressSpace(), DAG); in LowerLOAD()
1352 // non-constant ptr can't be folded, keeps it as a v4f32 load in LowerLOAD()
1356 DAG.getConstant(LoadNode->getAddressSpace() - in LowerLOAD()
1379 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { in LowerLOAD()
1382 ISD::EXTLOAD, DL, VT, Chain, Ptr, LoadNode->getPointerInfo(), MemVT, in LowerLOAD()
1383 LoadNode->getAlign(), LoadNode->getMemOperand()->getFlags()); in LowerLOAD()
1391 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) { in LowerLOAD()
1400 return DAG.getLoad(MVT::i32, DL, Chain, Ptr, LoadNode->getMemOperand()); in LowerLOAD()
1417 const R600FrameLowering *TFL = Subtarget->getFrameLowering(); in lowerFrameIndex()
1421 unsigned FrameIndex = FIN->getIndex(); in lowerFrameIndex()
1424 TFL->getFrameIndexReference(MF, FrameIndex, IgnoredFrameReg); in lowerFrameIndex()
1425 return DAG.getConstant(Offset.getFixed() * 4 * TFL->getStackWidth(MF), in lowerFrameIndex()
1453 /// separate calling conventions for kernel and non-kernel functions.
1504 // XXX - I think PartOffset should give you this, but it seems to give the in LowerFormalArguments()
1579 if (C->isZero()) { in CompactSwizzlableVector()
1582 } else if (C->isExactlyValue(1.0)) { in CompactSwizzlableVector()
1645 // Old -> New swizzle values in OptimizeSwizzle()
1650 unsigned Idx = Swz[i]->getAsZExtVal(); in OptimizeSwizzle()
1658 unsigned Idx = Swz[i]->getAsZExtVal(); in OptimizeSwizzle()
1669 EVT VT = LoadNode->getValueType(0); in constBufferLoad()
1670 SDValue Chain = LoadNode->getChain(); in constBufferLoad()
1671 SDValue Ptr = LoadNode->getBasePtr(); in constBufferLoad()
1675 if (LoadNode->getMemoryVT().getScalarType() != MVT::i32 || !ISD::isNON_EXTLoad(LoadNode)) in constBufferLoad()
1678 if (LoadNode->getAlign() < Align(4)) in constBufferLoad()
1712 //===----------------------------------------------------------------------===//
1713 // Custom DAG Optimizations
1714 //===----------------------------------------------------------------------===//
1721 switch (N->getOpcode()) { in PerformDAGCombine()
1722 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a) in PerformDAGCombine()
1724 SDValue Arg = N->getOperand(0); in PerformDAGCombine()
1726 return DAG.getNode(ISD::UINT_TO_FP, DL, N->getValueType(0), in PerformDAGCombine()
1732 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) -> in PerformDAGCombine()
1733 // (i32 select_cc f32, f32, -1, 0 cc) in PerformDAGCombine()
1738 SDValue FNeg = N->getOperand(0); in PerformDAGCombine()
1751 return DAG.getNode(ISD::SELECT_CC, DL, N->getValueType(0), in PerformDAGCombine()
1754 DAG.getConstant(-1, DL, MVT::i32), // True in PerformDAGCombine()
1762 SDValue InVec = N->getOperand(0); in PerformDAGCombine()
1763 SDValue InVal = N->getOperand(1); in PerformDAGCombine()
1764 SDValue EltNo = N->getOperand(2); in PerformDAGCombine()
1779 unsigned Elt = EltNo->getAsZExtVal(); in PerformDAGCombine()
1786 Ops.append(InVec.getNode()->op_begin(), in PerformDAGCombine()
1787 InVec.getNode()->op_end()); in PerformDAGCombine()
1811 // Extract_vec (Build_vector) generated by custom lowering in PerformDAGCombine()
1814 SDValue Arg = N->getOperand(0); in PerformDAGCombine()
1816 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { in PerformDAGCombine()
1817 unsigned Element = Const->getZExtValue(); in PerformDAGCombine()
1818 return Arg->getOperand(Element); in PerformDAGCombine()
1825 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) { in PerformDAGCombine()
1826 unsigned Element = Const->getZExtValue(); in PerformDAGCombine()
1827 return DAG.getNode(ISD::BITCAST, DL, N->getVTList(), in PerformDAGCombine()
1828 Arg->getOperand(0).getOperand(Element)); in PerformDAGCombine()
1839 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq -> in PerformDAGCombine()
1842 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne -> in PerformDAGCombine()
1844 SDValue LHS = N->getOperand(0); in PerformDAGCombine()
1849 SDValue RHS = N->getOperand(1); in PerformDAGCombine()
1850 SDValue True = N->getOperand(2); in PerformDAGCombine()
1851 SDValue False = N->getOperand(3); in PerformDAGCombine()
1852 ISD::CondCode NCC = cast<CondCodeSDNode>(N->getOperand(4))->get(); in PerformDAGCombine()
1864 ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); in PerformDAGCombine()
1881 SDValue Arg = N->getOperand(1); in PerformDAGCombine()
1886 N->getOperand(0), // Chain in PerformDAGCombine()
1888 N->getOperand(2), // ArrayBase in PerformDAGCombine()
1889 N->getOperand(3), // Type in PerformDAGCombine()
1890 N->getOperand(4), // SWZ_X in PerformDAGCombine()
1891 N->getOperand(5), // SWZ_Y in PerformDAGCombine()
1892 N->getOperand(6), // SWZ_Z in PerformDAGCombine()
1893 N->getOperand(7) // SWZ_W in PerformDAGCombine()
1895 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL); in PerformDAGCombine()
1896 return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, N->getVTList(), NewArgs); in PerformDAGCombine()
1899 SDValue Arg = N->getOperand(1); in PerformDAGCombine()
1904 N->getOperand(0), in PerformDAGCombine()
1905 N->getOperand(1), in PerformDAGCombine()
1906 N->getOperand(2), in PerformDAGCombine()
1907 N->getOperand(3), in PerformDAGCombine()
1908 N->getOperand(4), in PerformDAGCombine()
1909 N->getOperand(5), in PerformDAGCombine()
1910 N->getOperand(6), in PerformDAGCombine()
1911 N->getOperand(7), in PerformDAGCombine()
1912 N->getOperand(8), in PerformDAGCombine()
1913 N->getOperand(9), in PerformDAGCombine()
1914 N->getOperand(10), in PerformDAGCombine()
1915 N->getOperand(11), in PerformDAGCombine()
1916 N->getOperand(12), in PerformDAGCombine()
1917 N->getOperand(13), in PerformDAGCombine()
1918 N->getOperand(14), in PerformDAGCombine()
1919 N->getOperand(15), in PerformDAGCombine()
1920 N->getOperand(16), in PerformDAGCombine()
1921 N->getOperand(17), in PerformDAGCombine()
1922 N->getOperand(18), in PerformDAGCombine()
1924 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL); in PerformDAGCombine()
1925 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs); in PerformDAGCombine()
1930 SDValue Ptr = LoadNode->getBasePtr(); in PerformDAGCombine()
1931 if (LoadNode->getAddressSpace() == AMDGPUAS::PARAM_I_ADDRESS && in PerformDAGCombine()
1947 const R600InstrInfo *TII = Subtarget->getInstrInfo(); in FoldOperand()
1965 unsigned Opcode = ParentNode->getMachineOpcode(); in FoldOperand()
1966 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1; in FoldOperand()
1972 if (ParentNode->getValueType(0).isVector()) in FoldOperand()
1977 TII->getOperandIdx(Opcode, R600::OpName::src0), in FoldOperand()
1978 TII->getOperandIdx(Opcode, R600::OpName::src1), in FoldOperand()
1979 TII->getOperandIdx(Opcode, R600::OpName::src2), in FoldOperand()
1980 TII->getOperandIdx(Opcode, R600::OpName::src0_X), in FoldOperand()
1981 TII->getOperandIdx(Opcode, R600::OpName::src0_Y), in FoldOperand()
1982 TII->getOperandIdx(Opcode, R600::OpName::src0_Z), in FoldOperand()
1983 TII->getOperandIdx(Opcode, R600::OpName::src0_W), in FoldOperand()
1984 TII->getOperandIdx(Opcode, R600::OpName::src1_X), in FoldOperand()
1985 TII->getOperandIdx(Opcode, R600::OpName::src1_Y), in FoldOperand()
1986 TII->getOperandIdx(Opcode, R600::OpName::src1_Z), in FoldOperand()
1987 TII->getOperandIdx(Opcode, R600::OpName::src1_W) in FoldOperand()
1991 int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx); in FoldOperand()
1995 OtherSrcIdx--; in FoldOperand()
1996 OtherSelIdx--; in FoldOperand()
1999 dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) { in FoldOperand()
2000 if (Reg->getReg() == R600::ALU_CONST) { in FoldOperand()
2001 Consts.push_back(ParentNode->getConstantOperandVal(OtherSelIdx)); in FoldOperand()
2007 Consts.push_back(Cst->getZExtValue()); in FoldOperand()
2008 if (!TII->fitsConstReadLimitations(Consts)) { in FoldOperand()
2018 if (Imm->getAsZExtVal()) in FoldOperand()
2030 float FloatValue = FPC->getValueAPF().convertToFloat(); in FoldOperand()
2038 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue(); in FoldOperand()
2058 if (C->getZExtValue()) in FoldOperand()
2073 const R600InstrInfo *TII = Subtarget->getInstrInfo(); in PostISelFolding()
2074 if (!Node->isMachineOpcode()) in PostISelFolding()
2077 unsigned Opcode = Node->getMachineOpcode(); in PostISelFolding()
2080 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end()); in PostISelFolding()
2084 TII->getOperandIdx(Opcode, R600::OpName::src0_X), in PostISelFolding()
2085 TII->getOperandIdx(Opcode, R600::OpName::src0_Y), in PostISelFolding()
2086 TII->getOperandIdx(Opcode, R600::OpName::src0_Z), in PostISelFolding()
2087 TII->getOperandIdx(Opcode, R600::OpName::src0_W), in PostISelFolding()
2088 TII->getOperandIdx(Opcode, R600::OpName::src1_X), in PostISelFolding()
2089 TII->getOperandIdx(Opcode, R600::OpName::src1_Y), in PostISelFolding()
2090 TII->getOperandIdx(Opcode, R600::OpName::src1_Z), in PostISelFolding()
2091 TII->getOperandIdx(Opcode, R600::OpName::src1_W) in PostISelFolding()
2094 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_X), in PostISelFolding()
2095 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Y), in PostISelFolding()
2096 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Z), in PostISelFolding()
2097 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_W), in PostISelFolding()
2098 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_X), in PostISelFolding()
2099 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Y), in PostISelFolding()
2100 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Z), in PostISelFolding()
2101 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_W) in PostISelFolding()
2104 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_X), in PostISelFolding()
2105 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Y), in PostISelFolding()
2106 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Z), in PostISelFolding()
2107 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_W), in PostISelFolding()
2108 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_X), in PostISelFolding()
2109 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Y), in PostISelFolding()
2110 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Z), in PostISelFolding()
2111 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_W) in PostISelFolding()
2116 SDValue &Src = Ops[OperandIdx[i] - 1]; in PostISelFolding()
2117 SDValue &Neg = Ops[NegIdx[i] - 1]; in PostISelFolding()
2118 SDValue &Abs = Ops[AbsIdx[i] - 1]; in PostISelFolding()
2119 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1; in PostISelFolding()
2120 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); in PostISelFolding()
2122 SelIdx--; in PostISelFolding()
2123 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; in PostISelFolding()
2125 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); in PostISelFolding()
2128 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) { in PostISelFolding()
2131 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); in PostISelFolding()
2134 if (!TII->hasInstrModifiers(Opcode)) in PostISelFolding()
2137 TII->getOperandIdx(Opcode, R600::OpName::src0), in PostISelFolding()
2138 TII->getOperandIdx(Opcode, R600::OpName::src1), in PostISelFolding()
2139 TII->getOperandIdx(Opcode, R600::OpName::src2) in PostISelFolding()
2142 TII->getOperandIdx(Opcode, R600::OpName::src0_neg), in PostISelFolding()
2143 TII->getOperandIdx(Opcode, R600::OpName::src1_neg), in PostISelFolding()
2144 TII->getOperandIdx(Opcode, R600::OpName::src2_neg) in PostISelFolding()
2147 TII->getOperandIdx(Opcode, R600::OpName::src0_abs), in PostISelFolding()
2148 TII->getOperandIdx(Opcode, R600::OpName::src1_abs), in PostISelFolding()
2149 -1 in PostISelFolding()
2154 SDValue &Src = Ops[OperandIdx[i] - 1]; in PostISelFolding()
2155 SDValue &Neg = Ops[NegIdx[i] - 1]; in PostISelFolding()
2157 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs; in PostISelFolding()
2158 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1; in PostISelFolding()
2159 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]); in PostISelFolding()
2160 int ImmIdx = TII->getOperandIdx(Opcode, R600::OpName::literal); in PostISelFolding()
2162 SelIdx--; in PostISelFolding()
2163 ImmIdx--; in PostISelFolding()
2165 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp; in PostISelFolding()
2168 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops); in PostISelFolding()
2177 switch (RMW->getOperation()) { in shouldExpandAtomicRMWInIR()