Lines Matching +full:high +full:- +full:vt
1 //===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
31 #define DEBUG_TYPE "aarch64-isel"
34 //===--------------------------------------------------------------------===//
35 /// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
42 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
60 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
66 template <signed Low, signed High, signed Scale>
137 int64_t C = CI->getSExtValue(); in SelectAddrModeIndexedUImm()
145 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64); in SelectAddrModeIndexedUImm()
162 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST) in SelectExtractHigh()
163 N = N->getOperand(0); in SelectExtractHigh()
164 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR || in SelectExtractHigh()
165 !isa<ConstantSDNode>(N->getOperand(1))) in SelectExtractHigh()
167 EVT VT = N->getValueType(0); in SelectExtractHigh() local
168 EVT LVT = N->getOperand(0).getValueType(); in SelectExtractHigh()
169 unsigned Index = N->getConstantOperandVal(1); in SelectExtractHigh()
170 if (!VT.is64BitVector() || !LVT.is128BitVector() || in SelectExtractHigh()
171 Index != VT.getVectorNumElements()) in SelectExtractHigh()
173 Res = N->getOperand(0); in SelectExtractHigh()
180 SDValue Op = N->getOperand(0); in SelectRoundingVLShr()
181 EVT VT = Op.getValueType(); in SelectRoundingVLShr() local
182 unsigned ShtAmt = N->getConstantOperandVal(1); in SelectRoundingVLShr()
183 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD) in SelectRoundingVLShr()
188 Imm = APInt(VT.getScalarSizeInBits(), in SelectRoundingVLShr()
193 Imm = APInt(VT.getScalarSizeInBits(), in SelectRoundingVLShr()
198 if (Imm != 1ULL << (ShtAmt - 1)) in SelectRoundingVLShr()
202 Res2 = CurDAG->getTargetConstant(ShtAmt, SDLoc(N), MVT::i32); in SelectRoundingVLShr()
207 switch(N->getOpcode()) { in SelectDupZeroOrUndef()
212 auto Opnd0 = N->getOperand(0); in SelectDupZeroOrUndef()
227 switch(N->getOpcode()) { in SelectDupZero()
230 auto Opnd0 = N->getOperand(0); in SelectDupZero()
243 switch(N->getOpcode()) { in SelectDupNegativeZero()
246 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(N->getOperand(0)); in SelectDupNegativeZero()
247 return Const && Const->isZero() && Const->isNegative(); in SelectDupNegativeZero()
254 template<MVT::SimpleValueType VT>
256 return SelectSVEAddSubImm(N, VT, Imm, Shift); in SelectSVEAddSubImm()
259 template <MVT::SimpleValueType VT, bool Negate>
261 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate); in SelectSVEAddSubSSatImm()
264 template <MVT::SimpleValueType VT>
266 return SelectSVECpyDupImm(N, VT, Imm, Shift); in SelectSVECpyDupImm()
269 template <MVT::SimpleValueType VT, bool Invert = false>
271 return SelectSVELogicalImm(N, VT, Imm, Invert); in SelectSVELogicalImm()
274 template <MVT::SimpleValueType VT>
276 return SelectSVEArithImm(N, VT, Imm); in SelectSVEArithImm()
279 template <unsigned Low, unsigned High, bool AllowSaturation = false>
281 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm); in SelectSVEShiftImm()
285 if (N->getOpcode() != ISD::SPLAT_VECTOR) in SelectSVEShiftSplatImmR()
288 EVT EltVT = N->getValueType(0).getVectorElementType(); in SelectSVEShiftSplatImmR()
289 return SelectSVEShiftImm(N->getOperand(0), /* Low */ 1, in SelectSVEShiftSplatImmR()
290 /* High */ EltVT.getFixedSizeInBits(), in SelectSVEShiftSplatImmR()
300 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue(); in SelectCntImm()
309 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32); in SelectCntImm()
321 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue(); in SelectEXTImm()
325 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32); in SelectEXTImm()
335 uint64_t C = CI->getZExtValue(); in ImmToReg()
340 Imm = CurDAG->getRegister(BaseReg + C, MVT::Other); in ImmToReg()
346 /// Form sequences of consecutive 64/128-bit registers for use in NEON
347 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
357 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
494 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
495 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
497 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
498 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
501 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
504 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
528 /// isIntImmediate - This method tests to see if the node is a constant in INITIALIZE_PASS()
529 /// operand. If so Imm will receive the 32-bit value. in INITIALIZE_PASS()
532 Imm = C->getZExtValue(); in INITIALIZE_PASS()
538 // isIntImmediate - This method tests to see if a constant operand.
544 // isOpcWithIntImmediate - This method tests to see if the node is a specific
549 return N->getOpcode() == Opc && in isOpcWithIntImmediate()
550 isIntImmediate(N->getOperand(1).getNode(), Imm); in isOpcWithIntImmediate()
553 // isIntImmediateEq - This method tests to see if N is a constant operand that
575 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); in SelectInlineAsmMemoryOperand()
576 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF); in SelectInlineAsmMemoryOperand()
578 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64); in SelectInlineAsmMemoryOperand()
580 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, in SelectInlineAsmMemoryOperand()
589 /// SelectArithImmed - Select an immediate value that can be represented as
590 /// a 12-bit value shifted left by either 0 or 12. If so, return true with
591 /// Val set to the 12-bit value and Shift set to the shifter operand.
598 // root-level opcode matching. in SelectArithImmed()
602 uint64_t Immed = N.getNode()->getAsZExtVal(); in SelectArithImmed()
615 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32); in SelectArithImmed()
616 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32); in SelectArithImmed()
620 /// SelectNegArithImmed - As above, but negates the value before trying to
628 // root-level opcode matching. in SelectNegArithImmed()
632 // The immediate operand must be a 24-bit zero-extended immediate. in SelectNegArithImmed()
633 uint64_t Immed = N.getNode()->getAsZExtVal(); in SelectNegArithImmed()
649 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val, in SelectNegArithImmed()
653 /// getShiftTypeForNode - Translate a shift node to the corresponding
678 unsigned ShiftVal = CSD->getZExtValue(); in isWorthFoldingSHL()
682 // Check if this particular node is reused in any non-memory related in isWorthFoldingSHL()
686 for (SDNode *UI : Node->uses()) in isWorthFoldingSHL()
688 for (SDNode *UII : UI->uses()) in isWorthFoldingSHL()
699 if (CurDAG->shouldOptForSize() || V.hasOneUse()) in isWorthFoldingAddr()
703 // costs additional micro-ops. in isWorthFoldingAddr()
704 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16)) in isWorthFoldingAddr()
708 // it's used by a non-address operation. in isWorthFoldingAddr()
724 /// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
728 EVT VT = N.getValueType(); in SelectShiftedRegisterFromAnd() local
729 if (VT != MVT::i32 && VT != MVT::i64) in SelectShiftedRegisterFromAnd()
732 if (N->getOpcode() != ISD::AND || !N->hasOneUse()) in SelectShiftedRegisterFromAnd()
735 if (!LHS->hasOneUse()) in SelectShiftedRegisterFromAnd()
738 unsigned LHSOpcode = LHS->getOpcode(); in SelectShiftedRegisterFromAnd()
746 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue(); in SelectShiftedRegisterFromAnd()
751 APInt AndMask = RHSC->getAPIntValue(); in SelectShiftedRegisterFromAnd()
766 NewShiftC = LowZBits - ShiftAmtC; in SelectShiftedRegisterFromAnd()
767 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri; in SelectShiftedRegisterFromAnd()
777 // SRA need all high bits in SelectShiftedRegisterFromAnd()
781 // SRL high bits can be 0 or 1 in SelectShiftedRegisterFromAnd()
786 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri; in SelectShiftedRegisterFromAnd()
788 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri; in SelectShiftedRegisterFromAnd()
792 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC, DL, VT); in SelectShiftedRegisterFromAnd()
793 SDValue BitWidthMinus1 = CurDAG->getTargetConstant(BitWidth - 1, DL, VT); in SelectShiftedRegisterFromAnd()
794 Reg = SDValue(CurDAG->getMachineNode(NewShiftOp, DL, VT, LHS->getOperand(0), in SelectShiftedRegisterFromAnd()
798 Shift = CurDAG->getTargetConstant(ShVal, DL, MVT::i32); in SelectShiftedRegisterFromAnd()
802 /// getExtendTypeForNode - Translate an extend node to the corresponding
810 SrcVT = cast<VTSDNode>(N.getOperand(1))->getVT(); in getExtendTypeForNode()
820 assert(SrcVT != MVT::i64 && "extend from 64-bits?"); in getExtendTypeForNode()
832 assert(SrcVT != MVT::i64 && "extend from 64-bits?"); in getExtendTypeForNode()
839 uint64_t AndMask = CSD->getZExtValue(); in getExtendTypeForNode()
863 if (CurDAG->shouldOptForSize() || V.hasOneUse()) in isWorthFoldingALU()
868 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL && in isWorthFoldingALU()
877 /// SelectShiftedRegister - Select a "shifted register" operand. If the value
895 unsigned Val = RHS->getZExtValue() & (BitSize - 1); in SelectShiftedRegister()
899 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32); in SelectShiftedRegister()
915 return CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl, MVT::i32, N); in narrowIfNeeded()
919 template<signed Low, signed High, signed Scale>
924 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue(); in SelectRDVLImm()
927 if ((RDVLImm >= Low) && (RDVLImm <= High)) { in SelectRDVLImm()
928 Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32); in SelectRDVLImm()
936 /// SelectArithExtendedRegister - Select a "extended register" operand. This
947 ShiftVal = CSD->getZExtValue(); in SelectArithExtendedRegister()
963 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the in SelectArithExtendedRegister()
972 if (Ext == AArch64_AM::UXTW && Reg->getValueType(0).getSizeInBits() == 32 && in SelectArithExtendedRegister()
980 // there might not be an actual 32-bit value in the program. We can in SelectArithExtendedRegister()
984 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N), in SelectArithExtendedRegister()
989 /// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1002 ShiftVal = CSD->getZExtValue(); in SelectArithUXTXRegister()
1008 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N), in SelectArithUXTXRegister()
1016 /// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1019 for (auto *Use : N->uses()) { in isWorthFoldingADDlow()
1020 if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE && in isWorthFoldingADDlow()
1021 Use->getOpcode() != ISD::ATOMIC_LOAD && in isWorthFoldingADDlow()
1022 Use->getOpcode() != ISD::ATOMIC_STORE) in isWorthFoldingADDlow()
1027 if (isStrongerThanMonotonic(cast<MemSDNode>(Use)->getSuccessOrdering())) in isWorthFoldingADDlow()
1037 if ((Offset & (Size - 1)) == 0 && Offset >= 0 && in isValidAsScaledImmediate()
1043 /// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1051 const DataLayout &DL = CurDAG->getDataLayout(); in SelectAddrModeIndexedBitWidth()
1054 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrModeIndexedBitWidth()
1055 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexedBitWidth()
1056 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
1060 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed in SelectAddrModeIndexedBitWidth()
1062 if (CurDAG->isBaseWithConstantOffset(N)) { in SelectAddrModeIndexedBitWidth()
1065 int64_t RHSC = RHS->getSExtValue(); in SelectAddrModeIndexedBitWidth()
1067 int64_t Range = 0x1LL << (BW - 1); in SelectAddrModeIndexedBitWidth()
1069 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) && in SelectAddrModeIndexedBitWidth()
1073 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrModeIndexedBitWidth()
1074 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexedBitWidth()
1076 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
1081 uint64_t RHSC = RHS->getZExtValue(); in SelectAddrModeIndexedBitWidth()
1085 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) { in SelectAddrModeIndexedBitWidth()
1088 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrModeIndexedBitWidth()
1089 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexedBitWidth()
1091 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
1102 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexedBitWidth()
1106 /// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1112 const DataLayout &DL = CurDAG->getDataLayout(); in SelectAddrModeIndexed()
1115 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrModeIndexed()
1116 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexed()
1117 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexed()
1129 if (GAN->getOffset() % Size == 0 && in SelectAddrModeIndexed()
1130 GAN->getGlobal()->getPointerAlignment(DL) >= Size) in SelectAddrModeIndexed()
1134 if (CurDAG->isBaseWithConstantOffset(N)) { in SelectAddrModeIndexed()
1136 int64_t RHSC = (int64_t)RHS->getZExtValue(); in SelectAddrModeIndexed()
1141 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrModeIndexed()
1142 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexed()
1144 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64); in SelectAddrModeIndexed()
1160 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeIndexed()
1164 /// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1172 if (!CurDAG->isBaseWithConstantOffset(N)) in SelectAddrModeUnscaled()
1175 int64_t RHSC = RHS->getSExtValue(); in SelectAddrModeUnscaled()
1176 if (RHSC >= -256 && RHSC < 256) { in SelectAddrModeUnscaled()
1179 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrModeUnscaled()
1181 Base = CurDAG->getTargetFrameIndex( in SelectAddrModeUnscaled()
1182 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectAddrModeUnscaled()
1184 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64); in SelectAddrModeUnscaled()
1194 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0); in Widen()
1195 return CurDAG->getTargetInsertSubreg(AArch64::sub_32, dl, MVT::i64, ImpDef, in Widen()
1206 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue()) in SelectExtendedSHL()
1217 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl, in SelectExtendedSHL()
1221 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32); in SelectExtendedSHL()
1225 unsigned ShiftVal = CSD->getZExtValue(); in SelectExtendedSHL()
1244 // to the register-immediate addressing modes. in SelectAddrModeWRO()
1248 // Check if this particular node is reused in any non-memory related in SelectAddrModeWRO()
1252 for (SDNode *UI : Node->uses()) { in SelectAddrModeWRO()
1264 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32); in SelectAddrModeWRO()
1272 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32); in SelectAddrModeWRO()
1277 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32); in SelectAddrModeWRO()
1286 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl, in SelectAddrModeWRO()
1298 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl, in SelectAddrModeWRO()
1332 // Check if this particular node is reused in any non-memory related in SelectAddrModeXRO()
1336 for (SDNode *UI : Node->uses()) { in SelectAddrModeXRO()
1353 int64_t ImmOff = (int64_t)RHS->getAsZExtVal(); in SelectAddrModeXRO()
1356 // checked by using -ImmOff). in SelectAddrModeXRO()
1358 isPreferredADD(ImmOff) || isPreferredADD(-ImmOff)) in SelectAddrModeXRO()
1363 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops); in SelectAddrModeXRO()
1366 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV); in SelectAddrModeXRO()
1376 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32); in SelectAddrModeXRO()
1384 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32); in SelectAddrModeXRO()
1388 // Match any non-shifted, non-extend, non-immediate add expression. in SelectAddrModeXRO()
1391 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32); in SelectAddrModeXRO()
1392 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32); in SelectAddrModeXRO()
1440 // There's no special register-class for a vector-list of 1 element: it's just in createTuple()
1453 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32)); in createTuple()
1455 // Then we get pairs of source & subregister-position for the components. in createTuple()
1458 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32)); in createTuple()
1462 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops); in createTuple()
1469 EVT VT = N->getValueType(0); in SelectTable() local
1475 SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Off, in SelectTable()
1476 N->op_begin() + Vec0Off + NumVecs); in SelectTable()
1481 Ops.push_back(N->getOperand(1)); in SelectTable()
1483 Ops.push_back(N->getOperand(NumVecs + ExtOff + 1)); in SelectTable()
1484 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops)); in SelectTable()
1494 // Otherwise, it's either a constant discriminator, or a non-blended in extractPtrauthBlendDiscriminators()
1496 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN && in extractPtrauthBlendDiscriminators()
1497 Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) { in extractPtrauthBlendDiscriminators()
1498 AddrDisc = Disc->getOperand(1); in extractPtrauthBlendDiscriminators()
1499 ConstDisc = Disc->getOperand(2); in extractPtrauthBlendDiscriminators()
1505 // discriminator value) isn't a 16-bit constant, bail out, and let the in extractPtrauthBlendDiscriminators()
1508 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue())) in extractPtrauthBlendDiscriminators()
1509 return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc); in extractPtrauthBlendDiscriminators()
1513 AddrDisc = DAG->getRegister(AArch64::XZR, MVT::i64); in extractPtrauthBlendDiscriminators()
1516 DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64), in extractPtrauthBlendDiscriminators()
1523 SDValue Val = N->getOperand(1); in SelectPtrauthAuth()
1524 SDValue AUTKey = N->getOperand(2); in SelectPtrauthAuth()
1525 SDValue AUTDisc = N->getOperand(3); in SelectPtrauthAuth()
1527 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue(); in SelectPtrauthAuth()
1528 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64); in SelectPtrauthAuth()
1534 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, in SelectPtrauthAuth()
1538 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT, DL, MVT::i64, Ops); in SelectPtrauthAuth()
1546 SDValue Val = N->getOperand(1); in SelectPtrauthResign()
1547 SDValue AUTKey = N->getOperand(2); in SelectPtrauthResign()
1548 SDValue AUTDisc = N->getOperand(3); in SelectPtrauthResign()
1549 SDValue PACKey = N->getOperand(4); in SelectPtrauthResign()
1550 SDValue PACDisc = N->getOperand(5); in SelectPtrauthResign()
1552 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue(); in SelectPtrauthResign()
1553 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue(); in SelectPtrauthResign()
1555 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64); in SelectPtrauthResign()
1556 PACKey = CurDAG->getTargetConstant(PACKeyC, DL, MVT::i64); in SelectPtrauthResign()
1566 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, in SelectPtrauthResign()
1572 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC, DL, MVT::i64, Ops); in SelectPtrauthResign()
1579 if (LD->isUnindexed()) in tryIndexedLoad()
1581 EVT VT = LD->getMemoryVT(); in tryIndexedLoad() local
1582 EVT DstVT = N->getValueType(0); in tryIndexedLoad()
1583 ISD::MemIndexedMode AM = LD->getAddressingMode(); in tryIndexedLoad()
1591 ISD::LoadExtType ExtType = LD->getExtensionType(); in tryIndexedLoad()
1593 if (VT == MVT::i64) in tryIndexedLoad()
1595 else if (VT == MVT::i32) { in tryIndexedLoad()
1607 } else if (VT == MVT::i16) { in tryIndexedLoad()
1620 } else if (VT == MVT::i8) { in tryIndexedLoad()
1633 } else if (VT == MVT::f16) { in tryIndexedLoad()
1635 } else if (VT == MVT::bf16) { in tryIndexedLoad()
1637 } else if (VT == MVT::f32) { in tryIndexedLoad()
1639 } else if (VT == MVT::f64 || VT.is64BitVector()) { in tryIndexedLoad()
1641 } else if (VT.is128BitVector()) { in tryIndexedLoad()
1645 SDValue Chain = LD->getChain(); in tryIndexedLoad()
1646 SDValue Base = LD->getBasePtr(); in tryIndexedLoad()
1647 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset()); in tryIndexedLoad()
1648 int OffsetVal = (int)OffsetOp->getZExtValue(); in tryIndexedLoad()
1650 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64); in tryIndexedLoad()
1652 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT, in tryIndexedLoad()
1656 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in tryIndexedLoad()
1657 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {MemOp}); in tryIndexedLoad()
1662 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32); in tryIndexedLoad()
1664 SDValue(CurDAG->getMachineNode( in tryIndexedLoad()
1666 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal, in tryIndexedLoad()
1674 CurDAG->RemoveDeadNode(N); in tryIndexedLoad()
1681 EVT VT = N->getValueType(0); in SelectLoad() local
1682 SDValue Chain = N->getOperand(0); in SelectLoad()
1684 SDValue Ops[] = {N->getOperand(2), // Mem operand; in SelectLoad()
1689 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectLoad()
1693 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg)); in SelectLoad()
1700 MachineMemOperand *MemOp = MemIntr->getMemOperand(); in SelectLoad()
1701 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp}); in SelectLoad()
1704 CurDAG->RemoveDeadNode(N); in SelectLoad()
1710 EVT VT = N->getValueType(0); in SelectPostLoad() local
1711 SDValue Chain = N->getOperand(0); in SelectPostLoad()
1713 SDValue Ops[] = {N->getOperand(1), // Mem operand in SelectPostLoad()
1714 N->getOperand(2), // Incremental in SelectPostLoad()
1720 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectPostLoad()
1732 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg)); in SelectPostLoad()
1736 CurDAG->RemoveDeadNode(N); in SelectPostLoad()
1751 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>( in findAddrModeSVELoadStore()
1771 /// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
1774 static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) { in SelectOpcodeFromVT() argument
1776 if (!VT.isScalableVector()) in SelectOpcodeFromVT()
1779 EVT EltVT = VT.getVectorElementType(); in SelectOpcodeFromVT()
1780 unsigned Key = VT.getVectorMinNumElements(); in SelectOpcodeFromVT()
1804 case 16: // 8-bit or bf16 in SelectOpcodeFromVT()
1807 case 8: // 16-bit in SelectOpcodeFromVT()
1810 case 4: // 32-bit in SelectOpcodeFromVT()
1813 case 2: // 64-bit in SelectOpcodeFromVT()
1828 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(N->getOperand(2))) in SelectPExtPair()
1829 if (Imm->getZExtValue() > 1) in SelectPExtPair()
1833 EVT VT = N->getValueType(0); in SelectPExtPair() local
1834 SDValue Ops[] = {N->getOperand(1), N->getOperand(2)}; in SelectPExtPair()
1835 SDNode *WhilePair = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops); in SelectPExtPair()
1839 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg( in SelectPExtPair()
1840 AArch64::psub0 + I, DL, VT, SuperReg)); in SelectPExtPair()
1842 CurDAG->RemoveDeadNode(N); in SelectPExtPair()
1847 EVT VT = N->getValueType(0); in SelectWhilePair() local
1849 SDValue Ops[] = {N->getOperand(1), N->getOperand(2)}; in SelectWhilePair()
1851 SDNode *WhilePair = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops); in SelectWhilePair()
1855 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg( in SelectWhilePair()
1856 AArch64::psub0 + I, DL, VT, SuperReg)); in SelectWhilePair()
1858 CurDAG->RemoveDeadNode(N); in SelectWhilePair()
1863 EVT VT = N->getValueType(0); in SelectCVTIntrinsic() local
1864 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); in SelectCVTIntrinsic()
1867 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, Ops); in SelectCVTIntrinsic()
1870 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg( in SelectCVTIntrinsic()
1871 AArch64::zsub0 + i, DL, VT, SuperReg)); in SelectCVTIntrinsic()
1873 CurDAG->RemoveDeadNode(N); in SelectCVTIntrinsic()
1884 EVT VT = N->getValueType(0); in SelectDestructiveMultiIntrinsic() local
1888 SmallVector<SDValue, 4> Regs(N->op_begin() + StartIdx, in SelectDestructiveMultiIntrinsic()
1889 N->op_begin() + StartIdx + NumVecs); in SelectDestructiveMultiIntrinsic()
1899 Zm = N->getOperand(NumVecs + FirstVecIdx); in SelectDestructiveMultiIntrinsic()
1903 Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, in SelectDestructiveMultiIntrinsic()
1904 N->getOperand(1), Zdn, Zm); in SelectDestructiveMultiIntrinsic()
1906 Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, Zdn, Zm); in SelectDestructiveMultiIntrinsic()
1909 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg( in SelectDestructiveMultiIntrinsic()
1910 AArch64::zsub0 + i, DL, VT, SuperReg)); in SelectDestructiveMultiIntrinsic()
1912 CurDAG->RemoveDeadNode(N); in SelectDestructiveMultiIntrinsic()
1920 EVT VT = N->getValueType(0); in SelectPredicatedLoad() local
1921 SDValue Chain = N->getOperand(0); in SelectPredicatedLoad()
1927 N, Opc_rr, Opc_ri, N->getOperand(IsIntr ? 3 : 2), in SelectPredicatedLoad()
1928 CurDAG->getTargetConstant(0, DL, MVT::i64), Scale); in SelectPredicatedLoad()
1930 SDValue Ops[] = {N->getOperand(IsIntr ? 2 : 1), // Predicate in SelectPredicatedLoad()
1936 SDNode *Load = CurDAG->getMachineNode(Opc, DL, ResTys, Ops); in SelectPredicatedLoad()
1939 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg( in SelectPredicatedLoad()
1940 AArch64::zsub0 + i, DL, VT, SuperReg)); in SelectPredicatedLoad()
1945 CurDAG->RemoveDeadNode(N); in SelectPredicatedLoad()
1955 EVT VT = N->getValueType(0); in SelectContiguousMultiVectorLoad() local
1956 SDValue Chain = N->getOperand(0); in SelectContiguousMultiVectorLoad()
1958 SDValue PNg = N->getOperand(2); in SelectContiguousMultiVectorLoad()
1959 SDValue Base = N->getOperand(3); in SelectContiguousMultiVectorLoad()
1960 SDValue Offset = CurDAG->getTargetConstant(0, DL, MVT::i64); in SelectContiguousMultiVectorLoad()
1965 SDValue Ops[] = {PNg, // Predicate-as-counter in SelectContiguousMultiVectorLoad()
1971 SDNode *Load = CurDAG->getMachineNode(Opc, DL, ResTys, Ops); in SelectContiguousMultiVectorLoad()
1974 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg( in SelectContiguousMultiVectorLoad()
1975 AArch64::zsub0 + i, DL, VT, SuperReg)); in SelectContiguousMultiVectorLoad()
1980 CurDAG->RemoveDeadNode(N); in SelectContiguousMultiVectorLoad()
1985 if (N->getValueType(0) != MVT::nxv4f32) in SelectFrintFromVT()
1993 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Node->getOperand(4))) in SelectMultiVectorLuti()
1994 if (Imm->getZExtValue() > MaxImm) in SelectMultiVectorLuti()
1998 if (!ImmToReg<AArch64::ZT0, 0>(Node->getOperand(2), ZtValue)) in SelectMultiVectorLuti()
2000 SDValue Ops[] = {ZtValue, Node->getOperand(3), Node->getOperand(4)}; in SelectMultiVectorLuti()
2002 EVT VT = Node->getValueType(0); in SelectMultiVectorLuti() local
2005 CurDAG->getMachineNode(Opc, DL, {MVT::Untyped, MVT::Other}, Ops); in SelectMultiVectorLuti()
2009 ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg( in SelectMultiVectorLuti()
2010 AArch64::zsub0 + I, DL, VT, SuperReg)); in SelectMultiVectorLuti()
2015 CurDAG->RemoveDeadNode(Node); in SelectMultiVectorLuti()
2021 EVT VT = N->getValueType(0); in SelectClamp() local
2023 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); in SelectClamp()
2025 SDValue Zn = N->getOperand(1 + NumVecs); in SelectClamp()
2026 SDValue Zm = N->getOperand(2 + NumVecs); in SelectClamp()
2030 SDNode *Intrinsic = CurDAG->getMachineNode(Op, DL, MVT::Untyped, Ops); in SelectClamp()
2033 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg( in SelectClamp()
2034 AArch64::zsub0 + i, DL, VT, SuperReg)); in SelectClamp()
2036 CurDAG->RemoveDeadNode(N); in SelectClamp()
2071 TileNum = N->getConstantOperandVal(2); in SelectMultiVectorMove()
2078 SliceBase = N->getOperand(2); in SelectMultiVectorMove()
2080 SliceBase = N->getOperand(3); in SelectMultiVectorMove()
2086 SDValue SubReg = CurDAG->getRegister(BaseReg, MVT::Other); in SelectMultiVectorMove()
2087 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(0)}; in SelectMultiVectorMove()
2088 SDNode *Mov = CurDAG->getMachineNode(Op, DL, {MVT::Untyped, MVT::Other}, Ops); in SelectMultiVectorMove()
2090 EVT VT = N->getValueType(0); in SelectMultiVectorMove() local
2093 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT, in SelectMultiVectorMove()
2098 CurDAG->RemoveDeadNode(N); in SelectMultiVectorMove()
2107 SDValue SliceBase = N->getOperand(2); in SelectMultiVectorMoveZ()
2109 SliceBase = N->getOperand(3); in SelectMultiVectorMoveZ()
2120 Ops.push_back(N->getOperand(2)); in SelectMultiVectorMoveZ()
2123 Ops.push_back(N->getOperand(0)); //Chain in SelectMultiVectorMoveZ()
2124 SDNode *Mov = CurDAG->getMachineNode(Op, DL, {MVT::Untyped, MVT::Other}, Ops); in SelectMultiVectorMoveZ()
2126 EVT VT = N->getValueType(0); in SelectMultiVectorMoveZ() local
2129 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT, in SelectMultiVectorMoveZ()
2135 CurDAG->RemoveDeadNode(N); in SelectMultiVectorMoveZ()
2143 EVT VT = N->getValueType(0); in SelectUnaryMultiIntrinsic() local
2144 unsigned NumInVecs = N->getNumOperands() - 1; in SelectUnaryMultiIntrinsic()
2149 "Don't know how to handle multi-register input!"); in SelectUnaryMultiIntrinsic()
2150 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, in SelectUnaryMultiIntrinsic()
2151 N->op_begin() + 1 + NumInVecs); in SelectUnaryMultiIntrinsic()
2156 Ops.push_back(N->getOperand(1 + I)); in SelectUnaryMultiIntrinsic()
2159 SDNode *Res = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops); in SelectUnaryMultiIntrinsic()
2163 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg( in SelectUnaryMultiIntrinsic()
2164 AArch64::zsub0 + I, DL, VT, SuperReg)); in SelectUnaryMultiIntrinsic()
2165 CurDAG->RemoveDeadNode(N); in SelectUnaryMultiIntrinsic()
2171 EVT VT = N->getOperand(2)->getValueType(0); in SelectStore() local
2174 bool Is128Bit = VT.getSizeInBits() == 128; in SelectStore()
2175 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs); in SelectStore()
2178 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)}; in SelectStore()
2179 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops); in SelectStore()
2182 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectStore()
2183 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); in SelectStore()
2194 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs); in SelectPredicatedStore()
2201 N, Opc_rr, Opc_ri, N->getOperand(NumVecs + 3), in SelectPredicatedStore()
2202 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale); in SelectPredicatedStore()
2204 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), // predicate in SelectPredicatedStore()
2207 N->getOperand(0)}; // chain in SelectPredicatedStore()
2208 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops); in SelectPredicatedStore()
2216 const DataLayout &DL = CurDAG->getDataLayout(); in SelectAddrModeFrameIndexSVE()
2221 int FI = FINode->getIndex(); in SelectAddrModeFrameIndexSVE()
2222 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeFrameIndexSVE()
2223 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64); in SelectAddrModeFrameIndexSVE()
2233 EVT VT = N->getOperand(2)->getValueType(0); in SelectPostStore() local
2238 bool Is128Bit = VT.getSizeInBits() == 128; in SelectPostStore()
2239 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); in SelectPostStore()
2243 N->getOperand(NumVecs + 1), // base register in SelectPostStore()
2244 N->getOperand(NumVecs + 2), // Incremental in SelectPostStore()
2245 N->getOperand(0)}; // Chain in SelectPostStore()
2246 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectPostStore()
2252 /// WidenVector - Given a value in the V64 register class, produce the
2261 EVT VT = V64Reg.getValueType(); in operator ()() local
2262 unsigned NarrowSize = VT.getVectorNumElements(); in operator ()()
2263 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in operator ()()
2274 /// NarrowVector - Given a value in the V128 register class, produce the
2277 EVT VT = V128Reg.getValueType(); in NarrowVector() local
2278 unsigned WideSize = VT.getVectorNumElements(); in NarrowVector()
2279 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
2289 EVT VT = N->getValueType(0); in SelectLoadLane() local
2290 bool Narrow = VT.getSizeInBits() == 64; in SelectLoadLane()
2293 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs); in SelectLoadLane()
2303 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 2); in SelectLoadLane()
2305 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectLoadLane()
2306 N->getOperand(NumVecs + 3), N->getOperand(0)}; in SelectLoadLane()
2307 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectLoadLane()
2310 EVT WideVT = RegSeq.getOperand(1)->getValueType(0); in SelectLoadLane()
2314 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg); in SelectLoadLane()
2321 CurDAG->RemoveDeadNode(N); in SelectLoadLane()
2327 EVT VT = N->getValueType(0); in SelectPostLoadLane() local
2328 bool Narrow = VT.getSizeInBits() == 64; in SelectPostLoadLane()
2331 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); in SelectPostLoadLane()
2340 RegSeq->getValueType(0), MVT::Other}; in SelectPostLoadLane()
2342 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 1); in SelectPostLoadLane()
2345 CurDAG->getTargetConstant(LaneNo, dl, in SelectPostLoadLane()
2347 N->getOperand(NumVecs + 2), // Base register in SelectPostLoadLane()
2348 N->getOperand(NumVecs + 3), // Incremental in SelectPostLoadLane()
2349 N->getOperand(0)}; in SelectPostLoadLane()
2350 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectPostLoadLane()
2361 EVT WideVT = RegSeq.getOperand(1)->getValueType(0); in SelectPostLoadLane()
2365 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, in SelectPostLoadLane()
2375 CurDAG->RemoveDeadNode(N); in SelectPostLoadLane()
2381 EVT VT = N->getOperand(2)->getValueType(0); in SelectStoreLane() local
2382 bool Narrow = VT.getSizeInBits() == 64; in SelectStoreLane()
2385 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs); in SelectStoreLane()
2393 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 2); in SelectStoreLane()
2395 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectStoreLane()
2396 N->getOperand(NumVecs + 3), N->getOperand(0)}; in SelectStoreLane()
2397 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in SelectStoreLane()
2400 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectStoreLane()
2401 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); in SelectStoreLane()
2409 EVT VT = N->getOperand(2)->getValueType(0); in SelectPostStoreLane() local
2410 bool Narrow = VT.getSizeInBits() == 64; in SelectPostStoreLane()
2413 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); in SelectPostStoreLane()
2424 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 1); in SelectPostStoreLane()
2426 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64), in SelectPostStoreLane()
2427 N->getOperand(NumVecs + 2), // Base Register in SelectPostStoreLane()
2428 N->getOperand(NumVecs + 3), // Incremental in SelectPostStoreLane()
2429 N->getOperand(0)}; in SelectPostStoreLane()
2430 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectPostStoreLane()
2433 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectPostStoreLane()
2434 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); in SelectPostStoreLane()
2444 assert(N->getOpcode() == ISD::AND && in isBitfieldExtractOpFromAnd()
2447 EVT VT = N->getValueType(0); in isBitfieldExtractOpFromAnd() local
2449 // Here we can test the type of VT and return false when the type does not in isBitfieldExtractOpFromAnd()
2452 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromAnd()
2455 // FIXME: simplify-demanded-bits in DAGCombine will probably have in isBitfieldExtractOpFromAnd()
2456 // changed the AND node to a 32-bit mask operation. We'll have to in isBitfieldExtractOpFromAnd()
2467 const SDNode *Op0 = N->getOperand(0).getNode(); in isBitfieldExtractOpFromAnd()
2469 // Because of simplify-demanded-bits in DAGCombine, the mask may have been in isBitfieldExtractOpFromAnd()
2480 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND && in isBitfieldExtractOpFromAnd()
2481 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL, SrlImm)) { in isBitfieldExtractOpFromAnd()
2482 // Extend the incoming operand of the SRL to 64-bit. in isBitfieldExtractOpFromAnd()
2483 Opd0 = Widen(CurDAG, Op0->getOperand(0).getOperand(0)); in isBitfieldExtractOpFromAnd()
2487 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE && in isBitfieldExtractOpFromAnd()
2488 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL, in isBitfieldExtractOpFromAnd()
2491 Opd0 = Op0->getOperand(0).getOperand(0); in isBitfieldExtractOpFromAnd()
2494 VT = Opd0->getValueType(0); in isBitfieldExtractOpFromAnd()
2496 Opd0 = Op0->getOperand(0); in isBitfieldExtractOpFromAnd()
2497 ClampMSB = (VT == MVT::i32); in isBitfieldExtractOpFromAnd()
2504 Opd0 = N->getOperand(0); in isBitfieldExtractOpFromAnd()
2510 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) { in isBitfieldExtractOpFromAnd()
2519 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm) in isBitfieldExtractOpFromAnd()
2520 : llvm::countr_one<uint64_t>(AndImm)) - in isBitfieldExtractOpFromAnd()
2529 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri; in isBitfieldExtractOpFromAnd()
2536 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG); in isBitfieldExtractOpFromSExtInReg()
2538 EVT VT = N->getValueType(0); in isBitfieldExtractOpFromSExtInReg() local
2539 unsigned BitWidth = VT.getSizeInBits(); in isBitfieldExtractOpFromSExtInReg()
2540 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromSExtInReg()
2543 SDValue Op = N->getOperand(0); in isBitfieldExtractOpFromSExtInReg()
2544 if (Op->getOpcode() == ISD::TRUNCATE) { in isBitfieldExtractOpFromSExtInReg()
2545 Op = Op->getOperand(0); in isBitfieldExtractOpFromSExtInReg()
2546 VT = Op->getValueType(0); in isBitfieldExtractOpFromSExtInReg()
2547 BitWidth = VT.getSizeInBits(); in isBitfieldExtractOpFromSExtInReg()
2555 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); in isBitfieldExtractOpFromSExtInReg()
2559 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri; in isBitfieldExtractOpFromSExtInReg()
2562 Imms = ShiftImm + Width - 1; in isBitfieldExtractOpFromSExtInReg()
2583 if (N->getOpcode() != ISD::SRL) in isSeveralBitsExtractOpFromShr()
2587 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, AndMask)) in isSeveralBitsExtractOpFromShr()
2590 Opd0 = N->getOperand(0).getOperand(0); in isSeveralBitsExtractOpFromShr()
2593 if (!isIntImmediate(N->getOperand(1), SrlImm)) in isSeveralBitsExtractOpFromShr()
2600 Opc = N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri; in isSeveralBitsExtractOpFromShr()
2609 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && in isBitfieldExtractOpFromShr()
2612 EVT VT = N->getValueType(0); in isBitfieldExtractOpFromShr() local
2614 // Here we can test the type of VT and return false when the type does not in isBitfieldExtractOpFromShr()
2617 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldExtractOpFromShr()
2627 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, ShlImm)) { in isBitfieldExtractOpFromShr()
2628 Opd0 = N->getOperand(0).getOperand(0); in isBitfieldExtractOpFromShr()
2629 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL && in isBitfieldExtractOpFromShr()
2630 N->getOperand(0).getNode()->getOpcode() == ISD::TRUNCATE) { in isBitfieldExtractOpFromShr()
2632 // be considered as setting high 32 bits as zero. Our strategy here is to in isBitfieldExtractOpFromShr()
2635 Opd0 = N->getOperand(0).getOperand(0); in isBitfieldExtractOpFromShr()
2636 TruncBits = Opd0->getValueType(0).getSizeInBits() - VT.getSizeInBits(); in isBitfieldExtractOpFromShr()
2637 VT = Opd0.getValueType(); in isBitfieldExtractOpFromShr()
2638 assert(VT == MVT::i64 && "the promoted type should be i64"); in isBitfieldExtractOpFromShr()
2643 Opd0 = N->getOperand(0); in isBitfieldExtractOpFromShr()
2649 if (ShlImm >= VT.getSizeInBits()) { in isBitfieldExtractOpFromShr()
2657 if (!isIntImmediate(N->getOperand(1), SrlImm)) in isBitfieldExtractOpFromShr()
2660 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() && in isBitfieldExtractOpFromShr()
2662 int immr = SrlImm - ShlImm; in isBitfieldExtractOpFromShr()
2663 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr; in isBitfieldExtractOpFromShr()
2664 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1; in isBitfieldExtractOpFromShr()
2666 if (VT == MVT::i32) in isBitfieldExtractOpFromShr()
2667 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri; in isBitfieldExtractOpFromShr()
2669 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri; in isBitfieldExtractOpFromShr()
2674 assert(N->getOpcode() == ISD::SIGN_EXTEND); in tryBitfieldExtractOpFromSExt()
2676 EVT VT = N->getValueType(0); in tryBitfieldExtractOpFromSExt() local
2677 EVT NarrowVT = N->getOperand(0)->getValueType(0); in tryBitfieldExtractOpFromSExt()
2678 if (VT != MVT::i64 || NarrowVT != MVT::i32) in tryBitfieldExtractOpFromSExt()
2682 SDValue Op = N->getOperand(0); in tryBitfieldExtractOpFromSExt()
2687 // Extend the incoming operand of the shift to 64-bits. in tryBitfieldExtractOpFromSExt()
2690 unsigned Imms = NarrowVT.getSizeInBits() - 1; in tryBitfieldExtractOpFromSExt()
2691 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT), in tryBitfieldExtractOpFromSExt()
2692 CurDAG->getTargetConstant(Imms, dl, VT)}; in tryBitfieldExtractOpFromSExt()
2693 CurDAG->SelectNodeTo(N, AArch64::SBFMXri, VT, Ops); in tryBitfieldExtractOpFromSExt()
2701 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64) in isBitfieldExtractOp()
2704 switch (N->getOpcode()) { in isBitfieldExtractOp()
2706 if (!N->isMachineOpcode()) in isBitfieldExtractOp()
2720 unsigned NOpc = N->getMachineOpcode(); in isBitfieldExtractOp()
2729 Opd0 = N->getOperand(0); in isBitfieldExtractOp()
2730 Immr = N->getConstantOperandVal(1); in isBitfieldExtractOp()
2731 Imms = N->getConstantOperandVal(2); in isBitfieldExtractOp()
2744 EVT VT = N->getValueType(0); in tryBitfieldExtractOp() local
2749 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) { in tryBitfieldExtractOp()
2750 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64), in tryBitfieldExtractOp()
2751 CurDAG->getTargetConstant(Imms, dl, MVT::i64)}; in tryBitfieldExtractOp()
2753 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64); in tryBitfieldExtractOp()
2754 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl, in tryBitfieldExtractOp()
2760 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT), in tryBitfieldExtractOp()
2761 CurDAG->getTargetConstant(Imms, dl, VT)}; in tryBitfieldExtractOp()
2762 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in tryBitfieldExtractOp()
2771 unsigned NumberOfIgnoredHighBits, EVT VT) { in isBitfieldDstMask() argument
2772 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldDstMask()
2774 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits; in isBitfieldDstMask()
2803 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue(); in getUsefulBitsFromAndWithImmediate()
2817 OpUsefulBits <<= MSB - Imm + 1; in getUsefulBitsFromBitfieldMoveOpd()
2818 --OpUsefulBits; in getUsefulBitsFromBitfieldMoveOpd()
2825 --OpUsefulBits; in getUsefulBitsFromBitfieldMoveOpd()
2827 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm; in getUsefulBitsFromBitfieldMoveOpd()
2830 OpUsefulBits.lshrInPlace(OpUsefulBits.getBitWidth() - Imm); in getUsefulBitsFromBitfieldMoveOpd()
2839 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue(); in getUsefulBitsFromUBFM()
2841 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue(); in getUsefulBitsFromUBFM()
2849 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue(); in getUsefulBitsFromOrWithShiftedReg()
2877 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue(); in getUsefulBitsFromBFM()
2879 cast<const ConstantSDNode>(Op.getOperand(3).getNode())->getZExtValue(); in getUsefulBitsFromBFM()
2892 uint64_t Width = MSB - Imm + 1; in getUsefulBitsFromBFM()
2896 --OpUsefulBits; in getUsefulBitsFromBFM()
2910 uint64_t LSB = UsefulBits.getBitWidth() - Imm; in getUsefulBitsFromBFM()
2913 --OpUsefulBits; in getUsefulBitsFromBFM()
2934 if (!UserNode->isMachineOpcode()) in getUsefulBitsForUse()
2937 switch (UserNode->getMachineOpcode()) { in getUsefulBitsForUse()
2953 if (UserNode->getOperand(0) != Orig && UserNode->getOperand(1) == Orig) in getUsefulBitsForUse()
2963 if (UserNode->getOperand(0) != Orig) in getUsefulBitsForUse()
2970 if (UserNode->getOperand(0) != Orig) in getUsefulBitsForUse()
2989 for (SDNode *Node : Op.getNode()->uses()) { in getUsefulBits()
3002 /// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3008 EVT VT = Op.getValueType(); in getLeftShift() local
3010 unsigned BitWidth = VT.getSizeInBits(); in getLeftShift()
3015 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt in getLeftShift()
3016 ShiftNode = CurDAG->getMachineNode( in getLeftShift()
3017 UBFMOpc, dl, VT, Op, in getLeftShift()
3018 CurDAG->getTargetConstant(BitWidth - ShlAmount, dl, VT), in getLeftShift()
3019 CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, dl, VT)); in getLeftShift()
3021 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1 in getLeftShift()
3023 int ShrAmount = -ShlAmount; in getLeftShift()
3024 ShiftNode = CurDAG->getMachineNode( in getLeftShift()
3025 UBFMOpc, dl, VT, Op, CurDAG->getTargetConstant(ShrAmount, dl, VT), in getLeftShift()
3026 CurDAG->getTargetConstant(BitWidth - 1, dl, VT)); in getLeftShift()
3032 // For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3039 // For bit-field-positioning pattern "shl VAL, N)".
3051 EVT VT = Op.getValueType(); in isBitfieldPositioningOp() local
3052 unsigned BitWidth = VT.getSizeInBits(); in isBitfieldPositioningOp()
3056 KnownBits Known = CurDAG->computeKnownBits(Op); in isBitfieldPositioningOp()
3058 // Non-zero in the sense that they're not provably zero, which is the key in isBitfieldPositioningOp()
3085 EVT VT = Op.getValueType(); in isBitfieldPositioningOpFromAnd() local
3086 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldPositioningOpFromAnd()
3087 "Caller guarantees VT is one of i32 or i64"); in isBitfieldPositioningOpFromAnd()
3088 (void)VT; in isBitfieldPositioningOpFromAnd()
3108 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'. in isBitfieldPositioningOpFromAnd()
3110 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND && in isBitfieldPositioningOpFromAnd()
3113 // For pattern "and(any_extend(shl(val, N)), shifted-mask)" in isBitfieldPositioningOpFromAnd()
3119 // expect VT to be MVT::i32. in isBitfieldPositioningOpFromAnd()
3120 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32."); in isBitfieldPositioningOpFromAnd()
3138 if (Width >= (int)VT.getSizeInBits()) { in isBitfieldPositioningOpFromAnd()
3139 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and in isBitfieldPositioningOpFromAnd()
3140 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to in isBitfieldPositioningOpFromAnd()
3142 // If VT is i32, what Width >= 32 means: in isBitfieldPositioningOpFromAnd()
3143 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op in isBitfieldPositioningOpFromAnd()
3144 // demands at least 'Width' bits (after dag-combiner). This together with in isBitfieldPositioningOpFromAnd()
3149 << "Found large Width in bit-field-positioning -- this indicates no " in isBitfieldPositioningOpFromAnd()
3162 Src = getLeftShift(CurDAG, ShlOp0, ShlImm - DstLSB); in isBitfieldPositioningOpFromAnd()
3207 EVT VT = Op.getValueType(); in isBitfieldPositioningOpFromShl() local
3208 assert((VT == MVT::i32 || VT == MVT::i64) && in isBitfieldPositioningOpFromShl()
3210 (void)VT; in isBitfieldPositioningOpFromShl()
3228 Src = getLeftShift(CurDAG, Op.getOperand(0), ShlImm - DstLSB); in isBitfieldPositioningOpFromShl()
3232 static bool isShiftedMask(uint64_t Mask, EVT VT) { in isShiftedMask() argument
3233 assert(VT == MVT::i32 || VT == MVT::i64); in isShiftedMask()
3234 if (VT == MVT::i32) in isShiftedMask()
3242 assert(N->getOpcode() == ISD::OR && "Expect a OR operation"); in tryBitfieldInsertOpFromOrAndImm()
3244 EVT VT = N->getValueType(0); in tryBitfieldInsertOpFromOrAndImm() local
3245 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertOpFromOrAndImm()
3248 unsigned BitWidth = VT.getSizeInBits(); in tryBitfieldInsertOpFromOrAndImm()
3261 SDValue And = N->getOperand(0); in tryBitfieldInsertOpFromOrAndImm()
3269 KnownBits Known = CurDAG->computeKnownBits(And); in tryBitfieldInsertOpFromOrAndImm()
3271 // Non-zero in the sense that they're not provably zero, which is the key in tryBitfieldInsertOpFromOrAndImm()
3276 if (!isShiftedMask(Known.Zero.getZExtValue(), VT)) in tryBitfieldInsertOpFromOrAndImm()
3288 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount(); in tryBitfieldInsertOpFromOrAndImm()
3291 unsigned ImmR = (BitWidth - LSB) % BitWidth; in tryBitfieldInsertOpFromOrAndImm()
3292 unsigned ImmS = Width - 1; in tryBitfieldInsertOpFromOrAndImm()
3302 // with a ORR-immediate with the zero register. in tryBitfieldInsertOpFromOrAndImm()
3316 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm; in tryBitfieldInsertOpFromOrAndImm()
3317 SDNode *MOVI = CurDAG->getMachineNode( in tryBitfieldInsertOpFromOrAndImm()
3318 MOVIOpc, DL, VT, CurDAG->getTargetConstant(BFIImm, DL, VT)); in tryBitfieldInsertOpFromOrAndImm()
3322 CurDAG->getTargetConstant(ImmR, DL, VT), in tryBitfieldInsertOpFromOrAndImm()
3323 CurDAG->getTargetConstant(ImmS, DL, VT)}; in tryBitfieldInsertOpFromOrAndImm()
3324 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOrAndImm()
3325 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in tryBitfieldInsertOpFromOrAndImm()
3332 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR. in isWorthFoldingIntoOrrWithShift()
3336 EVT VT = Dst.getValueType(); in isWorthFoldingIntoOrrWithShift() local
3337 assert((VT == MVT::i32 || VT == MVT::i64) && in isWorthFoldingIntoOrrWithShift()
3338 "Caller should guarantee that VT is one of i32 or i64"); in isWorthFoldingIntoOrrWithShift()
3339 const unsigned SizeInBits = VT.getSizeInBits(); in isWorthFoldingIntoOrrWithShift()
3370 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in isWorthFoldingIntoOrrWithShift()
3371 SDNode *UBFMNode = CurDAG->getMachineNode( in isWorthFoldingIntoOrrWithShift()
3372 UBFMOpc, DL, VT, DstOp0.getOperand(0), in isWorthFoldingIntoOrrWithShift()
3373 CurDAG->getTargetConstant(SrlImm + NumTrailingZeroInShiftedMask, DL, in isWorthFoldingIntoOrrWithShift()
3374 VT), in isWorthFoldingIntoOrrWithShift()
3375 CurDAG->getTargetConstant( in isWorthFoldingIntoOrrWithShift()
3376 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT)); in isWorthFoldingIntoOrrWithShift()
3407 EVT VT = N->getValueType(0); in tryOrrWithShift() local
3408 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node"); in tryOrrWithShift()
3409 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) || in tryOrrWithShift()
3410 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) && in tryOrrWithShift()
3412 assert((VT == MVT::i32 || VT == MVT::i64) && in tryOrrWithShift()
3420 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs; in tryOrrWithShift()
3429 // nodes from Dst. If ORR with left-shifted operand also simplifies away in tryOrrWithShift()
3438 CurDAG->getTargetConstant(EncodedShiftImm, DL, VT)}; in tryOrrWithShift()
3439 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops); in tryOrrWithShift()
3453 CurDAG->getTargetConstant( in tryOrrWithShift()
3454 AArch64_AM::getShifterImm(AArch64_AM::LSL, ShlImm), DL, VT)}; in tryOrrWithShift()
3455 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops); in tryOrrWithShift()
3459 // Select the following pattern to left-shifted operand rather than BFI. in tryOrrWithShift()
3472 CurDAG->getTargetConstant( in tryOrrWithShift()
3473 AArch64_AM::getShifterImm(AArch64_AM::LSL, ShlImm), DL, VT)}; in tryOrrWithShift()
3474 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops); in tryOrrWithShift()
3481 // Select the following pattern to right-shifted operand rather than BFXIL. in tryOrrWithShift()
3494 CurDAG->getTargetConstant( in tryOrrWithShift()
3495 AArch64_AM::getShifterImm(AArch64_AM::LSR, SrlImm), DL, VT)}; in tryOrrWithShift()
3496 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops); in tryOrrWithShift()
3506 assert(N->getOpcode() == ISD::OR && "Expect a OR operation"); in tryBitfieldInsertOpFromOr()
3508 EVT VT = N->getValueType(0); in tryBitfieldInsertOpFromOr() local
3509 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertOpFromOr()
3512 unsigned BitWidth = VT.getSizeInBits(); in tryBitfieldInsertOpFromOr()
3514 // Because of simplify-demanded-bits in DAGCombine, involved masks may not in tryBitfieldInsertOpFromOr()
3524 // countTrailingZeros(mask2) == imm2 - imm + 1 in tryBitfieldInsertOpFromOr()
3543 SDValue OrOpd0Val = N->getOperand(I % 2); in tryBitfieldInsertOpFromOr()
3545 SDValue OrOpd1Val = N->getOperand((I + 1) % 2); in tryBitfieldInsertOpFromOr()
3554 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) || in tryBitfieldInsertOpFromOr()
3555 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32)) in tryBitfieldInsertOpFromOr()
3560 Width = ImmS - ImmR + 1; in tryBitfieldInsertOpFromOr()
3568 // can share the ImmR and ImmS values from the already-computed UBFM. in tryBitfieldInsertOpFromOr()
3572 ImmR = (BitWidth - DstLSB) % BitWidth; in tryBitfieldInsertOpFromOr()
3573 ImmS = Width - 1; in tryBitfieldInsertOpFromOr()
3578 EVT VT = OrOpd1Val.getValueType(); in tryBitfieldInsertOpFromOr() local
3579 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand"); in tryBitfieldInsertOpFromOr()
3583 // AND with imm. Indeed, simplify-demanded-bits may have removed in tryBitfieldInsertOpFromOr()
3585 KnownBits Known = CurDAG->computeKnownBits(OrOpd1Val); in tryBitfieldInsertOpFromOr()
3598 isBitfieldDstMask(Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT)) in tryBitfieldInsertOpFromOr()
3600 Dst = OrOpd1->getOperand(0); in tryBitfieldInsertOpFromOr()
3602 // Maybe the AND has been removed by simplify-demanded-bits in tryBitfieldInsertOpFromOr()
3614 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(ImmR, DL, VT), in tryBitfieldInsertOpFromOr()
3615 CurDAG->getTargetConstant(ImmS, DL, VT)}; in tryBitfieldInsertOpFromOr()
3616 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOr()
3617 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in tryBitfieldInsertOpFromOr()
3625 SDValue And0 = N->getOperand(0); in tryBitfieldInsertOpFromOr()
3626 SDValue And1 = N->getOperand(1); in tryBitfieldInsertOpFromOr()
3631 (isShiftedMask(Mask0Imm, VT) || isShiftedMask(Mask1Imm, VT))) { in tryBitfieldInsertOpFromOr()
3636 if (isShiftedMask(Mask0Imm, VT)) { in tryBitfieldInsertOpFromOr()
3641 SDValue Src = And1->getOperand(0); in tryBitfieldInsertOpFromOr()
3642 SDValue Dst = And0->getOperand(0); in tryBitfieldInsertOpFromOr()
3644 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount(); in tryBitfieldInsertOpFromOr()
3646 // The BFXIL inserts the low-order bits from a source register, so right in tryBitfieldInsertOpFromOr()
3649 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in tryBitfieldInsertOpFromOr()
3651 if (Src->hasOneUse() && in tryBitfieldInsertOpFromOr()
3654 Src = Src->getOperand(0); in tryBitfieldInsertOpFromOr()
3658 SDNode *LSR = CurDAG->getMachineNode( in tryBitfieldInsertOpFromOr()
3659 ShiftOpc, DL, VT, Src, CurDAG->getTargetConstant(LsrImm, DL, VT), in tryBitfieldInsertOpFromOr()
3660 CurDAG->getTargetConstant(BitWidth - 1, DL, VT)); in tryBitfieldInsertOpFromOr()
3663 unsigned ImmR = (BitWidth - LSB) % BitWidth; in tryBitfieldInsertOpFromOr()
3664 unsigned ImmS = Width - 1; in tryBitfieldInsertOpFromOr()
3668 CurDAG->getTargetConstant(ImmR, DL, VT), in tryBitfieldInsertOpFromOr()
3669 CurDAG->getTargetConstant(ImmS, DL, VT)}; in tryBitfieldInsertOpFromOr()
3670 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri; in tryBitfieldInsertOpFromOr()
3671 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in tryBitfieldInsertOpFromOr()
3679 if (N->getOpcode() != ISD::OR) in tryBitfieldInsertOp()
3687 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0)); in tryBitfieldInsertOp()
3697 /// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
3701 if (N->getOpcode() != ISD::AND) in tryBitfieldInsertInZeroOp()
3704 EVT VT = N->getValueType(0); in tryBitfieldInsertInZeroOp() local
3705 if (VT != MVT::i32 && VT != MVT::i64) in tryBitfieldInsertInZeroOp()
3715 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits(); in tryBitfieldInsertInZeroOp()
3717 unsigned ImmS = Width - 1; in tryBitfieldInsertInZeroOp()
3720 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT), in tryBitfieldInsertInZeroOp()
3721 CurDAG->getTargetConstant(ImmS, DL, VT)}; in tryBitfieldInsertInZeroOp()
3722 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri; in tryBitfieldInsertInZeroOp()
3723 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in tryBitfieldInsertInZeroOp()
3727 /// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
3730 EVT VT = N->getValueType(0); in tryShiftAmountMod() local
3733 switch (N->getOpcode()) { in tryShiftAmountMod()
3735 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr; in tryShiftAmountMod()
3738 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr; in tryShiftAmountMod()
3741 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr; in tryShiftAmountMod()
3744 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr; in tryShiftAmountMod()
3752 if (VT == MVT::i32) { in tryShiftAmountMod()
3755 } else if (VT == MVT::i64) { in tryShiftAmountMod()
3761 SDValue ShiftAmt = N->getOperand(1); in tryShiftAmountMod()
3766 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND || in tryShiftAmountMod()
3767 ShiftAmt->getOpcode() == ISD::ANY_EXTEND) in tryShiftAmountMod()
3768 ShiftAmt = ShiftAmt->getOperand(0); in tryShiftAmountMod()
3770 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) { in tryShiftAmountMod()
3771 SDValue Add0 = ShiftAmt->getOperand(0); in tryShiftAmountMod()
3772 SDValue Add1 = ShiftAmt->getOperand(1); in tryShiftAmountMod()
3776 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X in tryShiftAmountMod()
3779 } else if (ShiftAmt->getOpcode() == ISD::SUB && in tryShiftAmountMod()
3782 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X in tryShiftAmountMod()
3786 EVT SubVT = ShiftAmt->getValueType(0); in tryShiftAmountMod()
3796 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT); in tryShiftAmountMod()
3798 CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1); in tryShiftAmountMod()
3800 } else if (ShiftAmt->getOpcode() == ISD::SUB && in tryShiftAmountMod()
3801 isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) { in tryShiftAmountMod()
3802 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X in tryShiftAmountMod()
3806 EVT SubVT = ShiftAmt->getValueType(0); in tryShiftAmountMod()
3816 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT); in tryShiftAmountMod()
3818 CurDAG->getMachineNode(NotOpc, DL, SubVT, Zero, Add1); in tryShiftAmountMod()
3834 NewShiftAmt = ShiftAmt->getOperand(0); in tryShiftAmountMod()
3838 if (VT == MVT::i32) in tryShiftAmountMod()
3840 else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) { in tryShiftAmountMod()
3841 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32); in tryShiftAmountMod()
3842 MachineSDNode *Ext = CurDAG->getMachineNode( in tryShiftAmountMod()
3843 AArch64::SUBREG_TO_REG, DL, VT, in tryShiftAmountMod()
3844 CurDAG->getTargetConstant(0, DL, MVT::i64), NewShiftAmt, SubReg); in tryShiftAmountMod()
3848 SDValue Ops[] = {N->getOperand(0), NewShiftAmt}; in tryShiftAmountMod()
3849 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in tryShiftAmountMod()
3859 FVal = CN->getValueAPF(); in checkCVTFixedPointOperandWithFBits()
3862 if (LN->getOperand(1).getOpcode() != AArch64ISD::ADDlow || in checkCVTFixedPointOperandWithFBits()
3863 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1))) in checkCVTFixedPointOperandWithFBits()
3867 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)); in checkCVTFixedPointOperandWithFBits()
3868 FVal = cast<ConstantFP>(CN->getConstVal())->getValueAPF(); in checkCVTFixedPointOperandWithFBits()
3873 // is between 1 and 32 for a destination w-register, or 1 and 64 for an in checkCVTFixedPointOperandWithFBits()
3874 // x-register. in checkCVTFixedPointOperandWithFBits()
3885 // fbits is between 1 and 64 in the worst-case, which means the fmul in checkCVTFixedPointOperandWithFBits()
3899 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32); in checkCVTFixedPointOperandWithFBits()
3924 return -1; in getIntOperandFromRegisterString()
3939 "Unexpected non-integer value in special register string."); in getIntOperandFromRegisterString()
3953 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1)); in tryReadRegister()
3954 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0)); in tryReadRegister()
3957 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS; in tryReadRegister()
3960 int Imm = getIntOperandFromRegisterString(RegString->getString()); in tryReadRegister()
3961 if (Imm == -1) { in tryReadRegister()
3965 AArch64SysReg::lookupSysRegByName(RegString->getString()); in tryReadRegister()
3966 if (TheReg && TheReg->Readable && in tryReadRegister()
3967 TheReg->haveFeatures(Subtarget->getFeatureBits())) in tryReadRegister()
3968 Imm = TheReg->Encoding; in tryReadRegister()
3970 Imm = AArch64SysReg::parseGenericRegister(RegString->getString()); in tryReadRegister()
3972 if (Imm == -1) { in tryReadRegister()
3974 if (!ReadIs128Bit && RegString->getString() == "pc") { in tryReadRegister()
3983 SDValue InChain = N->getOperand(0); in tryReadRegister()
3984 SDValue SysRegImm = CurDAG->getTargetConstant(Imm, DL, MVT::i32); in tryReadRegister()
3986 CurDAG->SelectNodeTo(N, Opcode64Bit, MVT::i64, MVT::Other /* Chain */, in tryReadRegister()
3989 SDNode *MRRS = CurDAG->getMachineNode( in tryReadRegister()
3996 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64, DL, MVT::i64, in tryReadRegister()
3998 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64, DL, MVT::i64, in tryReadRegister()
4014 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1)); in tryWriteRegister()
4015 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0)); in tryWriteRegister()
4018 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR; in tryWriteRegister()
4028 assert(isa<ConstantSDNode>(N->getOperand(2)) && in tryWriteRegister()
4030 unsigned Reg = PMapper->Encoding; in tryWriteRegister()
4031 uint64_t Immed = N->getConstantOperandVal(2); in tryWriteRegister()
4032 CurDAG->SelectNodeTo( in tryWriteRegister()
4033 N, State, MVT::Other, CurDAG->getTargetConstant(Reg, DL, MVT::i32), in tryWriteRegister()
4034 CurDAG->getTargetConstant(Immed, DL, MVT::i16), N->getOperand(0)); in tryWriteRegister()
4041 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()), in tryWriteRegister()
4045 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()), in tryWriteRegister()
4050 int Imm = getIntOperandFromRegisterString(RegString->getString()); in tryWriteRegister()
4051 if (Imm == -1) { in tryWriteRegister()
4055 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString()); in tryWriteRegister()
4056 if (TheReg && TheReg->Writeable && in tryWriteRegister()
4057 TheReg->haveFeatures(Subtarget->getFeatureBits())) in tryWriteRegister()
4058 Imm = TheReg->Encoding; in tryWriteRegister()
4060 Imm = AArch64SysReg::parseGenericRegister(RegString->getString()); in tryWriteRegister()
4062 if (Imm == -1) in tryWriteRegister()
4066 SDValue InChain = N->getOperand(0); in tryWriteRegister()
4068 CurDAG->SelectNodeTo(N, AArch64::MSR, MVT::Other, in tryWriteRegister()
4069 CurDAG->getTargetConstant(Imm, DL, MVT::i32), in tryWriteRegister()
4070 N->getOperand(2), InChain); in tryWriteRegister()
4074 SDNode *Pair = CurDAG->getMachineNode( in tryWriteRegister()
4076 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(), DL, in tryWriteRegister()
4078 N->getOperand(2), in tryWriteRegister()
4079 CurDAG->getTargetConstant(AArch64::sube64, DL, MVT::i32), in tryWriteRegister()
4080 N->getOperand(3), in tryWriteRegister()
4081 CurDAG->getTargetConstant(AArch64::subo64, DL, MVT::i32)}); in tryWriteRegister()
4083 CurDAG->SelectNodeTo(N, AArch64::MSRR, MVT::Other, in tryWriteRegister()
4084 CurDAG->getTargetConstant(Imm, DL, MVT::i32), in tryWriteRegister()
4091 /// We've got special pseudo-instructions for these
4094 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT(); in SelectCMP_SWAP()
4097 if (Subtarget->hasLSE()) return false; in SelectCMP_SWAP()
4111 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3), in SelectCMP_SWAP()
4112 N->getOperand(0)}; in SelectCMP_SWAP()
4113 SDNode *CmpSwap = CurDAG->getMachineNode( in SelectCMP_SWAP()
4115 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops); in SelectCMP_SWAP()
4117 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in SelectCMP_SWAP()
4118 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp}); in SelectCMP_SWAP()
4122 CurDAG->RemoveDeadNode(N); in SelectCMP_SWAP()
4127 bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, in SelectSVEAddSubImm() argument
4134 ->getAPIntValue() in SelectSVEAddSubImm()
4135 .trunc(VT.getFixedSizeInBits()) in SelectSVEAddSubImm()
4138 switch (VT.SimpleTy) { in SelectSVEAddSubImm()
4141 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubImm()
4142 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); in SelectSVEAddSubImm()
4149 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubImm()
4150 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); in SelectSVEAddSubImm()
4155 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); in SelectSVEAddSubImm()
4156 Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32); in SelectSVEAddSubImm()
4167 bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT, in SelectSVEAddSubSSatImm() argument
4175 ->getAPIntValue() in SelectSVEAddSubSSatImm()
4176 .trunc(VT.getFixedSizeInBits()) in SelectSVEAddSubSSatImm()
4180 Val = -Val; in SelectSVEAddSubSSatImm()
4184 // means we can only use the immediate form when the operand is non-negative. in SelectSVEAddSubSSatImm()
4188 switch (VT.SimpleTy) { in SelectSVEAddSubSSatImm()
4191 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubSSatImm()
4192 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); in SelectSVEAddSubSSatImm()
4199 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVEAddSubSSatImm()
4200 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); in SelectSVEAddSubSSatImm()
4205 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); in SelectSVEAddSubSSatImm()
4206 Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32); in SelectSVEAddSubSSatImm()
4217 bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, in SelectSVECpyDupImm() argument
4224 ->getAPIntValue() in SelectSVECpyDupImm()
4225 .trunc(VT.getFixedSizeInBits()) in SelectSVECpyDupImm()
4228 switch (VT.SimpleTy) { in SelectSVECpyDupImm()
4231 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVECpyDupImm()
4232 Imm = CurDAG->getTargetConstant(Val & 0xFF, DL, MVT::i32); in SelectSVECpyDupImm()
4238 if (Val >= -128 && Val <= 127) { in SelectSVECpyDupImm()
4239 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); in SelectSVECpyDupImm()
4240 Imm = CurDAG->getTargetConstant(Val & 0xFF, DL, MVT::i32); in SelectSVECpyDupImm()
4244 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) { in SelectSVECpyDupImm()
4245 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); in SelectSVECpyDupImm()
4246 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF, DL, MVT::i32); in SelectSVECpyDupImm()
4259 int64_t ImmVal = CNode->getSExtValue(); in SelectSVESignedArithImm()
4261 if (ImmVal >= -128 && ImmVal < 128) { in SelectSVESignedArithImm()
4262 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32); in SelectSVESignedArithImm()
4269 bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) { in SelectSVEArithImm() argument
4271 uint64_t ImmVal = CNode->getZExtValue(); in SelectSVEArithImm()
4273 switch (VT.SimpleTy) { in SelectSVEArithImm()
4290 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32); in SelectSVEArithImm()
4297 bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, in SelectSVELogicalImm() argument
4300 uint64_t ImmVal = CNode->getZExtValue(); in SelectSVELogicalImm()
4307 switch (VT.SimpleTy) { in SelectSVELogicalImm()
4331 Imm = CurDAG->getTargetConstant(encoding, DL, MVT::i64); in SelectSVELogicalImm()
4344 uint64_t High, bool AllowSaturation, in SelectSVEShiftImm() argument
4347 uint64_t ImmVal = CN->getZExtValue(); in SelectSVEShiftImm()
4354 if (ImmVal > High) { in SelectSVEShiftImm()
4357 ImmVal = High; in SelectSVEShiftImm()
4360 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32); in SelectSVEShiftImm()
4369 // since the offset between FrameIndex and IRGstack is a compile-time in trySelectStackSlotTagP()
4371 if (!(isa<FrameIndexSDNode>(N->getOperand(1)))) { in trySelectStackSlotTagP()
4375 SDValue IRG_SP = N->getOperand(2); in trySelectStackSlotTagP()
4376 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN || in trySelectStackSlotTagP()
4377 IRG_SP->getConstantOperandVal(1) != Intrinsic::aarch64_irg_sp) { in trySelectStackSlotTagP()
4383 int FI = cast<FrameIndexSDNode>(N->getOperand(1))->getIndex(); in trySelectStackSlotTagP()
4384 SDValue FiOp = CurDAG->getTargetFrameIndex( in trySelectStackSlotTagP()
4385 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in trySelectStackSlotTagP()
4386 int TagOffset = N->getConstantOperandVal(3); in trySelectStackSlotTagP()
4388 SDNode *Out = CurDAG->getMachineNode( in trySelectStackSlotTagP()
4390 {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2), in trySelectStackSlotTagP()
4391 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); in trySelectStackSlotTagP()
4397 assert(isa<ConstantSDNode>(N->getOperand(3)) && in SelectTagP()
4402 // compile-time constant, not just for stack allocations. in SelectTagP()
4406 int TagOffset = N->getConstantOperandVal(3); in SelectTagP()
4407 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64, in SelectTagP()
4408 {N->getOperand(1), N->getOperand(2)}); in SelectTagP()
4409 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64, in SelectTagP()
4410 {SDValue(N1, 0), N->getOperand(2)}); in SelectTagP()
4411 SDNode *N3 = CurDAG->getMachineNode( in SelectTagP()
4413 {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64), in SelectTagP()
4414 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)}); in SelectTagP()
4419 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!"); in trySelectCastFixedLengthToScalableVector()
4422 if (N->getConstantOperandVal(2) != 0) in trySelectCastFixedLengthToScalableVector()
4424 if (!N->getOperand(0).isUndef()) in trySelectCastFixedLengthToScalableVector()
4428 EVT VT = N->getValueType(0); in trySelectCastFixedLengthToScalableVector() local
4429 EVT InVT = N->getOperand(1).getValueType(); in trySelectCastFixedLengthToScalableVector()
4430 if (VT.isFixedLengthVector() || InVT.isScalableVector()) in trySelectCastFixedLengthToScalableVector()
4439 assert(VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock && in trySelectCastFixedLengthToScalableVector()
4443 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64); in trySelectCastFixedLengthToScalableVector()
4444 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, in trySelectCastFixedLengthToScalableVector()
4445 N->getOperand(1), RC)); in trySelectCastFixedLengthToScalableVector()
4450 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!"); in trySelectCastScalableToFixedLengthVector()
4453 if (N->getConstantOperandVal(1) != 0) in trySelectCastScalableToFixedLengthVector()
4457 EVT VT = N->getValueType(0); in trySelectCastScalableToFixedLengthVector() local
4458 EVT InVT = N->getOperand(0).getValueType(); in trySelectCastScalableToFixedLengthVector()
4459 if (VT.isScalableVector() || InVT.isFixedLengthVector()) in trySelectCastScalableToFixedLengthVector()
4461 if (VT.getSizeInBits() <= 128) in trySelectCastScalableToFixedLengthVector()
4472 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64); in trySelectCastScalableToFixedLengthVector()
4473 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, in trySelectCastScalableToFixedLengthVector()
4474 N->getOperand(0), RC)); in trySelectCastScalableToFixedLengthVector()
4479 assert(N->getOpcode() == ISD::OR && "Expected OR instruction"); in trySelectXAR()
4481 SDValue N0 = N->getOperand(0); in trySelectXAR()
4482 SDValue N1 = N->getOperand(1); in trySelectXAR()
4483 EVT VT = N->getValueType(0); in trySelectXAR() local
4485 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm) in trySelectXAR()
4489 // OR N0, N1 -> xar (x, y, imm) in trySelectXAR()
4491 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount in trySelectXAR()
4492 // N0 = SHL_PRED true, V, splat(bits-imm) in trySelectXAR()
4494 if (VT.isScalableVector() && in trySelectXAR()
4495 (Subtarget->hasSVE2() || in trySelectXAR()
4496 (Subtarget->hasSME() && Subtarget->isStreaming()))) { in trySelectXAR()
4505 if (!TLI->isAllActivePredicate(*CurDAG, N0.getOperand(0)) || in trySelectXAR()
4506 !TLI->isAllActivePredicate(*CurDAG, N1.getOperand(0))) in trySelectXAR()
4518 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits()) in trySelectXAR()
4523 CurDAG->getTargetConstant(ShrAmt.getZExtValue(), DL, MVT::i32); in trySelectXAR()
4527 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S, in trySelectXAR()
4529 CurDAG->SelectNodeTo(N, Opc, VT, Ops); in trySelectXAR()
4535 if (!Subtarget->hasSHA3()) in trySelectXAR()
4538 if (N0->getOpcode() != AArch64ISD::VSHL || in trySelectXAR()
4539 N1->getOpcode() != AArch64ISD::VLSHR) in trySelectXAR()
4542 if (N0->getOperand(0) != N1->getOperand(0) || in trySelectXAR()
4543 N1->getOperand(0)->getOpcode() != ISD::XOR) in trySelectXAR()
4554 SDValue Imm = CurDAG->getTargetConstant( in trySelectXAR()
4561 CurDAG->SelectNodeTo(N, AArch64::XAR, N0.getValueType(), Ops); in trySelectXAR()
4568 if (Node->isMachineOpcode()) { in Select()
4569 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n"); in Select()
4570 Node->setNodeId(-1); in Select()
4575 EVT VT = Node->getValueType(0); in Select() local
4577 switch (Node->getOpcode()) { in Select()
4649 if (ConstNode->isZero()) { in Select()
4650 if (VT == MVT::i32) { in Select()
4651 SDValue New = CurDAG->getCopyFromReg( in Select()
4652 CurDAG->getEntryNode(), SDLoc(Node), AArch64::WZR, MVT::i32); in Select()
4655 } else if (VT == MVT::i64) { in Select()
4656 SDValue New = CurDAG->getCopyFromReg( in Select()
4657 CurDAG->getEntryNode(), SDLoc(Node), AArch64::XZR, MVT::i64); in Select()
4667 int FI = cast<FrameIndexSDNode>(Node)->getIndex(); in Select()
4670 SDValue TFI = CurDAG->getTargetFrameIndex( in Select()
4671 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in Select()
4673 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32), in Select()
4674 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) }; in Select()
4675 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops); in Select()
4679 unsigned IntNo = Node->getConstantOperandVal(1); in Select()
4685 SDValue Chain = Node->getOperand(0); in Select()
4686 SDValue Val = Node->getOperand(2); in Select()
4687 SDValue Zero = CurDAG->getCopyFromReg(Chain, DL, AArch64::XZR, MVT::i64); in Select()
4689 CurDAG->getMachineNode(AArch64::GCSSS1, DL, MVT::Other, Val, Chain); in Select()
4690 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2, DL, MVT::i64, in Select()
4699 SDValue MemAddr = Node->getOperand(2); in Select()
4701 SDValue Chain = Node->getOperand(0); in Select()
4703 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64, in Select()
4708 cast<MemIntrinsicSDNode>(Node)->getMemOperand(); in Select()
4709 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp}); in Select()
4718 SDValue Chain = Node->getOperand(0); in Select()
4719 SDValue ValLo = Node->getOperand(2); in Select()
4720 SDValue ValHi = Node->getOperand(3); in Select()
4721 SDValue MemAddr = Node->getOperand(4); in Select()
4726 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops); in Select()
4729 cast<MemIntrinsicSDNode>(Node)->getMemOperand(); in Select()
4730 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); in Select()
4736 if (VT == MVT::v8i8) { in Select()
4739 } else if (VT == MVT::v16i8) { in Select()
4742 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4745 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4748 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4751 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4754 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4757 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4763 if (VT == MVT::v8i8) { in Select()
4766 } else if (VT == MVT::v16i8) { in Select()
4769 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4772 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4775 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4778 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4781 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4784 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4790 if (VT == MVT::v8i8) { in Select()
4793 } else if (VT == MVT::v16i8) { in Select()
4796 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4799 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4802 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4805 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4808 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4811 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4817 if (VT == MVT::v8i8) { in Select()
4820 } else if (VT == MVT::v16i8) { in Select()
4823 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4826 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4829 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4832 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4835 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4838 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4844 if (VT == MVT::v8i8) { in Select()
4847 } else if (VT == MVT::v16i8) { in Select()
4850 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4853 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4856 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4859 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4862 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4865 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4871 if (VT == MVT::v8i8) { in Select()
4874 } else if (VT == MVT::v16i8) { in Select()
4877 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4880 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4883 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4886 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4889 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4892 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4898 if (VT == MVT::v8i8) { in Select()
4901 } else if (VT == MVT::v16i8) { in Select()
4904 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4907 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4910 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4913 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4916 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4919 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4925 if (VT == MVT::v8i8) { in Select()
4928 } else if (VT == MVT::v16i8) { in Select()
4931 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4934 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4937 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4940 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4943 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4946 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4952 if (VT == MVT::v8i8) { in Select()
4955 } else if (VT == MVT::v16i8) { in Select()
4958 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
4961 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
4964 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
4967 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
4970 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
4973 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
4979 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
4982 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
4983 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
4986 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
4987 VT == MVT::v2f32) { in Select()
4990 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
4991 VT == MVT::v1f64) { in Select()
4997 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
5000 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
5001 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
5004 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
5005 VT == MVT::v2f32) { in Select()
5008 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
5009 VT == MVT::v1f64) { in Select()
5015 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
5018 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
5019 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
5022 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
5023 VT == MVT::v2f32) { in Select()
5026 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
5027 VT == MVT::v1f64) { in Select()
5048 if (VT == MVT::nxv16i8) { in Select()
5052 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5053 VT == MVT::nxv8bf16) { in Select()
5057 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5061 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5069 if (VT == MVT::nxv16i8) { in Select()
5070 if (Subtarget->hasSME2()) in Select()
5073 else if (Subtarget->hasSVE2p1()) in Select()
5079 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5080 VT == MVT::nxv8bf16) { in Select()
5081 if (Subtarget->hasSME2()) in Select()
5084 else if (Subtarget->hasSVE2p1()) in Select()
5090 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5091 if (Subtarget->hasSME2()) in Select()
5094 else if (Subtarget->hasSVE2p1()) in Select()
5100 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5101 if (Subtarget->hasSME2()) in Select()
5104 else if (Subtarget->hasSVE2p1()) in Select()
5114 if (VT == MVT::nxv16i8) { in Select()
5115 if (Subtarget->hasSME2()) in Select()
5118 else if (Subtarget->hasSVE2p1()) in Select()
5124 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5125 VT == MVT::nxv8bf16) { in Select()
5126 if (Subtarget->hasSME2()) in Select()
5129 else if (Subtarget->hasSVE2p1()) in Select()
5135 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5136 if (Subtarget->hasSME2()) in Select()
5139 else if (Subtarget->hasSVE2p1()) in Select()
5145 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5146 if (Subtarget->hasSME2()) in Select()
5149 else if (Subtarget->hasSVE2p1()) in Select()
5159 if (VT == MVT::nxv16i8) { in Select()
5160 if (Subtarget->hasSME2()) in Select()
5164 else if (Subtarget->hasSVE2p1()) in Select()
5170 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5171 VT == MVT::nxv8bf16) { in Select()
5172 if (Subtarget->hasSME2()) in Select()
5176 else if (Subtarget->hasSVE2p1()) in Select()
5182 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5183 if (Subtarget->hasSME2()) in Select()
5187 else if (Subtarget->hasSVE2p1()) in Select()
5193 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5194 if (Subtarget->hasSME2()) in Select()
5198 else if (Subtarget->hasSVE2p1()) in Select()
5208 if (VT == MVT::nxv16i8) { in Select()
5209 if (Subtarget->hasSME2()) in Select()
5213 else if (Subtarget->hasSVE2p1()) in Select()
5219 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5220 VT == MVT::nxv8bf16) { in Select()
5221 if (Subtarget->hasSME2()) in Select()
5225 else if (Subtarget->hasSVE2p1()) in Select()
5231 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5232 if (Subtarget->hasSME2()) in Select()
5236 else if (Subtarget->hasSVE2p1()) in Select()
5242 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5243 if (Subtarget->hasSME2()) in Select()
5247 else if (Subtarget->hasSVE2p1()) in Select()
5257 if (VT == MVT::nxv16i8) { in Select()
5261 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5262 VT == MVT::nxv8bf16) { in Select()
5266 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5270 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5278 if (VT == MVT::nxv16i8) { in Select()
5282 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5283 VT == MVT::nxv8bf16) { in Select()
5287 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5291 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5299 if (VT == MVT::nxv16i8) { in Select()
5303 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5304 VT == MVT::nxv8bf16) { in Select()
5308 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5312 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5320 if (VT == MVT::nxv16i8) { in Select()
5324 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5325 VT == MVT::nxv8bf16) { in Select()
5329 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5333 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5341 if (VT == MVT::nxv16i8) { in Select()
5345 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5346 VT == MVT::nxv8bf16) { in Select()
5350 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5354 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5362 if (VT == MVT::nxv16i8) { in Select()
5366 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5367 VT == MVT::nxv8bf16) { in Select()
5371 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5375 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5393 if (VT == MVT::nxv16i8) { in Select()
5396 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5397 VT == MVT::nxv8bf16) { in Select()
5400 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5403 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5410 if (VT == MVT::nxv16i8) { in Select()
5413 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5414 VT == MVT::nxv8bf16) { in Select()
5417 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5420 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5427 if (VT == MVT::nxv16i8) { in Select()
5430 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5431 VT == MVT::nxv8bf16) { in Select()
5434 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5437 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5444 if (VT == MVT::nxv16i8) { in Select()
5447 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
5448 VT == MVT::nxv8bf16) { in Select()
5451 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
5454 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
5472 SDValue Chain = Node->getOperand(0); in Select()
5473 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, DL, AArch64::FP, MVT::i64); in Select()
5475 CurDAG->getMachineNode(AArch64::SUBXri, DL, MVT::i64, CopyFP, in Select()
5476 CurDAG->getTargetConstant(8, DL, MVT::i32), in Select()
5477 CurDAG->getTargetConstant(0, DL, MVT::i32)), in Select()
5481 CurDAG->RemoveDeadNode(Node); in Select()
5483 auto &MF = CurDAG->getMachineFunction(); in Select()
5485 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true); in Select()
5490 Node->getValueType(0), in Select()
5499 Node->getValueType(0), in Select()
5507 Node->getValueType(0), in Select()
5516 Node->getValueType(0), in Select()
5526 unsigned IntNo = Node->getConstantOperandVal(0); in Select()
5544 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two, in Select()
5548 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three in Select()
5553 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four in Select()
5559 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two, in Select()
5563 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three in Select()
5568 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four in Select()
5574 Node->getValueType(0), in Select()
5581 Node->getValueType(0), in Select()
5588 Node->getValueType(0), in Select()
5595 Node->getValueType(0), in Select()
5602 Node->getValueType(0), in Select()
5609 Node->getValueType(0), in Select()
5616 Node->getValueType(0), in Select()
5623 Node->getValueType(0), in Select()
5630 Node->getValueType(0), in Select()
5637 Node->getValueType(0), in Select()
5644 Node->getValueType(0), in Select()
5651 Node->getValueType(0), in Select()
5658 Node->getValueType(0), in Select()
5665 Node->getValueType(0), in Select()
5672 Node->getValueType(0), in Select()
5679 Node->getValueType(0), in Select()
5686 Node->getValueType(0), in Select()
5693 Node->getValueType(0), in Select()
5700 Node->getValueType(0), in Select()
5707 Node->getValueType(0), in Select()
5714 Node->getValueType(0), in Select()
5721 Node->getValueType(0), in Select()
5728 Node->getValueType(0), in Select()
5735 Node->getValueType(0), in Select()
5742 Node->getValueType(0), in Select()
5749 Node->getValueType(0), in Select()
5756 Node->getValueType(0), in Select()
5763 Node->getValueType(0), in Select()
5770 Node->getValueType(0), in Select()
5777 Node->getValueType(0), in Select()
5784 Node->getValueType(0), in Select()
5791 Node->getValueType(0), in Select()
5798 Node->getValueType(0), in Select()
5805 Node->getValueType(0), in Select()
5812 Node->getValueType(0), in Select()
5819 Node->getValueType(0), in Select()
5826 Node->getValueType(0), in Select()
5833 Node->getValueType(0), in Select()
5840 Node->getValueType(0), in Select()
5847 Node->getValueType(0), in Select()
5854 Node->getValueType(0), in Select()
5861 Node->getValueType(0), in Select()
5868 Node->getValueType(0), in Select()
5875 Node->getValueType(0), in Select()
5882 Node->getValueType(0), in Select()
5889 Node->getValueType(0), in Select()
5896 Node->getValueType(0), in Select()
5903 Node->getValueType(0), in Select()
5910 Node->getValueType(0), in Select()
5917 Node->getValueType(0), in Select()
5924 Node->getValueType(0), in Select()
5931 Node->getValueType(0), in Select()
5968 Node->getValueType(0), in Select()
5975 Node->getValueType(0), in Select()
5982 Node->getValueType(0), in Select()
5992 Node->getValueType(0), in Select()
5999 Node->getValueType(0), in Select()
6006 Node->getValueType(0), in Select()
6016 Node->getValueType(0), in Select()
6023 Node->getValueType(0), in Select()
6030 Node->getValueType(0), in Select()
6041 Node->getValueType(0), in Select()
6052 Node->getValueType(0), in Select()
6063 Node->getValueType(0), in Select()
6074 Node->getValueType(0), in Select()
6081 Node->getValueType(0), in Select()
6112 Node->getValueType(0), in Select()
6119 Node->getValueType(0), in Select()
6126 Node->getValueType(0), in Select()
6133 Node->getValueType(0), in Select()
6140 Node->getValueType(0), in Select()
6150 unsigned IntNo = Node->getConstantOperandVal(1); in Select()
6151 if (Node->getNumOperands() >= 3) in Select()
6152 VT = Node->getOperand(2)->getValueType(0); in Select()
6157 if (VT == MVT::v8i8) { in Select()
6160 } else if (VT == MVT::v16i8) { in Select()
6163 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6164 VT == MVT::v4bf16) { in Select()
6167 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
6168 VT == MVT::v8bf16) { in Select()
6171 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6174 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6177 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6180 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6187 if (VT == MVT::v8i8) { in Select()
6190 } else if (VT == MVT::v16i8) { in Select()
6193 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6194 VT == MVT::v4bf16) { in Select()
6197 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
6198 VT == MVT::v8bf16) { in Select()
6201 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6204 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6207 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6210 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6217 if (VT == MVT::v8i8) { in Select()
6220 } else if (VT == MVT::v16i8) { in Select()
6223 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6224 VT == MVT::v4bf16) { in Select()
6227 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
6228 VT == MVT::v8bf16) { in Select()
6231 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6234 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6237 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6240 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6247 if (VT == MVT::v8i8) { in Select()
6250 } else if (VT == MVT::v16i8) { in Select()
6253 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6254 VT == MVT::v4bf16) { in Select()
6257 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
6258 VT == MVT::v8bf16) { in Select()
6261 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6264 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6267 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6270 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6277 if (VT == MVT::v8i8) { in Select()
6280 } else if (VT == MVT::v16i8) { in Select()
6283 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6284 VT == MVT::v4bf16) { in Select()
6287 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
6288 VT == MVT::v8bf16) { in Select()
6291 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6294 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6297 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6300 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6307 if (VT == MVT::v8i8) { in Select()
6310 } else if (VT == MVT::v16i8) { in Select()
6313 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6314 VT == MVT::v4bf16) { in Select()
6317 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || in Select()
6318 VT == MVT::v8bf16) { in Select()
6321 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6324 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6327 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6330 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6337 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6340 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6341 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6344 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6345 VT == MVT::v2f32) { in Select()
6348 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6349 VT == MVT::v1f64) { in Select()
6356 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6359 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6360 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6363 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6364 VT == MVT::v2f32) { in Select()
6367 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6368 VT == MVT::v1f64) { in Select()
6375 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6378 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6379 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6382 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6383 VT == MVT::v2f32) { in Select()
6386 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6387 VT == MVT::v1f64) { in Select()
6406 if (VT == MVT::nxv16i8) { in Select()
6409 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
6410 VT == MVT::nxv8bf16) { in Select()
6413 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
6416 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
6423 if (VT == MVT::nxv16i8) { in Select()
6426 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
6427 VT == MVT::nxv8bf16) { in Select()
6430 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
6433 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
6440 if (VT == MVT::nxv16i8) { in Select()
6443 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
6444 VT == MVT::nxv8bf16) { in Select()
6447 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
6450 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
6460 if (VT == MVT::v8i8) { in Select()
6463 } else if (VT == MVT::v16i8) { in Select()
6466 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6469 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6472 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6475 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6478 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6481 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6488 if (VT == MVT::v8i8) { in Select()
6491 } else if (VT == MVT::v16i8) { in Select()
6494 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6497 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6500 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6503 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6506 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6509 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6516 if (VT == MVT::v8i8) { in Select()
6519 } else if (VT == MVT::v16i8) { in Select()
6522 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6525 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6528 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6531 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6534 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6537 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6544 if (VT == MVT::v8i8) { in Select()
6547 } else if (VT == MVT::v16i8) { in Select()
6550 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6553 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6556 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6559 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6562 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6565 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6572 if (VT == MVT::v8i8) { in Select()
6575 } else if (VT == MVT::v16i8) { in Select()
6578 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6581 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6584 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6587 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6590 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6593 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6600 if (VT == MVT::v8i8) { in Select()
6603 } else if (VT == MVT::v16i8) { in Select()
6606 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6609 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6612 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6615 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6618 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6621 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6628 if (VT == MVT::v8i8) { in Select()
6631 } else if (VT == MVT::v16i8) { in Select()
6634 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6637 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6640 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6643 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6646 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6649 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6656 if (VT == MVT::v8i8) { in Select()
6659 } else if (VT == MVT::v16i8) { in Select()
6662 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6665 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6668 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6671 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6674 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6677 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6684 if (VT == MVT::v8i8) { in Select()
6687 } else if (VT == MVT::v16i8) { in Select()
6690 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6693 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6696 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6699 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6702 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6705 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6712 if (VT == MVT::v8i8) { in Select()
6715 } else if (VT == MVT::v16i8) { in Select()
6718 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6721 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6724 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6727 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6730 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6733 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6740 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6743 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6744 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6747 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6748 VT == MVT::v2f32) { in Select()
6751 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6752 VT == MVT::v1f64) { in Select()
6759 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6762 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6763 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6766 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6767 VT == MVT::v2f32) { in Select()
6770 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6771 VT == MVT::v1f64) { in Select()
6778 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6781 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6782 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6785 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6786 VT == MVT::v2f32) { in Select()
6789 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6790 VT == MVT::v1f64) { in Select()
6797 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6800 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6801 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6804 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6805 VT == MVT::v2f32) { in Select()
6808 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
6809 VT == MVT::v1f64) { in Select()
6816 VT = Node->getOperand(1).getValueType(); in Select()
6817 if (VT == MVT::v8i8) { in Select()
6820 } else if (VT == MVT::v16i8) { in Select()
6823 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6826 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6829 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6832 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6835 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6838 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6845 VT = Node->getOperand(1).getValueType(); in Select()
6846 if (VT == MVT::v8i8) { in Select()
6849 } else if (VT == MVT::v16i8) { in Select()
6852 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6855 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6858 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6861 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6864 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6867 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6874 VT = Node->getOperand(1).getValueType(); in Select()
6875 if (VT == MVT::v8i8) { in Select()
6878 } else if (VT == MVT::v16i8) { in Select()
6881 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6884 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6887 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6890 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6893 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6896 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6903 VT = Node->getOperand(1).getValueType(); in Select()
6904 if (VT == MVT::v8i8) { in Select()
6907 } else if (VT == MVT::v16i8) { in Select()
6910 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6913 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6916 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6919 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6922 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6925 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6932 VT = Node->getOperand(1).getValueType(); in Select()
6933 if (VT == MVT::v8i8) { in Select()
6936 } else if (VT == MVT::v16i8) { in Select()
6939 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6942 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) { in Select()
6945 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6948 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6951 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6954 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6961 VT = Node->getOperand(1).getValueType(); in Select()
6962 if (VT == MVT::v8i8) { in Select()
6965 } else if (VT == MVT::v16i8) { in Select()
6968 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) { in Select()
6971 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) { in Select()
6974 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) { in Select()
6977 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) { in Select()
6980 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) { in Select()
6983 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) { in Select()
6990 VT = Node->getOperand(1).getValueType(); in Select()
6991 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
6994 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
6995 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
6998 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
6999 VT == MVT::v2f32) { in Select()
7002 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
7003 VT == MVT::v1f64) { in Select()
7010 VT = Node->getOperand(1).getValueType(); in Select()
7011 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
7014 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
7015 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
7018 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
7019 VT == MVT::v2f32) { in Select()
7022 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
7023 VT == MVT::v1f64) { in Select()
7030 VT = Node->getOperand(1).getValueType(); in Select()
7031 if (VT == MVT::v16i8 || VT == MVT::v8i8) { in Select()
7034 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 || in Select()
7035 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in Select()
7038 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 || in Select()
7039 VT == MVT::v2f32) { in Select()
7042 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 || in Select()
7043 VT == MVT::v1f64) { in Select()
7050 if (VT == MVT::nxv16i8) { in Select()
7053 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
7054 VT == MVT::nxv8bf16) { in Select()
7057 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
7060 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
7067 if (VT == MVT::nxv16i8) { in Select()
7070 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
7071 VT == MVT::nxv8bf16) { in Select()
7074 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
7077 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
7084 if (VT == MVT::nxv16i8) { in Select()
7087 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 || in Select()
7088 VT == MVT::nxv8bf16) { in Select()
7091 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) { in Select()
7094 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) { in Select()
7106 /// createAArch64ISelDag - This pass converts a legalized DAG into a
7107 /// AArch64-specific DAG, ready for instruction scheduling.
7142 return cast<MemSDNode>(Root)->getMemoryVT(); in getMemVTFromNode()
7145 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT(); in getMemVTFromNode()
7147 const unsigned Opcode = Root->getOpcode(); in getMemVTFromNode()
7155 return cast<VTSDNode>(Root->getOperand(3))->getVT(); in getMemVTFromNode()
7157 return cast<VTSDNode>(Root->getOperand(4))->getVT(); in getMemVTFromNode()
7160 Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/2); in getMemVTFromNode()
7163 Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/3); in getMemVTFromNode()
7166 Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/4); in getMemVTFromNode()
7174 switch (Root->getConstantOperandVal(1)) { in getMemVTFromNode()
7184 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1); in getMemVTFromNode()
7188 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/2); in getMemVTFromNode()
7191 Ctx, Root->getOperand(4)->getValueType(0), /*NumVec=*/2); in getMemVTFromNode()
7195 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/3); in getMemVTFromNode()
7198 Ctx, Root->getOperand(5)->getValueType(0), /*NumVec=*/3); in getMemVTFromNode()
7202 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/4); in getMemVTFromNode()
7205 Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4); in getMemVTFromNode()
7215 /// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7222 const EVT MemVT = getMemVTFromNode(*(CurDAG->getContext()), Root); in SelectAddrModeIndexedSVE()
7223 const DataLayout &DL = CurDAG->getDataLayout(); in SelectAddrModeIndexedSVE()
7224 const MachineFrameInfo &MFI = MF->getFrameInfo(); in SelectAddrModeIndexedSVE()
7227 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrModeIndexedSVE()
7231 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexedSVE()
7232 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64); in SelectAddrModeIndexedSVE()
7251 int64_t MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue(); in SelectAddrModeIndexedSVE()
7262 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrModeIndexedSVE()
7266 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL)); in SelectAddrModeIndexedSVE()
7269 OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64); in SelectAddrModeIndexedSVE()
7294 int64_t ImmOff = C->getSExtValue(); in SelectSVERegRegAddrMode()
7304 Offset = CurDAG->getTargetConstant(ImmOff >> Scale, DL, MVT::i64); in SelectSVERegRegAddrMode()
7306 SDNode *MI = CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops); in SelectSVERegRegAddrMode()
7317 if (C->getZExtValue() == Scale) { in SelectSVERegRegAddrMode()
7330 return TLI->isAllActivePredicate(*CurDAG, N); in SelectAllActivePredicate()
7334 EVT VT = N.getValueType(); in SelectAnyPredicate() local
7335 return VT.isScalableVector() && VT.getVectorElementType() == MVT::i1; in SelectAnyPredicate()
7344 int64_t ImmOff = C->getSExtValue(); in SelectSMETileSlice()
7347 Offset = CurDAG->getTargetConstant(ImmOff / Scale, SDLoc(N), MVT::i64); in SelectSMETileSlice()
7354 Offset = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64); in SelectSMETileSlice()