Lines Matching +full:un +full:- +full:masked
1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
42 #define DEBUG_TYPE "arm-isel"
46 DisableShifterOp("disable-shifter-op", cl::Hidden,
47 cl::desc("Disable isel of shifter-op"),
50 //===--------------------------------------------------------------------===//
51 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
57 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
76 /// getI32Imm - Return a target constant of type i32 with the specified
79 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); in getI32Imm()
118 Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32); in SelectCMOVPred()
119 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32); in SelectCMOVPred()
179 return ARM_AM::getSOImmVal(Imm) != -1; in is_so_imm()
183 return ARM_AM::getSOImmVal(~Imm) != -1; in is_so_imm_not()
187 return ARM_AM::getT2SOImmVal(Imm) != -1; in is_t2_so_imm()
191 return ARM_AM::getT2SOImmVal(~Imm) != -1; in is_t2_so_imm_not()
211 /// SelectVLD - Select NEON load intrinsics. NumVecs should be
219 /// SelectVST - Select NEON store intrinsics. NumVecs should
227 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
247 /// SelectMVE_WB - Select MVE writeback load/store intrinsics.
250 /// SelectMVE_LongShift - Select MVE 64-bit scalar shift intrinsics.
254 /// SelectMVE_VADCSBC - Select MVE vector add/sub-with-carry intrinsics.
258 /// SelectMVE_VSHLC - Select MVE intrinsics for a shift that carries between
265 /// 2 for long non-rounding variants, vml{a,s}ldav[a][x]: [i16, i32]
274 /// Select a 64-bit MVE vector reduction with two vector operands
278 /// Select a 72-bit MVE vector rounding reduction with two vector operands
283 /// SelectMVE_VLD - Select MVE interleaving load intrinsics. NumVecs
285 /// used for 8, 16 and 32-bit lane sizes respectively, and each
286 /// pointer points to a set of NumVecs sub-opcodes used for the
291 /// SelectMVE_VxDUP - Select MVE incrementing-dup instructions. Opcodes is an
292 /// array of 3 elements for the 8, 16 and 32-bit lane sizes.
296 /// Select SelectCDE_CXxD - Select CDE dual-GPR instruction (one of CX1D,
305 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
330 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
377 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant in INITIALIZE_PASS()
378 /// operand. If so Imm will receive the 32-bit value. in INITIALIZE_PASS()
380 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { in INITIALIZE_PASS()
381 Imm = N->getAsZExtVal(); in INITIALIZE_PASS()
387 // isInt32Immediate - This method tests to see if a constant operand.
393 // isOpcWithIntImmediate - This method tests to see if the node is a specific
397 return N->getOpcode() == Opc && in isOpcWithIntImmediate()
398 isInt32Immediate(N->getOperand(1).getNode(), Imm); in isOpcWithIntImmediate()
404 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
415 ScaledConstant = (int) C->getZExtValue(); in isScaledConstantInRange()
424 if (!Subtarget->hasV6T2Ops()) in PreprocessISelDAG()
427 bool isThumb2 = Subtarget->isThumb(); in PreprocessISelDAG()
429 for (SDNode &N : llvm::make_early_inc_range(CurDAG->allnodes())) { in PreprocessISelDAG()
491 Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32, in PreprocessISelDAG()
493 CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl), in PreprocessISelDAG()
495 N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32, in PreprocessISelDAG()
497 CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32)); in PreprocessISelDAG()
498 N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32, in PreprocessISelDAG()
499 N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32)); in PreprocessISelDAG()
500 CurDAG->UpdateNodeOperands(&N, N0, N1); in PreprocessISelDAG()
504 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
511 if (!Subtarget->hasVMLxHazards()) in hasNoVMLxHazardUse()
514 if (!N->hasOneUse()) in hasNoVMLxHazardUse()
517 SDNode *Use = *N->use_begin(); in hasNoVMLxHazardUse()
518 if (Use->getOpcode() == ISD::CopyToReg) in hasNoVMLxHazardUse()
520 if (Use->isMachineOpcode()) { in hasNoVMLxHazardUse()
522 CurDAG->getSubtarget().getInstrInfo()); in hasNoVMLxHazardUse()
524 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); in hasNoVMLxHazardUse()
538 // This adds up to about 18 - 19 cycles. in hasNoVMLxHazardUse()
543 return TII->isFpMLxInstruction(Opcode); in hasNoVMLxHazardUse()
552 if (!Subtarget->isLikeA9() && !Subtarget->isSwift()) in isShifterOpProfitable()
558 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1)); in isShifterOpProfitable()
576 if (!MulConst->hasOneUse()) return false; in canExtractShiftFromMul()
577 unsigned MulConstVal = MulConst->getZExtValue(); in canExtractShiftFromMul()
583 --PowerOfTwo; in canExtractShiftFromMul()
589 NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32); in canExtractShiftFromMul()
596 CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode()); in replaceDAGValue()
607 // If N is a multiply-by-constant and it's profitable to extract a shift and in SelectImmShifterOperand()
617 Opc = CurDAG->getTargetConstant( in SelectImmShifterOperand()
633 ShImmVal = RHS->getZExtValue() & 31; in SelectImmShifterOperand()
634 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), in SelectImmShifterOperand()
661 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), in SelectRegShifterOperand()
669 assert(Parent->getOpcode() == ISD::OR && "unexpected parent"); in SelectAddLikeOr()
671 return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1)); in SelectAddLikeOr()
682 !CurDAG->isBaseWithConstantOffset(N)) { in SelectAddrModeImm12()
685 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrModeImm12()
686 Base = CurDAG->getTargetFrameIndex( in SelectAddrModeImm12()
687 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectAddrModeImm12()
688 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
699 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
704 int RHSC = (int)RHS->getSExtValue(); in SelectAddrModeImm12()
706 RHSC = -RHSC; in SelectAddrModeImm12()
708 if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits in SelectAddrModeImm12()
711 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrModeImm12()
712 Base = CurDAG->getTargetFrameIndex( in SelectAddrModeImm12()
713 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectAddrModeImm12()
715 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
722 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectAddrModeImm12()
731 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { in SelectLdStSOReg()
733 // X * [3,5,9] -> X + X * [2,4,8] etc. in SelectLdStSOReg()
734 int RHSC = (int)RHS->getZExtValue(); in SelectLdStSOReg()
740 RHSC = - RHSC; in SelectLdStSOReg()
745 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, in SelectLdStSOReg()
756 !CurDAG->isBaseWithConstantOffset(N)) in SelectLdStSOReg()
759 // Leave simple R +/- imm12 operands for LDRi12 in SelectLdStSOReg()
763 -0x1000+1, 0x1000, RHSC)) // 12 bits. in SelectLdStSOReg()
767 // Otherwise this is R +/- [possibly shifted] R. in SelectLdStSOReg()
781 ShAmt = Sh->getZExtValue(); in SelectLdStSOReg()
795 !(Subtarget->isLikeA9() || Subtarget->isSwift() || in SelectLdStSOReg()
803 ShAmt = Sh->getZExtValue(); in SelectLdStSOReg()
817 // If Offset is a multiply-by-constant and it's profitable to extract a shift in SelectLdStSOReg()
831 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), in SelectLdStSOReg()
838 unsigned Opcode = Op->getOpcode(); in SelectAddrMode2OffsetReg()
840 ? cast<LoadSDNode>(Op)->getAddressingMode() in SelectAddrMode2OffsetReg()
841 : cast<StoreSDNode>(Op)->getAddressingMode(); in SelectAddrMode2OffsetReg()
855 ShAmt = Sh->getZExtValue(); in SelectAddrMode2OffsetReg()
867 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), in SelectAddrMode2OffsetReg()
874 unsigned Opcode = Op->getOpcode(); in SelectAddrMode2OffsetImmPre()
876 ? cast<LoadSDNode>(Op)->getAddressingMode() in SelectAddrMode2OffsetImmPre()
877 : cast<StoreSDNode>(Op)->getAddressingMode(); in SelectAddrMode2OffsetImmPre()
882 if (AddSub == ARM_AM::sub) Val *= -1; in SelectAddrMode2OffsetImmPre()
883 Offset = CurDAG->getRegister(0, MVT::i32); in SelectAddrMode2OffsetImmPre()
884 Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32); in SelectAddrMode2OffsetImmPre()
894 unsigned Opcode = Op->getOpcode(); in SelectAddrMode2OffsetImm()
896 ? cast<LoadSDNode>(Op)->getAddressingMode() in SelectAddrMode2OffsetImm()
897 : cast<StoreSDNode>(Op)->getAddressingMode(); in SelectAddrMode2OffsetImm()
902 Offset = CurDAG->getRegister(0, MVT::i32); in SelectAddrMode2OffsetImm()
903 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, in SelectAddrMode2OffsetImm()
921 // X - C is canonicalize to X + -C, no need to handle it here. in SelectAddrMode3()
924 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N), in SelectAddrMode3()
929 if (!CurDAG->isBaseWithConstantOffset(N)) { in SelectAddrMode3()
932 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectAddrMode3()
933 Base = CurDAG->getTargetFrameIndex( in SelectAddrMode3()
934 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectAddrMode3()
936 Offset = CurDAG->getRegister(0, MVT::i32); in SelectAddrMode3()
937 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N), in SelectAddrMode3()
942 // If the RHS is +/- imm8, fold into addr mode. in SelectAddrMode3()
945 -256 + 1, 256, RHSC)) { // 8 bits. in SelectAddrMode3()
948 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectAddrMode3()
949 Base = CurDAG->getTargetFrameIndex( in SelectAddrMode3()
950 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectAddrMode3()
952 Offset = CurDAG->getRegister(0, MVT::i32); in SelectAddrMode3()
957 RHSC = -RHSC; in SelectAddrMode3()
959 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N), in SelectAddrMode3()
966 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N), in SelectAddrMode3()
973 unsigned Opcode = Op->getOpcode(); in SelectAddrMode3Offset()
975 ? cast<LoadSDNode>(Op)->getAddressingMode() in SelectAddrMode3Offset()
976 : cast<StoreSDNode>(Op)->getAddressingMode(); in SelectAddrMode3Offset()
981 Offset = CurDAG->getRegister(0, MVT::i32); in SelectAddrMode3Offset()
982 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op), in SelectAddrMode3Offset()
988 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op), in SelectAddrMode3Offset()
995 if (!CurDAG->isBaseWithConstantOffset(N)) { in IsAddressingMode5()
998 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in IsAddressingMode5()
999 Base = CurDAG->getTargetFrameIndex( in IsAddressingMode5()
1000 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in IsAddressingMode5()
1007 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), in IsAddressingMode5()
1012 // If the RHS is +/- imm8, fold into addr mode. in IsAddressingMode5()
1016 if (isScaledConstantInRange(N.getOperand(1), Scale, -255, 256, RHSC)) { in IsAddressingMode5()
1019 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in IsAddressingMode5()
1020 Base = CurDAG->getTargetFrameIndex( in IsAddressingMode5()
1021 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in IsAddressingMode5()
1027 RHSC = -RHSC; in IsAddressingMode5()
1031 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC), in IsAddressingMode5()
1034 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), in IsAddressingMode5()
1043 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0), in IsAddressingMode5()
1046 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), in IsAddressingMode5()
1071 ((MemN->getOpcode() == ARMISD::VST1_UPD || in SelectAddrMode6()
1072 MemN->getOpcode() == ARMISD::VLD1_UPD) && in SelectAddrMode6()
1073 MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) { in SelectAddrMode6()
1074 // This case occurs only for VLD1-lane/dup and VST1-lane instructions. in SelectAddrMode6()
1076 llvm::Align MMOAlign = MemN->getAlign(); in SelectAddrMode6()
1077 unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8; in SelectAddrMode6()
1084 Alignment = MemN->getAlign().value(); in SelectAddrMode6()
1087 Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32); in SelectAddrMode6()
1094 ISD::MemIndexedMode AM = LdSt->getAddressingMode(); in SelectAddrMode6Offset()
1099 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) in SelectAddrMode6Offset()
1100 Offset = CurDAG->getRegister(0, MVT::i32); in SelectAddrMode6Offset()
1110 Label = CurDAG->getTargetConstant(N1->getAsZExtVal(), SDLoc(N), MVT::i32); in SelectAddrModePC()
1118 //===----------------------------------------------------------------------===//
1120 //===----------------------------------------------------------------------===//
1131 return C->getSExtValue() < 0 && C->getSExtValue() >= -255; in shouldUseZeroOffsetLdSt()
1138 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { in SelectThumbAddrModeRRSext()
1163 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectThumbAddrModeImm5S()
1167 if (!CurDAG->isBaseWithConstantOffset(N)) { in SelectThumbAddrModeImm5S()
1180 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectThumbAddrModeImm5S()
1188 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectThumbAddrModeImm5S()
1217 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectThumbAddrModeSP()
1220 MachineFrameInfo &MFI = MF->getFrameInfo(); in SelectThumbAddrModeSP()
1223 Base = CurDAG->getTargetFrameIndex( in SelectThumbAddrModeSP()
1224 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectThumbAddrModeSP()
1225 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectThumbAddrModeSP()
1229 if (!CurDAG->isBaseWithConstantOffset(N)) in SelectThumbAddrModeSP()
1237 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectThumbAddrModeSP()
1239 // allocate an emergency spill slot. (An out-of-range access is UB, but in SelectThumbAddrModeSP()
1241 MachineFrameInfo &MFI = MF->getFrameInfo(); in SelectThumbAddrModeSP()
1244 // indexed by the LHS must be 4-byte aligned. in SelectThumbAddrModeSP()
1248 Base = CurDAG->getTargetFrameIndex( in SelectThumbAddrModeSP()
1249 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectThumbAddrModeSP()
1250 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectThumbAddrModeSP()
1263 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) { in SelectTAddrModeImm7()
1265 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80, in SelectTAddrModeImm7()
1269 RHSC = -RHSC; in SelectTAddrModeImm7()
1271 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32); in SelectTAddrModeImm7()
1278 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectTAddrModeImm7()
1283 //===----------------------------------------------------------------------===//
1285 //===----------------------------------------------------------------------===//
1294 !CurDAG->isBaseWithConstantOffset(N)) { in SelectT2AddrModeImm12()
1297 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in SelectT2AddrModeImm12()
1298 Base = CurDAG->getTargetFrameIndex( in SelectT2AddrModeImm12()
1299 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectT2AddrModeImm12()
1300 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1313 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1319 // Let t2LDRi8 handle (R - imm8). in SelectT2AddrModeImm12()
1322 int RHSC = (int)RHS->getZExtValue(); in SelectT2AddrModeImm12()
1324 RHSC = -RHSC; in SelectT2AddrModeImm12()
1329 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectT2AddrModeImm12()
1330 Base = CurDAG->getTargetFrameIndex( in SelectT2AddrModeImm12()
1331 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectT2AddrModeImm12()
1333 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1340 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm12()
1347 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) { in SelectT2AddrModeImm8()
1349 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -255, 256, RHSC)) { in SelectT2AddrModeImm8()
1352 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectT2AddrModeImm8()
1353 Base = CurDAG->getTargetFrameIndex( in SelectT2AddrModeImm8()
1354 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectT2AddrModeImm8()
1358 RHSC = -RHSC; in SelectT2AddrModeImm8()
1360 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32); in SelectT2AddrModeImm8()
1367 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm8()
1373 // Match simple R - imm8 operands. in SelectT2AddrModeImm8()
1375 !CurDAG->isBaseWithConstantOffset(N)) in SelectT2AddrModeImm8()
1379 int RHSC = (int)RHS->getSExtValue(); in SelectT2AddrModeImm8()
1381 RHSC = -RHSC; in SelectT2AddrModeImm8()
1383 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) in SelectT2AddrModeImm8()
1386 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectT2AddrModeImm8()
1387 Base = CurDAG->getTargetFrameIndex( in SelectT2AddrModeImm8()
1388 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectT2AddrModeImm8()
1390 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32); in SelectT2AddrModeImm8()
1400 unsigned Opcode = Op->getOpcode(); in SelectT2AddrModeImm8Offset()
1402 ? cast<LoadSDNode>(Op)->getAddressingMode() in SelectT2AddrModeImm8Offset()
1403 : cast<StoreSDNode>(Op)->getAddressingMode(); in SelectT2AddrModeImm8Offset()
1407 ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32) in SelectT2AddrModeImm8Offset()
1408 : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32); in SelectT2AddrModeImm8Offset()
1418 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) { in SelectT2AddrModeImm7()
1420 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80, in SelectT2AddrModeImm7()
1424 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectT2AddrModeImm7()
1425 Base = CurDAG->getTargetFrameIndex( in SelectT2AddrModeImm7()
1426 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectT2AddrModeImm7()
1430 RHSC = -RHSC; in SelectT2AddrModeImm7()
1432 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32); in SelectT2AddrModeImm7()
1439 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeImm7()
1452 unsigned Opcode = Op->getOpcode(); in SelectT2AddrModeImm7Offset()
1456 AM = cast<LoadSDNode>(Op)->getAddressingMode(); in SelectT2AddrModeImm7Offset()
1459 AM = cast<StoreSDNode>(Op)->getAddressingMode(); in SelectT2AddrModeImm7Offset()
1462 AM = cast<MaskedLoadSDNode>(Op)->getAddressingMode(); in SelectT2AddrModeImm7Offset()
1465 AM = cast<MaskedStoreSDNode>(Op)->getAddressingMode(); in SelectT2AddrModeImm7Offset()
1476 ? CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32) in SelectT2AddrModeImm7Offset()
1477 : CurDAG->getTargetConstant(-RHSC * (1 << Shift), SDLoc(N), in SelectT2AddrModeImm7Offset()
1488 OffImm = CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32); in SelectImmediateInRange()
1497 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. in SelectT2AddrModeSoReg()
1498 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) in SelectT2AddrModeSoReg()
1501 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. in SelectT2AddrModeSoReg()
1503 int RHSC = (int)RHS->getZExtValue(); in SelectT2AddrModeSoReg()
1506 else if (RHSC < 0 && RHSC >= -255) // 8 bits in SelectT2AddrModeSoReg()
1527 ShAmt = Sh->getZExtValue(); in SelectT2AddrModeSoReg()
1536 // If OffReg is a multiply-by-constant and it's profitable to extract a shift in SelectT2AddrModeSoReg()
1549 ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32); in SelectT2AddrModeSoReg()
1559 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32); in SelectT2AddrModeExclusive()
1561 if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N)) in SelectT2AddrModeExclusive()
1568 uint32_t RHSC = (int)RHS->getZExtValue(); in SelectT2AddrModeExclusive()
1574 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); in SelectT2AddrModeExclusive()
1575 Base = CurDAG->getTargetFrameIndex( in SelectT2AddrModeExclusive()
1576 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in SelectT2AddrModeExclusive()
1579 OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32); in SelectT2AddrModeExclusive()
1583 //===--------------------------------------------------------------------===//
1585 /// getAL - Returns a ARMCC::AL immediate node.
1587 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32); in getAL()
1591 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in transferMemOperands()
1592 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); in transferMemOperands()
1597 ISD::MemIndexedMode AM = LD->getAddressingMode(); in tryARMIndexedLoad()
1601 EVT LoadedVT = LD->getMemoryVT(); in tryARMIndexedLoad()
1607 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1611 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1615 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1620 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1622 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) in tryARMIndexedLoad()
1626 if (LD->getExtensionType() == ISD::SEXTLOAD) { in tryARMIndexedLoad()
1627 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1633 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1637 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1640 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { in tryARMIndexedLoad()
1649 SDValue Chain = LD->getChain(); in tryARMIndexedLoad()
1650 SDValue Base = LD->getBasePtr(); in tryARMIndexedLoad()
1652 CurDAG->getRegister(0, MVT::i32), Chain }; in tryARMIndexedLoad()
1653 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, in tryARMIndexedLoad()
1659 SDValue Chain = LD->getChain(); in tryARMIndexedLoad()
1660 SDValue Base = LD->getBasePtr(); in tryARMIndexedLoad()
1662 CurDAG->getRegister(0, MVT::i32), Chain }; in tryARMIndexedLoad()
1663 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, in tryARMIndexedLoad()
1676 EVT LoadedVT = LD->getMemoryVT(); in tryT1IndexedLoad()
1677 ISD::MemIndexedMode AM = LD->getAddressingMode(); in tryT1IndexedLoad()
1678 if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD || in tryT1IndexedLoad()
1682 auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset()); in tryT1IndexedLoad()
1683 if (!COffs || COffs->getZExtValue() != 4) in tryT1IndexedLoad()
1686 // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}. in tryT1IndexedLoad()
1687 // The encoding of LDM is not how the rest of ISel expects a post-inc load to in tryT1IndexedLoad()
1690 SDValue Chain = LD->getChain(); in tryT1IndexedLoad()
1691 SDValue Base = LD->getBasePtr(); in tryT1IndexedLoad()
1693 CurDAG->getRegister(0, MVT::i32), Chain }; in tryT1IndexedLoad()
1694 SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32, in tryT1IndexedLoad()
1703 ISD::MemIndexedMode AM = LD->getAddressingMode(); in tryT2IndexedLoad()
1707 EVT LoadedVT = LD->getMemoryVT(); in tryT2IndexedLoad()
1708 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; in tryT2IndexedLoad()
1713 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { in tryT2IndexedLoad()
1738 SDValue Chain = LD->getChain(); in tryT2IndexedLoad()
1739 SDValue Base = LD->getBasePtr(); in tryT2IndexedLoad()
1741 CurDAG->getRegister(0, MVT::i32), Chain }; in tryT2IndexedLoad()
1742 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, in tryT2IndexedLoad()
1762 ISD::MemIndexedMode AM = LD->getAddressingMode(); in tryMVEIndexedLoad()
1765 LoadedVT = LD->getMemoryVT(); in tryMVEIndexedLoad()
1769 Chain = LD->getChain(); in tryMVEIndexedLoad()
1770 Base = LD->getBasePtr(); in tryMVEIndexedLoad()
1771 Offset = LD->getOffset(); in tryMVEIndexedLoad()
1772 Alignment = LD->getAlign(); in tryMVEIndexedLoad()
1773 isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; in tryMVEIndexedLoad()
1776 PredReg = CurDAG->getRegister(0, MVT::i32); in tryMVEIndexedLoad()
1778 ISD::MemIndexedMode AM = LD->getAddressingMode(); in tryMVEIndexedLoad()
1781 LoadedVT = LD->getMemoryVT(); in tryMVEIndexedLoad()
1785 Chain = LD->getChain(); in tryMVEIndexedLoad()
1786 Base = LD->getBasePtr(); in tryMVEIndexedLoad()
1787 Offset = LD->getOffset(); in tryMVEIndexedLoad()
1788 Alignment = LD->getAlign(); in tryMVEIndexedLoad()
1789 isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; in tryMVEIndexedLoad()
1792 PredReg = LD->getMask(); in tryMVEIndexedLoad()
1794 llvm_unreachable("Expected a Load or a Masked Load!"); in tryMVEIndexedLoad()
1796 // We allow LE non-masked loads to change the type (for example use a vldrb.8 in tryMVEIndexedLoad()
1799 bool CanChangeType = Subtarget->isLittle() && !isa<MaskedLoadSDNode>(N); in tryMVEIndexedLoad()
1838 CurDAG->getTargetConstant(Pred, SDLoc(N), MVT::i32), in tryMVEIndexedLoad()
1840 CurDAG->getRegister(0, MVT::i32), // tp_reg in tryMVEIndexedLoad()
1842 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, in tryMVEIndexedLoad()
1843 N->getValueType(0), MVT::Other, Ops); in tryMVEIndexedLoad()
1848 CurDAG->RemoveDeadNode(N); in tryMVEIndexedLoad()
1856 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32); in createGPRPairNode()
1857 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32); in createGPRPairNode()
1858 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32); in createGPRPairNode()
1860 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createGPRPairNode()
1867 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32); in createSRegPairNode()
1868 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32); in createSRegPairNode()
1869 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32); in createSRegPairNode()
1871 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createSRegPairNode()
1877 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl, in createDRegPairNode()
1879 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32); in createDRegPairNode()
1880 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32); in createDRegPairNode()
1882 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createDRegPairNode()
1888 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl, in createQRegPairNode()
1890 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32); in createQRegPairNode()
1891 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32); in createQRegPairNode()
1893 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createQRegPairNode()
1901 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32); in createQuadSRegsNode()
1902 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32); in createQuadSRegsNode()
1903 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32); in createQuadSRegsNode()
1904 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32); in createQuadSRegsNode()
1905 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32); in createQuadSRegsNode()
1908 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createQuadSRegsNode()
1915 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl, in createQuadDRegsNode()
1917 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32); in createQuadDRegsNode()
1918 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32); in createQuadDRegsNode()
1919 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32); in createQuadDRegsNode()
1920 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32); in createQuadDRegsNode()
1923 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createQuadDRegsNode()
1930 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl, in createQuadQRegsNode()
1932 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32); in createQuadQRegsNode()
1933 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32); in createQuadQRegsNode()
1934 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32); in createQuadQRegsNode()
1935 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32); in createQuadQRegsNode()
1938 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); in createQuadQRegsNode()
1941 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1950 unsigned Alignment = Align->getAsZExtVal(); in GetVLDSTAlign()
1960 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32); in GetVLDSTAlign()
2112 return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs; in isPerfectIncrement()
2119 assert(Subtarget->hasNEON()); in SelectVLD()
2120 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); in SelectVLD()
2127 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVLD()
2130 SDValue Chain = N->getOperand(0); in SelectVLD()
2131 EVT VT = N->getValueType(0); in SelectVLD()
2138 // Double-register operations: in SelectVLD()
2146 // Quad-register operations: in SelectVLD()
2164 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); in SelectVLD()
2173 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in SelectVLD()
2184 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVLD()
2200 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectVLD()
2210 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); in SelectVLD()
2212 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, in SelectVLD()
2220 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVLD()
2222 "only constant post-increment update allowed for VLD3/4"); in SelectVLD()
2230 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); in SelectVLD()
2234 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVLD()
2235 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp}); in SelectVLD()
2250 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); in SelectVLD()
2254 CurDAG->RemoveDeadNode(N); in SelectVLD()
2261 assert(Subtarget->hasNEON()); in SelectVST()
2262 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); in SelectVST()
2270 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVST()
2273 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVST()
2275 SDValue Chain = N->getOperand(0); in SelectVST()
2276 EVT VT = N->getOperand(Vec0Idx).getValueType(); in SelectVST()
2283 // Double-register operations: in SelectVST()
2291 // Quad-register operations: in SelectVST()
2308 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in SelectVST()
2315 SrcReg = N->getOperand(Vec0Idx); in SelectVST()
2318 SDValue V0 = N->getOperand(Vec0Idx + 0); in SelectVST()
2319 SDValue V1 = N->getOperand(Vec0Idx + 1); in SelectVST()
2323 SDValue V2 = N->getOperand(Vec0Idx + 2); in SelectVST()
2324 // If it's a vst3, form a quad D-register and leave the last part as in SelectVST()
2327 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) in SelectVST()
2328 : N->getOperand(Vec0Idx + 3); in SelectVST()
2333 SDValue Q0 = N->getOperand(Vec0Idx); in SelectVST()
2334 SDValue Q1 = N->getOperand(Vec0Idx + 1); in SelectVST()
2343 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVST()
2361 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectVST()
2364 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp}); in SelectVST()
2374 SDValue V0 = N->getOperand(Vec0Idx + 0); in SelectVST()
2375 SDValue V1 = N->getOperand(Vec0Idx + 1); in SelectVST()
2376 SDValue V2 = N->getOperand(Vec0Idx + 2); in SelectVST()
2378 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) in SelectVST()
2379 : N->getOperand(Vec0Idx + 3); in SelectVST()
2385 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, in SelectVST()
2388 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp}); in SelectVST()
2395 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVST()
2397 "only constant post-increment update allowed for VST3/4"); in SelectVST()
2405 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, in SelectVST()
2407 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp}); in SelectVST()
2415 assert(Subtarget->hasNEON()); in SelectVLDSTLane()
2416 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); in SelectVLDSTLane()
2424 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVLDSTLane()
2427 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVLDSTLane()
2429 SDValue Chain = N->getOperand(0); in SelectVLDSTLane()
2430 unsigned Lane = N->getConstantOperandVal(Vec0Idx + NumVecs); in SelectVLDSTLane()
2431 EVT VT = N->getOperand(Vec0Idx).getValueType(); in SelectVLDSTLane()
2436 Alignment = Align->getAsZExtVal(); in SelectVLDSTLane()
2443 Alignment = (Alignment & -Alignment); in SelectVLDSTLane()
2447 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32); in SelectVLDSTLane()
2452 // Double-register operations: in SelectVLDSTLane()
2459 // Quad-register operations: in SelectVLDSTLane()
2472 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), in SelectVLDSTLane()
2480 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in SelectVLDSTLane()
2486 SDValue Inc = N->getOperand(AddrOpIdx + 1); in SelectVLDSTLane()
2493 SDValue V0 = N->getOperand(Vec0Idx + 0); in SelectVLDSTLane()
2494 SDValue V1 = N->getOperand(Vec0Idx + 1); in SelectVLDSTLane()
2501 SDValue V2 = N->getOperand(Vec0Idx + 2); in SelectVLDSTLane()
2503 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) in SelectVLDSTLane()
2504 : N->getOperand(Vec0Idx + 3); in SelectVLDSTLane()
2518 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectVLDSTLane()
2519 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp}); in SelectVLDSTLane()
2533 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); in SelectVLDSTLane()
2537 CurDAG->RemoveDeadNode(N); in SelectVLDSTLane()
2543 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::Then, Loc, MVT::i32)); in AddMVEPredicateToOps()
2545 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg in AddMVEPredicateToOps()
2552 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::Then, Loc, MVT::i32)); in AddMVEPredicateToOps()
2554 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg in AddMVEPredicateToOps()
2560 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::None, Loc, MVT::i32)); in AddEmptyMVEPredicateToOps()
2561 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in AddEmptyMVEPredicateToOps()
2562 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg in AddEmptyMVEPredicateToOps()
2568 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::None, Loc, MVT::i32)); in AddEmptyMVEPredicateToOps()
2569 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in AddEmptyMVEPredicateToOps()
2570 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg in AddEmptyMVEPredicateToOps()
2572 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0)); in AddEmptyMVEPredicateToOps()
2581 switch (N->getValueType(1).getVectorElementType().getSizeInBits()) { in SelectMVE_WB()
2592 Ops.push_back(N->getOperand(2)); // vector of base addresses in SelectMVE_WB()
2594 int32_t ImmValue = N->getConstantOperandVal(3); in SelectMVE_WB()
2598 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4)); in SelectMVE_WB()
2602 Ops.push_back(N->getOperand(0)); // chain in SelectMVE_WB()
2605 VTs.push_back(N->getValueType(1)); in SelectMVE_WB()
2606 VTs.push_back(N->getValueType(0)); in SelectMVE_WB()
2607 VTs.push_back(N->getValueType(2)); in SelectMVE_WB()
2609 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), VTs, Ops); in SelectMVE_WB()
2614 CurDAG->RemoveDeadNode(N); in SelectMVE_WB()
2623 // Two 32-bit halves of the value to be shifted in SelectMVE_LongShift()
2624 Ops.push_back(N->getOperand(1)); in SelectMVE_LongShift()
2625 Ops.push_back(N->getOperand(2)); in SelectMVE_LongShift()
2629 int32_t ImmValue = N->getConstantOperandVal(3); in SelectMVE_LongShift()
2632 Ops.push_back(N->getOperand(3)); in SelectMVE_LongShift()
2637 int32_t SatOp = N->getConstantOperandVal(4); in SelectMVE_LongShift()
2642 // MVE scalar shifts are IT-predicable, so include the standard in SelectMVE_LongShift()
2645 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in SelectMVE_LongShift()
2647 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectMVE_LongShift()
2660 Ops.push_back(N->getOperand(FirstInputOp)); in SelectMVE_VADCSBC()
2661 Ops.push_back(N->getOperand(FirstInputOp + 1)); in SelectMVE_VADCSBC()
2662 SDValue CarryIn = N->getOperand(FirstInputOp + 2); in SelectMVE_VADCSBC()
2667 (CarryInConstant->getZExtValue() & CarryMask) == CarryExpected) { in SelectMVE_VADCSBC()
2676 N->getOperand(FirstInputOp + 3), // predicate in SelectMVE_VADCSBC()
2677 N->getOperand(FirstInputOp - 1)); // inactive in SelectMVE_VADCSBC()
2679 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0)); in SelectMVE_VADCSBC()
2681 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectMVE_VADCSBC()
2688 // One vector input, followed by a 32-bit word of bits to shift in in SelectMVE_VSHLC()
2690 Ops.push_back(N->getOperand(1)); in SelectMVE_VSHLC()
2691 Ops.push_back(N->getOperand(2)); in SelectMVE_VSHLC()
2692 int32_t ImmValue = N->getConstantOperandVal(3); in SelectMVE_VSHLC()
2696 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4)); in SelectMVE_VSHLC()
2700 CurDAG->SelectNodeTo(N, ARM::MVE_VSHLC, N->getVTList(), ArrayRef(Ops)); in SelectMVE_VSHLC()
2704 assert(isa<ConstantSDNode>(SDVal) && "expected a compile-time constant"); in SDValueToConstBool()
2706 uint64_t Value = SDValConstant->getZExtValue(); in SDValueToConstBool()
2716 bool IsUnsigned = SDValueToConstBool(N->getOperand(1)); in SelectBaseMVE_VMLLDAV()
2717 bool IsSub = SDValueToConstBool(N->getOperand(2)); in SelectBaseMVE_VMLLDAV()
2718 bool IsExchange = SDValueToConstBool(N->getOperand(3)); in SelectBaseMVE_VMLLDAV()
2727 return isNullConstant(N->getOperand(OpNo)); in SelectBaseMVE_VMLLDAV()
2747 Ops.push_back(N->getOperand(4)); in SelectBaseMVE_VMLLDAV()
2748 Ops.push_back(N->getOperand(5)); in SelectBaseMVE_VMLLDAV()
2751 Ops.push_back(N->getOperand(6)); in SelectBaseMVE_VMLLDAV()
2752 Ops.push_back(N->getOperand(7)); in SelectBaseMVE_VMLLDAV()
2755 AddMVEPredicateToOps(Ops, Loc, N->getOperand(8)); in SelectBaseMVE_VMLLDAV()
2759 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectBaseMVE_VMLLDAV()
2765 EVT VecTy = N->getOperand(6).getValueType(); in SelectMVE_VMLLDAV()
2785 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() == in SelectMVE_VRMLLDAVH()
2794 EVT VT = N->getValueType(0); in SelectMVE_VLD()
2812 EVT DataTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, NumVecs * 2); in SelectMVE_VLD()
2817 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0); in SelectMVE_VLD()
2818 SDValue Chain = N->getOperand(0); in SelectMVE_VLD()
2820 for (unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) { in SelectMVE_VLD()
2821 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain}; in SelectMVE_VLD()
2823 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops); in SelectMVE_VLD()
2831 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain}; in SelectMVE_VLD()
2833 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops); in SelectMVE_VLD()
2839 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT, in SelectMVE_VLD()
2844 CurDAG->RemoveDeadNode(N); in SelectMVE_VLD()
2849 EVT VT = N->getValueType(0); in SelectMVE_VxDUP()
2872 Inactive = N->getOperand(OpIdx++); in SelectMVE_VxDUP()
2874 Ops.push_back(N->getOperand(OpIdx++)); // base in SelectMVE_VxDUP()
2876 Ops.push_back(N->getOperand(OpIdx++)); // limit in SelectMVE_VxDUP()
2878 SDValue ImmOp = N->getOperand(OpIdx++); // step in SelectMVE_VxDUP()
2879 int ImmValue = ImmOp->getAsZExtVal(); in SelectMVE_VxDUP()
2883 AddMVEPredicateToOps(Ops, Loc, N->getOperand(OpIdx), Inactive); in SelectMVE_VxDUP()
2885 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0)); in SelectMVE_VxDUP()
2887 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops)); in SelectMVE_VxDUP()
2892 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian(); in SelectCDE_CXxD()
2899 SDValue ImmCorpoc = N->getOperand(OpIdx++); in SelectCDE_CXxD()
2900 uint32_t ImmCoprocVal = ImmCorpoc->getAsZExtVal(); in SelectCDE_CXxD()
2906 SDValue AccLo = N->getOperand(OpIdx++); in SelectCDE_CXxD()
2907 SDValue AccHi = N->getOperand(OpIdx++); in SelectCDE_CXxD()
2913 // Copy extra operands as-is. in SelectCDE_CXxD()
2915 Ops.push_back(N->getOperand(OpIdx++)); in SelectCDE_CXxD()
2918 SDValue Imm = N->getOperand(OpIdx); in SelectCDE_CXxD()
2919 uint32_t ImmVal = Imm->getAsZExtVal(); in SelectCDE_CXxD()
2922 // Accumulating variants are IT-predicable, add predicate operands. in SelectCDE_CXxD()
2925 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); in SelectCDE_CXxD()
2931 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops); in SelectCDE_CXxD()
2934 // The original intrinsic had two outputs, and the output of the dual-register in SelectCDE_CXxD()
2945 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc, in SelectCDE_CXxD()
2950 CurDAG->RemoveDeadNode(N); in SelectCDE_CXxD()
2958 assert(Subtarget->hasNEON()); in SelectVLDDup()
2959 assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); in SelectVLDDup()
2964 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) in SelectVLDDup()
2967 SDValue Chain = N->getOperand(0); in SelectVLDDup()
2968 EVT VT = N->getValueType(0); in SelectVLDDup()
2973 Alignment = Align->getAsZExtVal(); in SelectVLDDup()
2980 Alignment = (Alignment & -Alignment); in SelectVLDDup()
2984 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32); in SelectVLDDup()
2988 default: llvm_unreachable("unhandled vld-dup type"); in SelectVLDDup()
3009 EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); in SelectVLDDup()
3018 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in SelectVLDDup()
3027 SDValue Inc = N->getOperand(2); in SelectVLDDup()
3043 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); in SelectVLDDup()
3045 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, ResTy, in SelectVLDDup()
3055 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); in SelectVLDDup()
3058 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in SelectVLDDup()
3059 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp}); in SelectVLDDup()
3070 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); in SelectVLDDup()
3076 CurDAG->RemoveDeadNode(N); in SelectVLDDup()
3080 if (!Subtarget->hasMVEIntegerOps()) in tryInsertVectorElt()
3089 SDValue Ins2 = N->getOperand(0); in tryInsertVectorElt()
3127 SDValue NewExt = CurDAG->getTargetExtractSubreg( in tryInsertVectorElt()
3129 SDValue NewIns = CurDAG->getTargetInsertSubreg( in tryInsertVectorElt()
3138 if (VT == MVT::v8i16 && Subtarget->hasFullFP16()) { in tryInsertVectorElt()
3139 SDValue Inp1 = CurDAG->getTargetExtractSubreg( in tryInsertVectorElt()
3141 SDValue Inp2 = CurDAG->getTargetExtractSubreg( in tryInsertVectorElt()
3144 Inp1 = SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0); in tryInsertVectorElt()
3146 Inp2 = SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0); in tryInsertVectorElt()
3147 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1); in tryInsertVectorElt()
3149 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32, in tryInsertVectorElt()
3156 // The inserted values are not extracted - if they are f16 then insert them in tryInsertVectorElt()
3158 if (VT == MVT::v8f16 && Subtarget->hasFullFP16()) { in tryInsertVectorElt()
3159 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1); in tryInsertVectorElt()
3161 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32, in tryInsertVectorElt()
3174 auto Type = N->getValueType(0); in transformFixedFloatingPointConversion()
3179 SDNodeFlags FMulFlags = FMul->getFlags(); in transformFixedFloatingPointConversion()
3180 // The fixed-point vcvt and vcvt+vmul are not always equivalent if inf is in transformFixedFloatingPointConversion()
3185 SDValue ImmNode = FMul->getOperand(1); in transformFixedFloatingPointConversion()
3186 SDValue VecVal = FMul->getOperand(0); in transformFixedFloatingPointConversion()
3187 if (VecVal->getOpcode() == ISD::UINT_TO_FP || in transformFixedFloatingPointConversion()
3188 VecVal->getOpcode() == ISD::SINT_TO_FP) in transformFixedFloatingPointConversion()
3189 VecVal = VecVal->getOperand(0); in transformFixedFloatingPointConversion()
3226 // from float to fixed and multiplying by 2^-n will convert from fixed to in transformFixedFloatingPointConversion()
3246 VecVal, CurDAG->getConstant(FracBits, SDLoc(N), MVT::i32)}; in transformFixedFloatingPointConversion()
3268 ReplaceNode(N, CurDAG->getMachineNode(Opcode, SDLoc(N), Type, Ops)); in transformFixedFloatingPointConversion()
3273 // Transform a floating-point to fixed-point conversion to a VCVT in tryFP_TO_INT()
3274 if (!Subtarget->hasMVEFloatOps()) in tryFP_TO_INT()
3276 EVT Type = N->getValueType(0); in tryFP_TO_INT()
3281 bool IsUnsigned = N->getOpcode() == ISD::FP_TO_UINT || in tryFP_TO_INT()
3282 N->getOpcode() == ISD::FP_TO_UINT_SAT; in tryFP_TO_INT()
3283 SDNode *Node = N->getOperand(0).getNode(); in tryFP_TO_INT()
3285 // floating-point to fixed-point with one fractional bit gets turned into an in tryFP_TO_INT()
3287 if (Node->getOpcode() == ISD::FADD) { in tryFP_TO_INT()
3288 if (Node->getOperand(0) != Node->getOperand(1)) in tryFP_TO_INT()
3290 SDNodeFlags Flags = Node->getFlags(); in tryFP_TO_INT()
3291 // The fixed-point vcvt and vcvt+vmul are not always equivalent if inf is in tryFP_TO_INT()
3305 SmallVector<SDValue, 3> Ops{Node->getOperand(0), in tryFP_TO_INT()
3306 CurDAG->getConstant(1, dl, MVT::i32)}; in tryFP_TO_INT()
3309 ReplaceNode(N, CurDAG->getMachineNode(Opcode, dl, Type, Ops)); in tryFP_TO_INT()
3313 if (Node->getOpcode() != ISD::FMUL) in tryFP_TO_INT()
3320 // Transform a fixed-point to floating-point conversion to a VCVT in tryFMULFixed()
3321 if (!Subtarget->hasMVEFloatOps()) in tryFMULFixed()
3323 auto Type = N->getValueType(0); in tryFMULFixed()
3327 auto LHS = N->getOperand(0); in tryFMULFixed()
3336 if (!Subtarget->hasV6T2Ops()) in tryV6T2BitfieldExtractOp()
3340 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) in tryV6T2BitfieldExtractOp()
3341 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); in tryV6T2BitfieldExtractOp()
3346 if (N->getOpcode() == ISD::AND) { in tryV6T2BitfieldExtractOp()
3354 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, in tryV6T2BitfieldExtractOp()
3361 And_imm &= -1U >> Srl_imm; in tryV6T2BitfieldExtractOp()
3363 // Note: The width operand is encoded as width-1. in tryV6T2BitfieldExtractOp()
3364 unsigned Width = llvm::countr_one(And_imm) - 1; in tryV6T2BitfieldExtractOp()
3367 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in tryV6T2BitfieldExtractOp()
3369 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { in tryV6T2BitfieldExtractOp()
3371 if (Subtarget->isThumb()) { in tryV6T2BitfieldExtractOp()
3373 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3374 CurDAG->getTargetConstant(LSB, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3376 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3383 CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl, in tryV6T2BitfieldExtractOp()
3385 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, in tryV6T2BitfieldExtractOp()
3387 CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3392 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3393 CurDAG->getTargetConstant(LSB, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3394 CurDAG->getTargetConstant(Width, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3396 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3405 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { in tryV6T2BitfieldExtractOp()
3408 if (isInt32Immediate(N->getOperand(1), Srl_imm)) { in tryV6T2BitfieldExtractOp()
3410 // Note: The width operand is encoded as width-1. in tryV6T2BitfieldExtractOp()
3411 unsigned Width = 32 - Srl_imm - 1; in tryV6T2BitfieldExtractOp()
3412 int LSB = Srl_imm - Shl_imm; in tryV6T2BitfieldExtractOp()
3415 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in tryV6T2BitfieldExtractOp()
3417 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3418 CurDAG->getTargetConstant(LSB, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3419 CurDAG->getTargetConstant(Width, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3421 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3427 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) && in tryV6T2BitfieldExtractOp()
3432 if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) { in tryV6T2BitfieldExtractOp()
3435 // Note: The width operand is encoded as width-1. in tryV6T2BitfieldExtractOp()
3436 unsigned Width = MSB - LSB; in tryV6T2BitfieldExtractOp()
3437 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in tryV6T2BitfieldExtractOp()
3439 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3440 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3441 CurDAG->getTargetConstant(Width, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3443 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3448 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) { in tryV6T2BitfieldExtractOp()
3449 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); in tryV6T2BitfieldExtractOp()
3451 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) && in tryV6T2BitfieldExtractOp()
3452 !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB)) in tryV6T2BitfieldExtractOp()
3458 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in tryV6T2BitfieldExtractOp()
3460 SDValue Ops[] = { N->getOperand(0).getOperand(0), in tryV6T2BitfieldExtractOp()
3461 CurDAG->getTargetConstant(LSB, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3462 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32), in tryV6T2BitfieldExtractOp()
3464 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in tryV6T2BitfieldExtractOp()
3471 /// Target-specific DAG combining for ISD::SUB.
3472 /// Target-independent combining lowers SELECT_CC nodes of the form
3473 /// select_cc setg[ge] X, 0, X, -X
3474 /// select_cc setgt X, -1, X, -X
3475 /// select_cc setl[te] X, 0, -X, X
3476 /// select_cc setlt X, 1, -X, X
3478 /// Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
3482 SDValue SUBSrc0 = N->getOperand(0); in tryABSOp()
3483 SDValue SUBSrc1 = N->getOperand(1); in tryABSOp()
3484 EVT VT = N->getValueType(0); in tryABSOp()
3486 if (Subtarget->isThumb1Only()) in tryABSOp()
3498 unsigned Size = XType.getSizeInBits() - 1; in tryABSOp()
3501 SRAConstant != nullptr && Size == SRAConstant->getZExtValue()) { in tryABSOp()
3502 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; in tryABSOp()
3503 CurDAG->SelectNodeTo(N, Opcode, VT, XORSrc0); in tryABSOp()
3510 /// We've got special pseudo-instructions for these
3513 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT(); in SelectCMP_SWAP()
3515 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8; in SelectCMP_SWAP()
3517 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16; in SelectCMP_SWAP()
3519 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32; in SelectCMP_SWAP()
3523 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3), in SelectCMP_SWAP()
3524 N->getOperand(0)}; in SelectCMP_SWAP()
3525 SDNode *CmpSwap = CurDAG->getMachineNode( in SelectCMP_SWAP()
3527 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops); in SelectCMP_SWAP()
3529 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in SelectCMP_SWAP()
3530 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp}); in SelectCMP_SWAP()
3534 CurDAG->RemoveDeadNode(N); in SelectCMP_SWAP()
3539 unsigned FirstOne = A.getBitWidth() - A.countl_zero() - 1; in getContiguousRangeOfSetBits()
3541 if (A.popcount() != (FirstOne - LastOne + 1)) in getContiguousRangeOfSetBits()
3547 assert(N->getOpcode() == ARMISD::CMPZ); in SelectCMPZ()
3550 if (!Subtarget->isThumb()) in SelectCMPZ()
3551 // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and in SelectCMPZ()
3552 // LSR don't exist as standalone instructions - they need the barrel shifter. in SelectCMPZ()
3555 // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X)) in SelectCMPZ()
3556 SDValue And = N->getOperand(0); in SelectCMPZ()
3557 if (!And->hasOneUse()) in SelectCMPZ()
3560 SDValue Zero = N->getOperand(1); in SelectCMPZ()
3561 if (!isNullConstant(Zero) || And->getOpcode() != ISD::AND) in SelectCMPZ()
3568 auto Range = getContiguousRangeOfSetBits(C->getAPIntValue()); in SelectCMPZ()
3576 auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* { in SelectCMPZ()
3577 if (Subtarget->isThumb2()) { in SelectCMPZ()
3579 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32), in SelectCMPZ()
3580 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32), in SelectCMPZ()
3581 CurDAG->getRegister(0, MVT::i32) }; in SelectCMPZ()
3582 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops); in SelectCMPZ()
3584 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src, in SelectCMPZ()
3585 CurDAG->getTargetConstant(Imm, dl, MVT::i32), in SelectCMPZ()
3586 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)}; in SelectCMPZ()
3587 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops); in SelectCMPZ()
3591 if (Range->second == 0) { in SelectCMPZ()
3592 // 1. Mask includes the LSB -> Simply shift the top N bits off in SelectCMPZ()
3593 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first); in SelectCMPZ()
3595 } else if (Range->first == 31) { in SelectCMPZ()
3596 // 2. Mask includes the MSB -> Simply shift the bottom N bits off in SelectCMPZ()
3597 NewN = EmitShift(ARM::tLSRri, X, Range->second); in SelectCMPZ()
3599 } else if (Range->first == Range->second) { in SelectCMPZ()
3602 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first); in SelectCMPZ()
3606 } else if (!Subtarget->hasV6T2Ops()) { in SelectCMPZ()
3608 // thumb-1 mode as in thumb-2 we can use UBFX. in SelectCMPZ()
3609 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first); in SelectCMPZ()
3611 Range->second + (31 - Range->first)); in SelectCMPZ()
3635 if (N->isMachineOpcode()) { in Select()
3636 N->setNodeId(-1); in Select()
3640 switch (N->getOpcode()) { in Select()
3643 // For Thumb1, match an sp-relative store in C++. This is a little in Select()
3649 // This is only necessary on Thumb1 because Thumb1 sp-relative stores use in Select()
3650 // a different addressing mode from other four-byte stores. in Select()
3654 SDValue Ptr = ST->getBasePtr(); in Select()
3655 if (Subtarget->isThumb1Only() && ST->isUnindexed()) { in Select()
3662 cast<RegisterSDNode>(Ptr.getOperand(1))->getReg() == ARM::SP && in Select()
3663 Ptr.getOperand(0) == ST->getChain()) { in Select()
3664 SDValue Ops[] = {ST->getValue(), in Select()
3665 CurDAG->getRegister(ARM::SP, MVT::i32), in Select()
3666 CurDAG->getTargetConstant(RHSC, dl, MVT::i32), in Select()
3668 CurDAG->getRegister(0, MVT::i32), in Select()
3669 ST->getChain()}; in Select()
3671 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops); in Select()
3672 MachineMemOperand *MemOp = ST->getMemOperand(); in Select()
3673 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp}); in Select()
3700 unsigned Val = N->getAsZExtVal(); in Select()
3703 !Subtarget->genExecuteOnly()) { in Select()
3704 SDValue CPIdx = CurDAG->getTargetConstantPool( in Select()
3705 ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val), in Select()
3706 TLI->getPointerTy(CurDAG->getDataLayout())); in Select()
3709 if (Subtarget->isThumb()) { in Select()
3713 CurDAG->getRegister(0, MVT::i32), in Select()
3714 CurDAG->getEntryNode() in Select()
3716 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, in Select()
3721 CurDAG->getTargetConstant(0, dl, MVT::i32), in Select()
3723 CurDAG->getRegister(0, MVT::i32), in Select()
3724 CurDAG->getEntryNode() in Select()
3726 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, in Select()
3732 MachineFunction& MF = CurDAG->getMachineFunction(); in Select()
3737 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp}); in Select()
3748 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in Select()
3749 SDValue TFI = CurDAG->getTargetFrameIndex( in Select()
3750 FI, TLI->getPointerTy(CurDAG->getDataLayout())); in Select()
3751 if (Subtarget->isThumb1Only()) { in Select()
3754 MachineFrameInfo &MFI = MF->getFrameInfo(); in Select()
3757 CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI, in Select()
3758 CurDAG->getTargetConstant(0, dl, MVT::i32)); in Select()
3761 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? in Select()
3763 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32), in Select()
3764 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32), in Select()
3765 CurDAG->getRegister(0, MVT::i32) }; in Select()
3766 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); in Select()
3796 if (Subtarget->isThumb1Only()) in Select()
3798 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { in Select()
3799 unsigned RHSV = C->getZExtValue(); in Select()
3801 if (isPowerOf2_32(RHSV-1)) { // 2^n+1? in Select()
3802 unsigned ShImm = Log2_32(RHSV-1); in Select()
3805 SDValue V = N->getOperand(0); in Select()
3807 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32); in Select()
3808 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in Select()
3809 if (Subtarget->isThumb()) { in Select()
3811 CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops); in Select()
3816 CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops); in Select()
3820 if (isPowerOf2_32(RHSV+1)) { // 2^n-1? in Select()
3824 SDValue V = N->getOperand(0); in Select()
3826 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32); in Select()
3827 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in Select()
3828 if (Subtarget->isThumb()) { in Select()
3830 CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops); in Select()
3835 CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops); in Select()
3849 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)); in Select()
3850 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) { in Select()
3851 uint32_t Imm = (uint32_t) N1C->getZExtValue(); in Select()
3853 // In Thumb2 mode, an AND can take a 12-bit immediate. If this in Select()
3858 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm)); in Select()
3865 CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32); in Select()
3868 if (NewImm->getNodeId() == -1) in Select()
3869 CurDAG->RepositionNode(N->getIterator(), NewImm.getNode()); in Select()
3871 if (!Subtarget->hasThumb2()) { in Select()
3872 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), in Select()
3873 N->getOperand(0), NewImm, getAL(CurDAG, dl), in Select()
3874 CurDAG->getRegister(0, MVT::i32)}; in Select()
3875 ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops)); in Select()
3878 SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl), in Select()
3879 CurDAG->getRegister(0, MVT::i32), in Select()
3880 CurDAG->getRegister(0, MVT::i32)}; in Select()
3882 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops)); in Select()
3888 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits in Select()
3889 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits in Select()
3890 // are entirely contributed by c2 and lower 16-bits are entirely contributed in Select()
3893 EVT VT = N->getValueType(0); in Select()
3896 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) in Select()
3898 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); in Select()
3901 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in Select()
3905 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { in Select()
3910 unsigned N1CVal = N1C->getZExtValue(); in Select()
3911 unsigned N2CVal = N2C->getZExtValue(); in Select()
3915 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, in Select()
3918 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) }; in Select()
3919 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops)); in Select()
3927 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL; in Select()
3928 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), in Select()
3929 N->getOperand(2), N->getOperand(3), in Select()
3931 CurDAG->getRegister(0, MVT::i32) }; in Select()
3932 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops)); in Select()
3936 if (Subtarget->isThumb()) { in Select()
3937 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3938 N->getOperand(3), getAL(CurDAG, dl), in Select()
3939 CurDAG->getRegister(0, MVT::i32)}; in Select()
3941 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops)); in Select()
3944 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3945 N->getOperand(3), getAL(CurDAG, dl), in Select()
3946 CurDAG->getRegister(0, MVT::i32), in Select()
3947 CurDAG->getRegister(0, MVT::i32) }; in Select()
3948 ReplaceNode(N, CurDAG->getMachineNode( in Select()
3949 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl, in Select()
3955 if (Subtarget->isThumb()) { in Select()
3956 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3957 N->getOperand(3), getAL(CurDAG, dl), in Select()
3958 CurDAG->getRegister(0, MVT::i32)}; in Select()
3960 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops)); in Select()
3963 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), in Select()
3964 N->getOperand(3), getAL(CurDAG, dl), in Select()
3965 CurDAG->getRegister(0, MVT::i32), in Select()
3966 CurDAG->getRegister(0, MVT::i32) }; in Select()
3967 ReplaceNode(N, CurDAG->getMachineNode( in Select()
3968 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl, in Select()
3974 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) in Select()
3978 if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI || in Select()
3979 N->getOperand(2).getOpcode() != ARMISD::SUBC || in Select()
3983 if (Subtarget->isThumb()) in Select()
3984 assert(Subtarget->hasThumb2() && in Select()
3987 SDValue SmulLoHi = N->getOperand(1); in Select()
3988 SDValue Subc = N->getOperand(2); in Select()
3992 N->getOperand(1) != SmulLoHi.getValue(1) || in Select()
3993 N->getOperand(2) != Subc.getValue(1)) in Select()
3996 unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS; in Select()
3998 N->getOperand(0), getAL(CurDAG, dl), in Select()
3999 CurDAG->getRegister(0, MVT::i32) }; in Select()
4000 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops)); in Select()
4004 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N)) in Select()
4006 if (Subtarget->isThumb() && Subtarget->hasThumb2()) { in Select()
4009 } else if (Subtarget->isThumb()) { in Select()
4018 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N)) in Select()
4023 SDNode *New = CurDAG->getMachineNode(ARM::t2WhileLoopSetup, dl, MVT::i32, in Select()
4024 N->getOperand(0)); in Select()
4026 CurDAG->RemoveDeadNode(N); in Select()
4030 SDNode *New = CurDAG->getMachineNode(ARM::t2WhileLoopStart, dl, MVT::Other, in Select()
4031 N->getOperand(1), N->getOperand(2), in Select()
4032 N->getOperand(0)); in Select()
4034 CurDAG->RemoveDeadNode(N); in Select()
4038 SDValue Ops[] = { N->getOperand(1), in Select()
4039 N->getOperand(2), in Select()
4040 N->getOperand(0) }; in Select()
4042 SDNode *New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in Select()
4044 CurDAG->RemoveDeadNode(N); in Select()
4048 if (Subtarget->isThumb2()) in Select()
4051 const SDValue &Chain = N->getOperand(0); in Select()
4052 const SDValue &Addr = N->getOperand(1); in Select()
4054 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) { in Select()
4055 // The register-offset variant of LDRD mandates that the register in Select()
4060 RegOffset = CurDAG->getRegister(0, MVT::i32); in Select()
4063 SDNode *New = CurDAG->getMachineNode(ARM::LOADDUAL, dl, in Select()
4065 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, in Select()
4067 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, in Select()
4073 CurDAG->RemoveDeadNode(N); in Select()
4077 if (Subtarget->isThumb2()) in Select()
4080 const SDValue &Chain = N->getOperand(0); in Select()
4081 const SDValue &Addr = N->getOperand(3); in Select()
4083 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) { in Select()
4084 // The register-offset variant of STRD mandates that the register in Select()
4089 RegOffset = CurDAG->getRegister(0, MVT::i32); in Select()
4092 createGPRPairNode(MVT::Untyped, N->getOperand(1), N->getOperand(2)); in Select()
4094 SDNode *New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops); in Select()
4097 CurDAG->RemoveDeadNode(N); in Select()
4101 SDValue Ops[] = { N->getOperand(1), in Select()
4102 N->getOperand(2), in Select()
4103 N->getOperand(0) }; in Select()
4105 CurDAG->getMachineNode(ARM::t2LoopDec, dl, in Select()
4106 CurDAG->getVTList(MVT::i32, MVT::Other), Ops); in Select()
4108 CurDAG->RemoveDeadNode(N); in Select()
4124 unsigned Opc = Subtarget->isThumb() ? in Select()
4125 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; in Select()
4126 SDValue Chain = N->getOperand(0); in Select()
4127 SDValue N1 = N->getOperand(1); in Select()
4128 SDValue N2 = N->getOperand(2); in Select()
4129 SDValue N3 = N->getOperand(3); in Select()
4130 SDValue InGlue = N->getOperand(4); in Select()
4135 unsigned CC = (unsigned)N2->getAsZExtVal(); in Select()
4140 uint64_t ID = Int->getConstantOperandVal(1); in Select()
4142 // Handle low-overhead loops. in Select()
4145 SDValue Size = CurDAG->getTargetConstant(Int.getConstantOperandVal(3), in Select()
4150 CurDAG->getMachineNode(ARM::t2LoopDec, dl, in Select()
4151 CurDAG->getVTList(MVT::i32, MVT::Other), in Select()
4157 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs); in Select()
4160 CurDAG->RemoveDeadNode(N); in Select()
4161 CurDAG->RemoveDeadNode(InGlue.getNode()); in Select()
4162 CurDAG->RemoveDeadNode(Int.getNode()); in Select()
4169 InGlue = N->getOperand(4); in Select()
4184 SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32); in Select()
4186 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, in Select()
4189 if (N->getNumValues() == 2) { in Select()
4195 CurDAG->RemoveDeadNode(N); in Select()
4200 // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0) in Select()
4204 SDValue X = N->getOperand(0); in Select()
4205 auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode()); in Select()
4206 if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) { in Select()
4207 int64_t Addend = -C->getSExtValue(); in Select()
4211 // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3. in Select()
4212 // Outside that range we can just use a CMN which is 32-bit but has a in Select()
4213 // 12-bit immediate range. in Select()
4215 if (Subtarget->isThumb2()) { in Select()
4216 SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32), in Select()
4217 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32), in Select()
4218 CurDAG->getRegister(0, MVT::i32) }; in Select()
4219 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops); in Select()
4222 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X, in Select()
4223 CurDAG->getTargetConstant(Addend, dl, MVT::i32), in Select()
4224 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)}; in Select()
4225 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops); in Select()
4229 SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)}; in Select()
4230 CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2); in Select()
4238 SDValue InGlue = N->getOperand(4); in Select()
4245 SDValue ARMcc = N->getOperand(2); in Select()
4246 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal(); in Select()
4257 SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32); in Select()
4258 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc, in Select()
4259 N->getOperand(3), N->getOperand(4)}; in Select()
4260 CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops); in Select()
4268 EVT VT = N->getValueType(0); in Select()
4269 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. in Select()
4274 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); in Select()
4275 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg}; in Select()
4276 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops)); in Select()
4280 EVT VT = N->getValueType(0); in Select()
4281 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. in Select()
4286 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); in Select()
4287 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg}; in Select()
4288 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops)); in Select()
4292 EVT VT = N->getValueType(0); in Select()
4297 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); in Select()
4298 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg}; in Select()
4299 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops)); in Select()
4303 EVT VecVT = N->getValueType(0); in Select()
4309 N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1))); in Select()
4315 N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1))); in Select()
4320 createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), in Select()
4321 N->getOperand(2), N->getOperand(3))); in Select()
4427 if (Subtarget->hasNEON()) { in Select()
4464 if (Subtarget->hasNEON()) { in Select()
4492 if (Subtarget->hasNEON()) { in Select()
4506 if (Subtarget->hasNEON()) { in Select()
4523 if (Subtarget->hasNEON()) { in Select()
4583 if (Subtarget->hasNEON()) { in Select()
4612 if (Subtarget->hasNEON()) { in Select()
4629 if (Subtarget->hasNEON()) { in Select()
4645 if (Subtarget->hasNEON()) { in Select()
4665 if (Subtarget->hasNEON()) { in Select()
4715 unsigned IntNo = N->getConstantOperandVal(1); in Select()
4723 SDValue Chain = N->getOperand(0); in Select()
4726 if (Subtarget->isThumb()) in Select()
4732 Ops.push_back(getI32Imm(N->getConstantOperandVal(2), dl)); /* coproc */ in Select()
4733 Ops.push_back(getI32Imm(N->getConstantOperandVal(3), dl)); /* opc */ in Select()
4734 Ops.push_back(getI32Imm(N->getConstantOperandVal(4), dl)); /* CRm */ in Select()
4741 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in Select()
4749 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops)); in Select()
4755 SDValue Chain = N->getOperand(0); in Select()
4756 SDValue MemAddr = N->getOperand(2); in Select()
4757 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps(); in Select()
4774 CurDAG->getRegister(0, MVT::i32), Chain}; in Select()
4775 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); in Select()
4777 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in Select()
4778 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp}); in Select()
4788 CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32); in Select()
4789 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, in Select()
4801 CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32); in Select()
4802 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, in Select()
4809 CurDAG->RemoveDeadNode(N); in Select()
4815 SDValue Chain = N->getOperand(0); in Select()
4816 SDValue Val0 = N->getOperand(2); in Select()
4817 SDValue Val1 = N->getOperand(3); in Select()
4818 SDValue MemAddr = N->getOperand(4); in Select()
4824 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); in Select()
4835 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in Select()
4842 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); in Select()
4844 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); in Select()
4845 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); in Select()
5181 unsigned IntNo = N->getConstantOperandVal(0); in Select()
5186 // Scalar f32 -> bf16 in Select()
5189 const SDValue &Src = N->getOperand(1); in Select()
5190 llvm::EVT DestTy = N->getValueType(0); in Select()
5192 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in Select()
5194 CurDAG->SelectNodeTo(N, ARM::BF16_VCVTB, DestTy, Ops); in Select()
5198 // Vector v4f32 -> v4bf16 in Select()
5201 const SDValue &Src = N->getOperand(1); in Select()
5203 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); in Select()
5205 CurDAG->SelectNodeTo(N, ARM::BF16_VCVT, MVT::v4bf16, Ops); in Select()
5384 Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32)); in getIntOperandsFromRegisterString()
5388 "Unexpected non-integer value in special register string."); in getIntOperandsFromRegisterString()
5396 // which mode it is to be used, e.g. usr. Returns -1 to signify that the string
5401 return -1; in getBankedRegisterMask()
5402 return TheReg->Encoding; in getBankedRegisterMask()
5407 // value representing which flags were present, -1 if invalid.
5415 .Default(-1); in getMClassFlagsMask()
5420 // Returns -1 to signify that the string was invalid.
5423 const FeatureBitset &FeatureBits = Subtarget->getFeatureBits(); in getMClassRegisterMask()
5424 if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits)) in getMClassRegisterMask()
5425 return -1; in getMClassRegisterMask()
5426 return (int)(TheReg->Encoding & 0xFFF); // SYSm value in getMClassRegisterMask()
5432 // bits 3-0 contains the fields to be accessed in the special register, set by in getARClassRegisterMask()
5440 if (Mask == -1) in getARClassRegisterMask()
5441 return -1; in getARClassRegisterMask()
5446 return -1; in getARClassRegisterMask()
5476 return -1; in getARClassRegisterMask()
5491 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1)); in tryReadRegister()
5492 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0)); in tryReadRegister()
5493 bool IsThumb2 = Subtarget->isThumb2(); in tryReadRegister()
5497 getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops); in tryReadRegister()
5517 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in tryReadRegister()
5518 Ops.push_back(N->getOperand(0)); in tryReadRegister()
5519 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops)); in tryReadRegister()
5523 std::string SpecialReg = RegString->getString().lower(); in tryReadRegister()
5526 if (BankedReg != -1) { in tryReadRegister()
5527 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), in tryReadRegister()
5528 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryReadRegister()
5529 N->getOperand(0) }; in tryReadRegister()
5531 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked, in tryReadRegister()
5552 if (!Subtarget->hasVFP2Base()) in tryReadRegister()
5554 if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8Base()) in tryReadRegister()
5557 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryReadRegister()
5558 N->getOperand(0) }; in tryReadRegister()
5560 CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops)); in tryReadRegister()
5567 if (Subtarget->isMClass()) { in tryReadRegister()
5569 if (SYSmValue == -1) in tryReadRegister()
5572 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32), in tryReadRegister()
5573 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryReadRegister()
5574 N->getOperand(0) }; in tryReadRegister()
5576 N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops)); in tryReadRegister()
5583 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryReadRegister()
5584 N->getOperand(0) }; in tryReadRegister()
5585 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS, in tryReadRegister()
5591 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryReadRegister()
5592 N->getOperand(0) }; in tryReadRegister()
5594 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL, in tryReadRegister()
5606 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1)); in tryWriteRegister()
5607 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0)); in tryWriteRegister()
5608 bool IsThumb2 = Subtarget->isThumb2(); in tryWriteRegister()
5612 getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops); in tryWriteRegister()
5622 Ops.insert(Ops.begin()+2, N->getOperand(2)); in tryWriteRegister()
5627 SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) }; in tryWriteRegister()
5632 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); in tryWriteRegister()
5633 Ops.push_back(N->getOperand(0)); in tryWriteRegister()
5635 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); in tryWriteRegister()
5639 std::string SpecialReg = RegString->getString().lower(); in tryWriteRegister()
5641 if (BankedReg != -1) { in tryWriteRegister()
5642 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2), in tryWriteRegister()
5643 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryWriteRegister()
5644 N->getOperand(0) }; in tryWriteRegister()
5646 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked, in tryWriteRegister()
5663 if (!Subtarget->hasVFP2Base()) in tryWriteRegister()
5665 Ops = { N->getOperand(2), getAL(CurDAG, DL), in tryWriteRegister()
5666 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) }; in tryWriteRegister()
5667 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); in tryWriteRegister()
5678 if (Subtarget->isMClass()) { in tryWriteRegister()
5680 if (SYSmValue == -1) in tryWriteRegister()
5683 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32), in tryWriteRegister()
5684 N->getOperand(2), getAL(CurDAG, DL), in tryWriteRegister()
5685 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) }; in tryWriteRegister()
5686 ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops)); in tryWriteRegister()
5694 if (Mask != -1) { in tryWriteRegister()
5695 Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2), in tryWriteRegister()
5696 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32), in tryWriteRegister()
5697 N->getOperand(0) }; in tryWriteRegister()
5698 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR, in tryWriteRegister()
5710 unsigned NumOps = N->getNumOperands(); in tryInlineAsm()
5716 // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb, in tryInlineAsm()
5717 // the 64-bit data may be referred by H, Q, R modifiers, so we still pack in tryInlineAsm()
5721 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps - 1) : SDValue(); in tryInlineAsm()
5725 for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) { in tryInlineAsm()
5726 SDValue op = N->getOperand(i); in tryInlineAsm()
5732 if (const auto *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) in tryInlineAsm()
5733 Flag = InlineAsm::Flag(C->getZExtValue()); in tryInlineAsm()
5742 SDValue op = N->getOperand(++i); in tryInlineAsm()
5764 SDValue op = N->getOperand(++i); in tryInlineAsm()
5780 SDValue V0 = N->getOperand(i+1); in tryInlineAsm()
5781 SDValue V1 = N->getOperand(i+2); in tryInlineAsm()
5782 Register Reg0 = cast<RegisterSDNode>(V0)->getReg(); in tryInlineAsm()
5783 Register Reg1 = cast<RegisterSDNode>(V1)->getReg(); in tryInlineAsm()
5785 MachineRegisterInfo &MRI = MF->getRegInfo(); in tryInlineAsm()
5792 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); in tryInlineAsm()
5795 SDNode *GU = N->getGluedUser(); in tryInlineAsm()
5796 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped, in tryInlineAsm()
5800 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, in tryInlineAsm()
5802 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, in tryInlineAsm()
5804 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0, in tryInlineAsm()
5806 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1)); in tryInlineAsm()
5809 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1); in tryInlineAsm()
5811 CurDAG->UpdateNodeOperands(GU, Ops); in tryInlineAsm()
5818 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32, in tryInlineAsm()
5820 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32, in tryInlineAsm()
5824 // Copy REG_SEQ into a GPRPair-typed VR and replace the original two in tryInlineAsm()
5827 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); in tryInlineAsm()
5828 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1)); in tryInlineAsm()
5837 OpChanged[OpChanged.size() -1 ] = true; in tryInlineAsm()
5844 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant( in tryInlineAsm()
5858 SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N), in tryInlineAsm()
5859 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); in tryInlineAsm()
5860 New->setNodeId(-1); in tryInlineAsm()
5875 case InlineAsm::ConstraintCode::Un: in SelectInlineAsmMemoryOperand()
5890 /// createARMISelDag - This pass converts a legalized DAG into a
5891 /// ARM-specific DAG, ready for instruction scheduling.