Lines Matching +full:high +full:- +full:vt
1 //===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
23 #include "llvm/Config/llvm-config.h"
39 #define DEBUG_TYPE "x86-isel"
40 #define PASS_NAME "X86 DAG->DAG Instruction Selection"
44 static cl::opt<bool> AndImmShrink("x86-and-imm-shrink", cl::init(true),
49 "x86-promote-anyext-load", cl::init(true),
54 //===----------------------------------------------------------------------===//
56 //===----------------------------------------------------------------------===//
80 int JT = -1;
89 MCSym != nullptr || JT != -1 || BlockAddr != nullptr; in hasSymbolicDisplacement()
97 /// Return true if this addressing mode is already RIP-relative.
102 return RegNode->getReg() == X86::RIP; in isRIPRelative()
116 Base_Reg.getNode()->dump(DAG); in dump()
126 IndexReg.getNode()->dump(DAG); in dump()
132 GV->dump(); in dump()
137 CP->dump(); in dump()
158 //===--------------------------------------------------------------------===//
159 /// ISel - X86-specific code to select X86 machine instructions for
184 "indirect-tls-seg-refs"); in runOnMachineFunction()
266 MVT VT, SDValue &Base, SDValue &Scale, in getAddressOperands() argument
270 Base = CurDAG->getTargetFrameIndex( in getAddressOperands()
271 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout())); in getAddressOperands()
275 Base = CurDAG->getRegister(0, VT); in getAddressOperands()
279 #define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC) in getAddressOperands()
282 unsigned NegOpc = VT == MVT::i64 ? GET_ND_IF_ENABLED(X86::NEG64r) in getAddressOperands()
284 SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32, in getAddressOperands()
292 Index = CurDAG->getRegister(0, VT); in getAddressOperands()
294 // These are 32-bit even in 64-bit mode since RIP-relative offset in getAddressOperands()
295 // is 32-bit. in getAddressOperands()
297 Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), in getAddressOperands()
301 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment, in getAddressOperands()
304 assert(!AM.Disp && "Non-zero displacement is ignored with ES."); in getAddressOperands()
305 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); in getAddressOperands()
307 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); in getAddressOperands()
309 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); in getAddressOperands()
310 } else if (AM.JT != -1) { in getAddressOperands()
311 assert(!AM.Disp && "Non-zero displacement is ignored with JT."); in getAddressOperands()
312 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); in getAddressOperands()
314 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, in getAddressOperands()
317 Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); in getAddressOperands()
322 Segment = CurDAG->getRegister(0, MVT::i16); in getAddressOperands()
327 // At a high level, we'd like to avoid such instructions when
337 if (!CurDAG->shouldOptForSize()) in shouldAvoidImmediateInstFormsForSize()
341 for (const SDNode *User : N->uses()) { in shouldAvoidImmediateInstFormsForSize()
347 if (User->isMachineOpcode()) { in shouldAvoidImmediateInstFormsForSize()
353 if (User->getOpcode() == ISD::STORE && in shouldAvoidImmediateInstFormsForSize()
354 User->getOperand(1).getNode() == N) { in shouldAvoidImmediateInstFormsForSize()
365 if (User->getNumOperands() != 2) in shouldAvoidImmediateInstFormsForSize()
368 // If this is a sign-extended 8-bit integer immediate used in an ALU in shouldAvoidImmediateInstFormsForSize()
371 if (C && isInt<8>(C->getSExtValue())) in shouldAvoidImmediateInstFormsForSize()
378 if (User->getOpcode() == X86ISD::ADD || in shouldAvoidImmediateInstFormsForSize()
379 User->getOpcode() == ISD::ADD || in shouldAvoidImmediateInstFormsForSize()
380 User->getOpcode() == X86ISD::SUB || in shouldAvoidImmediateInstFormsForSize()
381 User->getOpcode() == ISD::SUB) { in shouldAvoidImmediateInstFormsForSize()
384 SDValue OtherOp = User->getOperand(0); in shouldAvoidImmediateInstFormsForSize()
386 OtherOp = User->getOperand(1); in shouldAvoidImmediateInstFormsForSize()
390 if (OtherOp->getOpcode() == ISD::CopyFromReg && in shouldAvoidImmediateInstFormsForSize()
392 OtherOp->getOperand(1).getNode()))) in shouldAvoidImmediateInstFormsForSize()
393 if ((RegNode->getReg() == X86::ESP) || in shouldAvoidImmediateInstFormsForSize()
394 (RegNode->getReg() == X86::RSP)) in shouldAvoidImmediateInstFormsForSize()
408 return CurDAG->getTargetConstant(Imm, DL, MVT::i8); in getI8Imm()
413 return CurDAG->getTargetConstant(Imm, DL, MVT::i32); in getI32Imm()
418 return CurDAG->getTargetConstant(Imm, DL, MVT::i64); in getI64Imm()
424 uint64_t Index = N->getConstantOperandVal(1); in getExtractVEXTRACTImmediate()
425 MVT VecVT = N->getOperand(0).getSimpleValueType(); in getExtractVEXTRACTImmediate()
432 uint64_t Index = N->getConstantOperandVal(2); in getInsertVINSERTImmediate()
433 MVT VecVT = N->getSimpleValueType(0); in getInsertVINSERTImmediate()
440 uint64_t Index = N->getConstantOperandVal(2); in getPermuteVINSERTCommutedImmediate()
441 MVT VecVT = N->getSimpleValueType(0); in getPermuteVINSERTCommutedImmediate()
444 // vinsert(0,sub,vec) -> [sub0][vec1] -> vperm2x128(0x30,vec,sub) in getPermuteVINSERTCommutedImmediate()
445 // vinsert(1,sub,vec) -> [vec0][sub0] -> vperm2x128(0x02,vec,sub) in getPermuteVINSERTCommutedImmediate()
451 MVT VT = N->getSimpleValueType(0); in getSBBZero() local
454 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32); in getSBBZero()
456 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0); in getSBBZero()
457 if (VT == MVT::i64) { in getSBBZero()
459 CurDAG->getMachineNode( in getSBBZero()
461 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero, in getSBBZero()
462 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)), in getSBBZero()
467 unsigned Opcode = N->getOpcode(); in getSBBZero()
472 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS, in getSBBZero()
473 N->getOperand(FlagOpIndex), SDValue()); in getSBBZero()
475 // Create a 64-bit instruction if the result is 64-bits otherwise use the in getSBBZero()
476 // 32-bit version. in getSBBZero()
477 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr; in getSBBZero()
478 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32; in getSBBZero()
479 VTs = CurDAG->getVTList(SBBVT, MVT::i32); in getSBBZero()
481 CurDAG->getMachineNode(Opc, dl, VTs, in getSBBZero()
489 assert(N->getOpcode() == ISD::AND && "Unexpected opcode"); in isUnneededShiftMask()
490 const APInt &Val = N->getConstantOperandAPInt(1); in isUnneededShiftMask()
495 APInt Mask = Val | CurDAG->computeKnownBits(N->getOperand(0)).Zero; in isUnneededShiftMask()
504 /// Return a reference to the TargetMachine, casted to the target-specific
510 /// Return a reference to the TargetInstrInfo, casted to the target-specific
513 return Subtarget->getInstrInfo(); in getInstrInfo()
519 /// Address-mode matching performs shift-of-and to and-of-shift
528 // Indicates we should prefer to use a non-temporal load for this load.
530 if (!N->isNonTemporal()) in useNonTemporalLoad()
533 unsigned StoreSize = N->getMemoryVT().getStoreSize(); in useNonTemporalLoad()
535 if (N->getAlign().value() < StoreSize) in useNonTemporalLoad()
544 return Subtarget->hasSSE41(); in useNonTemporalLoad()
546 return Subtarget->hasAVX2(); in useNonTemporalLoad()
548 return Subtarget->hasAVX512(); in useNonTemporalLoad()
567 const SDLoc &dl, MVT VT, SDNode *Node);
569 const SDLoc &dl, MVT VT, SDNode *Node,
596 unsigned Opcode = N->getOpcode(); in INITIALIZE_PASS()
600 // We can get 256-bit 8 element types here without VLX being enabled. When in INITIALIZE_PASS()
601 // this happens we will use 512-bit operations and the mask will not be in INITIALIZE_PASS()
603 EVT OpVT = N->getOperand(0).getValueType(); in INITIALIZE_PASS()
607 OpVT = N->getOperand(1).getValueType(); in INITIALIZE_PASS()
609 return Subtarget->hasVLX(); in INITIALIZE_PASS()
627 if (N->getOpcode() == ISD::AND) in isMaskZeroExtended()
628 return isLegalMaskCompare(N->getOperand(0).getNode(), Subtarget) || in isMaskZeroExtended()
629 isLegalMaskCompare(N->getOperand(1).getNode(), Subtarget); in isMaskZeroExtended()
645 // Don't fold non-temporal loads if we have an instruction for them. in IsProfitableToFold()
651 switch (U->getOpcode()) { in IsProfitableToFold()
665 SDValue Op1 = U->getOperand(1); in IsProfitableToFold()
667 // If the other operand is a 8-bit immediate we should fold the immediate in IsProfitableToFold()
678 if (Imm->getAPIntValue().isSignedIntN(8)) in IsProfitableToFold()
681 // If this is a 64-bit AND with an immediate that fits in 32-bits, in IsProfitableToFold()
686 if (U->getOpcode() == ISD::AND && in IsProfitableToFold()
687 Imm->getAPIntValue().getBitWidth() == 64 && in IsProfitableToFold()
688 Imm->getAPIntValue().isIntN(32)) in IsProfitableToFold()
693 // TODO: We could shrink the load and fold if it is non-volatile. in IsProfitableToFold()
694 if (U->getOpcode() == ISD::AND && in IsProfitableToFold()
695 (Imm->getAPIntValue() == UINT8_MAX || in IsProfitableToFold()
696 Imm->getAPIntValue() == UINT16_MAX || in IsProfitableToFold()
697 Imm->getAPIntValue() == UINT32_MAX)) in IsProfitableToFold()
702 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB) && in IsProfitableToFold()
703 (-Imm->getAPIntValue()).isSignedIntN(8)) in IsProfitableToFold()
706 if ((U->getOpcode() == X86ISD::ADD || U->getOpcode() == X86ISD::SUB) && in IsProfitableToFold()
707 (-Imm->getAPIntValue()).isSignedIntN(8) && in IsProfitableToFold()
721 // FIXME: This is probably also true for non-TLS addresses. in IsProfitableToFold()
730 // BTR: (and X, (rotl -2, n)) in IsProfitableToFold()
732 if (U->getOpcode() == ISD::OR || U->getOpcode() == ISD::XOR) { in IsProfitableToFold()
733 if (U->getOperand(0).getOpcode() == ISD::SHL && in IsProfitableToFold()
734 isOneConstant(U->getOperand(0).getOperand(0))) in IsProfitableToFold()
737 if (U->getOperand(1).getOpcode() == ISD::SHL && in IsProfitableToFold()
738 isOneConstant(U->getOperand(1).getOperand(0))) in IsProfitableToFold()
741 if (U->getOpcode() == ISD::AND) { in IsProfitableToFold()
742 SDValue U0 = U->getOperand(0); in IsProfitableToFold()
743 SDValue U1 = U->getOperand(1); in IsProfitableToFold()
746 if (C && C->getSExtValue() == -2) in IsProfitableToFold()
752 if (C && C->getSExtValue() == -2) in IsProfitableToFold()
766 if (isa<ConstantSDNode>(U->getOperand(1))) in IsProfitableToFold()
775 if (Root->getOpcode() == ISD::INSERT_SUBVECTOR && in IsProfitableToFold()
776 isNullConstant(Root->getOperand(2)) && in IsProfitableToFold()
777 (Root->getOperand(0).isUndef() || in IsProfitableToFold()
778 ISD::isBuildVectorAllZeros(Root->getOperand(0).getNode()))) in IsProfitableToFold()
785 // false will favor a masked register-register masked move or vblendm and the
789 (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::SELECTS) && in isProfitableToFormMaskedOp()
795 return N->getOperand(1).hasOneUse(); in isProfitableToFormMaskedOp()
815 CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops); in moveBelowOrigChain()
819 Ops.append(OrigChain->op_begin() + 1, OrigChain->op_end()); in moveBelowOrigChain()
820 CurDAG->UpdateNodeOperands(OrigChain.getNode(), Ops); in moveBelowOrigChain()
821 CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), in moveBelowOrigChain()
826 Ops.append(Call->op_begin() + 1, Call->op_end()); in moveBelowOrigChain()
827 CurDAG->UpdateNodeOperands(Call.getNode(), Ops); in moveBelowOrigChain()
844 !LD->isSimple() || in isCalleeLoad()
845 LD->getAddressingMode() != ISD::UNINDEXED || in isCalleeLoad()
846 LD->getExtensionType() != ISD::NON_EXTLOAD) in isCalleeLoad()
861 cast<MemSDNode>(Chain.getNode())->writeMem()) in isCalleeLoad()
893 static bool needBWI(MVT VT) { in needBWI() argument
894 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8); in needBWI()
899 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), in PreprocessISelDAG()
900 E = CurDAG->allnodes_end(); I != E; ) { in PreprocessISelDAG()
918 if (N->getOpcode() == ISD::Constant) { in PreprocessISelDAG()
919 MVT VT = N->getSimpleValueType(0); in PreprocessISelDAG() local
920 int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue(); in PreprocessISelDAG()
921 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB; in PreprocessISelDAG()
923 // Check that the cf-protection-branch is enabled. in PreprocessISelDAG()
925 MF->getFunction().getParent()->getModuleFlag( in PreprocessISelDAG()
926 "cf-protection-branch"); in PreprocessISelDAG()
929 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT, false, true); in PreprocessISelDAG()
930 Complement = CurDAG->getNOT(dl, Complement, VT); in PreprocessISelDAG()
931 --I; in PreprocessISelDAG()
932 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Complement); in PreprocessISelDAG()
942 if (N->getOpcode() == X86ISD::AND && !N->hasAnyUseOfValue(1)) { in PreprocessISelDAG()
943 SDValue Res = CurDAG->getNode(ISD::AND, SDLoc(N), N->getValueType(0), in PreprocessISelDAG()
944 N->getOperand(0), N->getOperand(1)); in PreprocessISelDAG()
945 --I; in PreprocessISelDAG()
946 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); in PreprocessISelDAG()
952 // Convert vector increment or decrement to sub/add with an all-ones in PreprocessISelDAG()
954 // add X, <1, 1...> --> sub X, <-1, -1...> in PreprocessISelDAG()
955 // sub X, <1, 1...> --> add X, <-1, -1...> in PreprocessISelDAG()
956 // The all-ones vector constant can be materialized using a pcmpeq in PreprocessISelDAG()
971 return X86::mayFoldLoad(N->getOperand(0), *Subtarget) && in PreprocessISelDAG()
972 N->getOpcode() == ISD::ADD && Subtarget->hasAVX() && in PreprocessISelDAG()
973 !N->getOperand(1).hasOneUse(); in PreprocessISelDAG()
975 if ((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in PreprocessISelDAG()
976 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) { in PreprocessISelDAG()
978 if (X86::isConstantSplat(N->getOperand(1), SplatVal) && in PreprocessISelDAG()
982 MVT VT = N->getSimpleValueType(0); in PreprocessISelDAG() local
983 unsigned NumElts = VT.getSizeInBits() / 32; in PreprocessISelDAG()
985 CurDAG->getAllOnesConstant(DL, MVT::getVectorVT(MVT::i32, NumElts)); in PreprocessISelDAG()
986 AllOnes = CurDAG->getBitcast(VT, AllOnes); in PreprocessISelDAG()
988 unsigned NewOpcode = N->getOpcode() == ISD::ADD ? ISD::SUB : ISD::ADD; in PreprocessISelDAG()
990 CurDAG->getNode(NewOpcode, DL, VT, N->getOperand(0), AllOnes); in PreprocessISelDAG()
991 --I; in PreprocessISelDAG()
992 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); in PreprocessISelDAG()
999 switch (N->getOpcode()) { in PreprocessISelDAG()
1001 MVT VT = N->getSimpleValueType(0); in PreprocessISelDAG() local
1003 if (!Subtarget->hasBWI() && needBWI(VT)) { in PreprocessISelDAG()
1004 MVT NarrowVT = VT.getHalfNumVectorElementsVT(); in PreprocessISelDAG()
1007 CurDAG->getNode(X86ISD::VBROADCAST, dl, NarrowVT, N->getOperand(0)); in PreprocessISelDAG()
1009 CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, CurDAG->getUNDEF(VT), in PreprocessISelDAG()
1010 NarrowBCast, CurDAG->getIntPtrConstant(0, dl)); in PreprocessISelDAG()
1012 Res = CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, NarrowBCast, in PreprocessISelDAG()
1013 CurDAG->getIntPtrConstant(Index, dl)); in PreprocessISelDAG()
1015 --I; in PreprocessISelDAG()
1016 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); in PreprocessISelDAG()
1025 MVT VT = N->getSimpleValueType(0); in PreprocessISelDAG() local
1027 if (!Subtarget->hasBWI() && needBWI(VT)) { in PreprocessISelDAG()
1028 MVT NarrowVT = VT.getHalfNumVectorElementsVT(); in PreprocessISelDAG()
1031 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other); in PreprocessISelDAG()
1032 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()}; in PreprocessISelDAG()
1033 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode( in PreprocessISelDAG()
1034 X86ISD::VBROADCAST_LOAD, dl, VTs, Ops, MemNode->getMemoryVT(), in PreprocessISelDAG()
1035 MemNode->getMemOperand()); in PreprocessISelDAG()
1037 CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, CurDAG->getUNDEF(VT), in PreprocessISelDAG()
1038 NarrowBCast, CurDAG->getIntPtrConstant(0, dl)); in PreprocessISelDAG()
1040 Res = CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, NarrowBCast, in PreprocessISelDAG()
1041 CurDAG->getIntPtrConstant(Index, dl)); in PreprocessISelDAG()
1043 --I; in PreprocessISelDAG()
1045 CurDAG->ReplaceAllUsesWith(N, To); in PreprocessISelDAG()
1057 MVT VT = N->getSimpleValueType(0); in PreprocessISelDAG() local
1058 if (!ISD::isNormalLoad(Ld) || !Ld->isSimple() || in PreprocessISelDAG()
1059 !(VT.is128BitVector() || VT.is256BitVector())) in PreprocessISelDAG()
1062 MVT MaxVT = VT; in PreprocessISelDAG()
1064 SDValue Ptr = Ld->getBasePtr(); in PreprocessISelDAG()
1065 SDValue Chain = Ld->getChain(); in PreprocessISelDAG()
1066 for (SDNode *User : Ptr->uses()) { in PreprocessISelDAG()
1068 MVT UserVT = User->getSimpleValueType(0); in PreprocessISelDAG()
1070 UserLd->getBasePtr() == Ptr && UserLd->getChain() == Chain && in PreprocessISelDAG()
1071 !User->hasAnyUseOfValue(1) && in PreprocessISelDAG()
1073 UserVT.getSizeInBits() > VT.getSizeInBits() && in PreprocessISelDAG()
1081 unsigned NumSubElts = VT.getSizeInBits() / MaxVT.getScalarSizeInBits(); in PreprocessISelDAG()
1083 SDValue Extract = CurDAG->getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, in PreprocessISelDAG()
1085 CurDAG->getIntPtrConstant(0, dl)); in PreprocessISelDAG()
1086 SDValue Res = CurDAG->getBitcast(VT, Extract); in PreprocessISelDAG()
1088 --I; in PreprocessISelDAG()
1090 CurDAG->ReplaceAllUsesWith(N, To); in PreprocessISelDAG()
1098 // Replace VSELECT with non-mask conditions with with BLENDV/VPTERNLOG. in PreprocessISelDAG()
1099 EVT EleVT = N->getOperand(0).getValueType().getVectorElementType(); in PreprocessISelDAG()
1103 assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!"); in PreprocessISelDAG()
1104 assert(N->getValueType(0).getVectorElementType() != MVT::i16 && in PreprocessISelDAG()
1107 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(N->getOperand(0)) == in PreprocessISelDAG()
1109 R = CurDAG->getNode(X86ISD::VPTERNLOG, SDLoc(N), N->getValueType(0), in PreprocessISelDAG()
1110 N->getOperand(0), N->getOperand(1), N->getOperand(2), in PreprocessISelDAG()
1111 CurDAG->getTargetConstant(0xCA, SDLoc(N), MVT::i8)); in PreprocessISelDAG()
1113 R = CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0), in PreprocessISelDAG()
1114 N->getOperand(0), N->getOperand(1), in PreprocessISelDAG()
1115 N->getOperand(2)); in PreprocessISelDAG()
1117 --I; in PreprocessISelDAG()
1118 CurDAG->ReplaceAllUsesWith(N, R.getNode()); in PreprocessISelDAG()
1131 if (!N->getSimpleValueType(0).isVector()) in PreprocessISelDAG()
1135 switch (N->getOpcode()) { in PreprocessISelDAG()
1145 if (N->isStrictFPOpcode()) in PreprocessISelDAG()
1147 CurDAG->getNode(NewOpc, SDLoc(N), {N->getValueType(0), MVT::Other}, in PreprocessISelDAG()
1148 {N->getOperand(0), N->getOperand(1)}); in PreprocessISelDAG()
1151 CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), in PreprocessISelDAG()
1152 N->getOperand(0)); in PreprocessISelDAG()
1153 --I; in PreprocessISelDAG()
1154 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); in PreprocessISelDAG()
1164 if (!N->getValueType(0).isVector()) in PreprocessISelDAG()
1168 switch (N->getOpcode()) { in PreprocessISelDAG()
1174 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), in PreprocessISelDAG()
1175 N->getOperand(0), N->getOperand(1)); in PreprocessISelDAG()
1176 --I; in PreprocessISelDAG()
1177 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); in PreprocessISelDAG()
1186 if (!N->getValueType(0).isVector()) in PreprocessISelDAG()
1190 if (N->getOperand(0).getScalarValueSizeInBits() == 1) { in PreprocessISelDAG()
1191 assert(N->getOpcode() == ISD::ANY_EXTEND && in PreprocessISelDAG()
1195 NewOpc = N->getOpcode() == ISD::ANY_EXTEND in PreprocessISelDAG()
1200 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), in PreprocessISelDAG()
1201 N->getOperand(0)); in PreprocessISelDAG()
1202 --I; in PreprocessISelDAG()
1203 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); in PreprocessISelDAG()
1223 switch (N->getOpcode()) { in PreprocessISelDAG()
1239 bool IsStrict = N->isStrictFPOpcode(); in PreprocessISelDAG()
1242 Res = CurDAG->getNode(X86ISD::STRICT_VRNDSCALE, dl, in PreprocessISelDAG()
1243 {N->getValueType(0), MVT::Other}, in PreprocessISelDAG()
1244 {N->getOperand(0), N->getOperand(1), in PreprocessISelDAG()
1245 CurDAG->getTargetConstant(Imm, dl, MVT::i32)}); in PreprocessISelDAG()
1247 Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl, N->getValueType(0), in PreprocessISelDAG()
1248 N->getOperand(0), in PreprocessISelDAG()
1249 CurDAG->getTargetConstant(Imm, dl, MVT::i32)); in PreprocessISelDAG()
1250 --I; in PreprocessISelDAG()
1251 CurDAG->ReplaceAllUsesWith(N, Res.getNode()); in PreprocessISelDAG()
1262 MVT VT = N->getSimpleValueType(0); in PreprocessISelDAG() local
1263 if (VT.isVector() || VT == MVT::f128) in PreprocessISelDAG()
1266 MVT VecVT = VT == MVT::f64 ? MVT::v2f64 in PreprocessISelDAG()
1267 : VT == MVT::f32 ? MVT::v4f32 in PreprocessISelDAG()
1271 SDValue Op0 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, in PreprocessISelDAG()
1272 N->getOperand(0)); in PreprocessISelDAG()
1273 SDValue Op1 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, in PreprocessISelDAG()
1274 N->getOperand(1)); in PreprocessISelDAG()
1277 if (Subtarget->hasSSE2()) { in PreprocessISelDAG()
1279 Op0 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op0); in PreprocessISelDAG()
1280 Op1 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op1); in PreprocessISelDAG()
1282 switch (N->getOpcode()) { in PreprocessISelDAG()
1289 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1); in PreprocessISelDAG()
1290 Res = CurDAG->getNode(ISD::BITCAST, dl, VecVT, Res); in PreprocessISelDAG()
1292 Res = CurDAG->getNode(N->getOpcode(), dl, VecVT, Op0, Op1); in PreprocessISelDAG()
1294 Res = CurDAG->getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, in PreprocessISelDAG()
1295 CurDAG->getIntPtrConstant(0, dl)); in PreprocessISelDAG()
1296 --I; in PreprocessISelDAG()
1297 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); in PreprocessISelDAG()
1307 !Subtarget->useIndirectThunkCalls() && in PreprocessISelDAG()
1308 ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) || in PreprocessISelDAG()
1309 (N->getOpcode() == X86ISD::TC_RETURN && in PreprocessISelDAG()
1310 (Subtarget->is64Bit() || in PreprocessISelDAG()
1321 /// / \-- in PreprocessISelDAG()
1331 bool HasCallSeq = N->getOpcode() == X86ISD::CALL; in PreprocessISelDAG()
1332 SDValue Chain = N->getOperand(0); in PreprocessISelDAG()
1333 SDValue Load = N->getOperand(1); in PreprocessISelDAG()
1349 // FIXME: This should only happen when not compiled with -O0. in PreprocessISelDAG()
1350 switch (N->getOpcode()) { in PreprocessISelDAG()
1355 MVT SrcVT = N->getOperand(0).getSimpleValueType(); in PreprocessISelDAG()
1356 MVT DstVT = N->getSimpleValueType(0); in PreprocessISelDAG()
1366 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); in PreprocessISelDAG()
1367 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); in PreprocessISelDAG()
1373 if (N->getOpcode() == ISD::FP_EXTEND) in PreprocessISelDAG()
1375 // If this is a value-preserving FPStack truncation, it is a noop. in PreprocessISelDAG()
1376 if (N->getConstantOperandVal(1)) in PreprocessISelDAG()
1380 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. in PreprocessISelDAG()
1383 MVT MemVT = (N->getOpcode() == ISD::FP_ROUND) ? DstVT : SrcVT; in PreprocessISelDAG()
1384 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); in PreprocessISelDAG()
1385 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex(); in PreprocessISelDAG()
1387 MachinePointerInfo::getFixedStack(CurDAG->getMachineFunction(), SPFI); in PreprocessISelDAG()
1392 SDValue Store = CurDAG->getTruncStore( in PreprocessISelDAG()
1393 CurDAG->getEntryNode(), dl, N->getOperand(0), MemTmp, MPI, MemVT); in PreprocessISelDAG()
1394 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, in PreprocessISelDAG()
1401 --I; in PreprocessISelDAG()
1402 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); in PreprocessISelDAG()
1411 MVT SrcVT = N->getOperand(1).getSimpleValueType(); in PreprocessISelDAG()
1412 MVT DstVT = N->getSimpleValueType(0); in PreprocessISelDAG()
1422 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); in PreprocessISelDAG()
1423 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); in PreprocessISelDAG()
1429 if (N->getOpcode() == ISD::STRICT_FP_EXTEND) in PreprocessISelDAG()
1431 // If this is a value-preserving FPStack truncation, it is a noop. in PreprocessISelDAG()
1432 if (N->getConstantOperandVal(2)) in PreprocessISelDAG()
1436 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. in PreprocessISelDAG()
1439 MVT MemVT = (N->getOpcode() == ISD::STRICT_FP_ROUND) ? DstVT : SrcVT; in PreprocessISelDAG()
1440 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); in PreprocessISelDAG()
1441 int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex(); in PreprocessISelDAG()
1443 MachinePointerInfo::getFixedStack(CurDAG->getMachineFunction(), SPFI); in PreprocessISelDAG()
1451 SDVTList VTs = CurDAG->getVTList(MVT::Other); in PreprocessISelDAG()
1452 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp}; in PreprocessISelDAG()
1453 Store = CurDAG->getMemIntrinsicNode(X86ISD::FST, dl, VTs, Ops, MemVT, in PreprocessISelDAG()
1456 if (N->getFlags().hasNoFPExcept()) { in PreprocessISelDAG()
1457 SDNodeFlags Flags = Store->getFlags(); in PreprocessISelDAG()
1459 Store->setFlags(Flags); in PreprocessISelDAG()
1462 assert(SrcVT == MemVT && "Unexpected VT!"); in PreprocessISelDAG()
1463 Store = CurDAG->getStore(N->getOperand(0), dl, N->getOperand(1), MemTmp, in PreprocessISelDAG()
1468 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other); in PreprocessISelDAG()
1470 Result = CurDAG->getMemIntrinsicNode( in PreprocessISelDAG()
1473 if (N->getFlags().hasNoFPExcept()) { in PreprocessISelDAG()
1474 SDNodeFlags Flags = Result->getFlags(); in PreprocessISelDAG()
1476 Result->setFlags(Flags); in PreprocessISelDAG()
1479 assert(DstVT == MemVT && "Unexpected VT!"); in PreprocessISelDAG()
1480 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI); in PreprocessISelDAG()
1487 --I; in PreprocessISelDAG()
1488 CurDAG->ReplaceAllUsesWith(N, Result.getNode()); in PreprocessISelDAG()
1502 CurDAG->RemoveDeadNodes(); in PreprocessISelDAG()
1505 // Look for a redundant movzx/movsx that can occur after an 8-bit divrem.
1507 unsigned Opc = N->getMachineOpcode(); in tryOptimizeRem8Extend()
1512 SDValue N0 = N->getOperand(0); in tryOptimizeRem8Extend()
1530 MachineSDNode *Extend = CurDAG->getMachineNode(X86::MOVSX64rr32, SDLoc(N), in tryOptimizeRem8Extend()
1542 // Skip peepholes at -O0. in PostprocessISelDAG()
1546 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); in PostprocessISelDAG()
1549 while (Position != CurDAG->allnodes_begin()) { in PostprocessISelDAG()
1550 SDNode *N = &*--Position; in PostprocessISelDAG()
1551 // Skip dead nodes and any non-machine opcodes. in PostprocessISelDAG()
1552 if (N->use_empty() || !N->isMachineOpcode()) in PostprocessISelDAG()
1560 unsigned Opc = N->getMachineOpcode(); in PostprocessISelDAG()
1564 // ANDrr/rm + TESTrr+ -> TESTrr/TESTmr in PostprocessISelDAG()
1569 // ANDrr/rm + CTESTrr -> CTESTrr/CTESTmr in PostprocessISelDAG()
1574 auto &Op0 = N->getOperand(0); in PostprocessISelDAG()
1575 if (Op0 != N->getOperand(1) || !Op0->hasNUsesOfValue(2, Op0.getResNo()) || in PostprocessISelDAG()
1578 SDValue And = N->getOperand(0); in PostprocessISelDAG()
1589 if (And->hasAnyUseOfValue(1)) in PostprocessISelDAG()
1591 SmallVector<SDValue> Ops(N->op_values()); in PostprocessISelDAG()
1595 CurDAG->getMachineNode(Opc, SDLoc(N), MVT::i32, Ops); in PostprocessISelDAG()
1604 if (And->hasAnyUseOfValue(1)) in PostprocessISelDAG()
1625 Ops.push_back(N->getOperand(2)); in PostprocessISelDAG()
1626 Ops.push_back(N->getOperand(3)); in PostprocessISelDAG()
1632 Ops.push_back(N->getOperand(4)); in PostprocessISelDAG()
1634 MachineSDNode *Test = CurDAG->getMachineNode( in PostprocessISelDAG()
1636 CurDAG->setNodeMemRefs( in PostprocessISelDAG()
1637 Test, cast<MachineSDNode>(And.getNode())->memoperands()); in PostprocessISelDAG()
1653 SDValue Op0 = N->getOperand(0); in PostprocessISelDAG()
1654 if (Op0 != N->getOperand(1) || !N->isOnlyUserOf(Op0.getNode()) || in PostprocessISelDAG()
1681 if (NewOpc == X86::KTESTWrr && !Subtarget->hasDQI()) in PostprocessISelDAG()
1684 MachineSDNode *KTest = CurDAG->getMachineNode( in PostprocessISelDAG()
1692 unsigned SubRegIdx = N->getConstantOperandVal(2); in PostprocessISelDAG()
1696 SDValue Move = N->getOperand(1); in PostprocessISelDAG()
1728 uint64_t TSFlags = getInstrInfo()->get(In.getMachineOpcode()).TSFlags; in PostprocessISelDAG()
1736 CurDAG->UpdateNodeOperands(N, N->getOperand(0), In, N->getOperand(2)); in PostprocessISelDAG()
1743 CurDAG->RemoveDeadNodes(); in PostprocessISelDAG()
1749 if (Subtarget->isTargetCygMing()) { in emitSpecialCodeForMain()
1751 auto &DL = CurDAG->getDataLayout(); in emitSpecialCodeForMain()
1754 CLI.setChain(CurDAG->getRoot()) in emitSpecialCodeForMain()
1755 .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()), in emitSpecialCodeForMain()
1756 CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)), in emitSpecialCodeForMain()
1758 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); in emitSpecialCodeForMain()
1760 CurDAG->setRoot(Result.second); in emitSpecialCodeForMain()
1766 const Function &F = MF->getFunction(); in emitFunctionEntryCode()
1772 // On 64-bit platforms, we can run into an issue where a frame index in isDispSafeForFrameIndex()
1775 // displacement fits into a 31-bit integer (which is only slightly more in isDispSafeForFrameIndex()
1777 // a 32-bit integer), a 31-bit disp should always be safe. in isDispSafeForFrameIndex()
1794 if (Subtarget->is64Bit()) { in foldOffsetIntoAddress()
1804 // In ILP32 (x32) mode, pointers are 32 bits and need to be zero-extended to in foldOffsetIntoAddress()
1805 // 64 bits. Instructions with 32-bit register addresses perform this zero in foldOffsetIntoAddress()
1806 // extension for us and we can safely ignore the high bits of Offset. in foldOffsetIntoAddress()
1807 // Instructions with only a 32-bit immediate address do not, though: they in foldOffsetIntoAddress()
1809 // is directly addressable, we need indirect addressing for the high 2GB of in foldOffsetIntoAddress()
1816 // addresses in LP64 mode, by adding the EIZ pseudo-register as an operand in foldOffsetIntoAddress()
1818 // pseudo-register is not part of any register class and therefore causes in foldOffsetIntoAddress()
1820 if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) && in foldOffsetIntoAddress()
1830 SDValue Address = N->getOperand(1); in matchLoadInAddress()
1832 // load gs:0 -> GS segment register. in matchLoadInAddress()
1833 // load fs:0 -> FS segment register. in matchLoadInAddress()
1836 // gs:0 (or fs:0 on X86-64) contains its own address. However, for X86-64 mode in matchLoadInAddress()
1837 // with 32-bit registers, as we get in ILP32 mode, those registers are first in matchLoadInAddress()
1838 // zero-extended to 64 bits and then added it to the base address, which gives in matchLoadInAddress()
1843 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() || in matchLoadInAddress()
1844 Subtarget->isTargetFuchsia())) { in matchLoadInAddress()
1845 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32) in matchLoadInAddress()
1847 switch (N->getPointerInfo().getAddrSpace()) { in matchLoadInAddress()
1849 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in matchLoadInAddress()
1852 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in matchLoadInAddress()
1879 // We can't use an addressing mode in the 64-bit large code model. in matchWrapper()
1885 if (Subtarget->is64Bit() && M == CodeModel::Large && !IsRIPRelTLS) in matchWrapper()
1898 AM.GV = G->getGlobal(); in matchWrapper()
1899 AM.SymbolFlags = G->getTargetFlags(); in matchWrapper()
1900 Offset = G->getOffset(); in matchWrapper()
1902 AM.CP = CP->getConstVal(); in matchWrapper()
1903 AM.Alignment = CP->getAlign(); in matchWrapper()
1904 AM.SymbolFlags = CP->getTargetFlags(); in matchWrapper()
1905 Offset = CP->getOffset(); in matchWrapper()
1907 AM.ES = S->getSymbol(); in matchWrapper()
1908 AM.SymbolFlags = S->getTargetFlags(); in matchWrapper()
1910 AM.MCSym = S->getMCSymbol(); in matchWrapper()
1912 AM.JT = J->getIndex(); in matchWrapper()
1913 AM.SymbolFlags = J->getTargetFlags(); in matchWrapper()
1915 AM.BlockAddr = BA->getBlockAddress(); in matchWrapper()
1916 AM.SymbolFlags = BA->getTargetFlags(); in matchWrapper()
1917 Offset = BA->getOffset(); in matchWrapper()
1922 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV && in matchWrapper()
1934 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); in matchWrapper()
1946 // Post-processing: Make a second attempt to fold a load, if we now know in matchAddress()
1948 // 64-bit ILP32 mode since 32-bit mode and 64-bit LP64 mode will have folded in matchAddress()
1950 if (Subtarget->isTarget64BitILP32() && in matchAddress()
1961 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has in matchAddress()
1962 // a smaller encoding and avoids a scaled-index. in matchAddress()
1970 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, in matchAddress()
1973 (!AM.GV || !TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() && in matchAddress()
1977 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); in matchAddress()
2024 if (N->getNodeId() == -1 || in insertDAGNode()
2027 DAG.RepositionNode(Pos->getIterator(), N.getNode()); in insertDAGNode()
2030 // Conservatively mark it with the same -abs(Id) to assure node id in insertDAGNode()
2032 N->setNodeId(Pos->getNodeId()); in insertDAGNode()
2037 // Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if
2038 // safe. This allows us to convert the shift and and into an h-register
2050 int ScaleLog = 8 - Shift.getConstantOperandVal(1); in foldMaskAndShiftToExtract()
2056 MVT VT = N.getSimpleValueType(); in foldMaskAndShiftToExtract() local
2062 SDValue Ext = DAG.getZExtOrTrunc(And, DL, VT); in foldMaskAndShiftToExtract()
2064 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Ext, ShlCount); in foldMaskAndShiftToExtract()
2067 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskAndShiftToExtract()
2069 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskAndShiftToExtract()
2095 int64_t Mask = cast<ConstantSDNode>(N->getOperand(1))->getSExtValue(); in foldMaskedShiftToScaledMask()
2125 MVT VT = N.getSimpleValueType(); in foldMaskedShiftToScaledMask() local
2128 SDValue NewX = DAG.getNode(ISD::ANY_EXTEND, DL, VT, X); in foldMaskedShiftToScaledMask()
2133 SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT); in foldMaskedShiftToScaledMask()
2134 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask); in foldMaskedShiftToScaledMask()
2135 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1)); in foldMaskedShiftToScaledMask()
2138 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskedShiftToScaledMask()
2140 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskedShiftToScaledMask()
2192 unsigned MaskLZ = 64 - (MaskIdx + MaskLen); in foldMaskAndShiftToScale()
2206 unsigned ScaleDown = (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt; in foldMaskAndShiftToScale()
2209 MaskLZ -= ScaleDown; in foldMaskAndShiftToScale()
2211 // The final check is to ensure that any masked out high bits of X are in foldMaskAndShiftToScale()
2219 unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() - in foldMaskAndShiftToScale()
2221 // Assume that we'll replace the any-extend with a zero-extend, and in foldMaskAndShiftToScale()
2224 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits; in foldMaskAndShiftToScale()
2234 MVT VT = N.getSimpleValueType(); in foldMaskAndShiftToScale() local
2236 assert(X.getValueType() != VT); in foldMaskAndShiftToScale()
2238 SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X); in foldMaskAndShiftToScale()
2247 SDValue NewExt = DAG.getZExtOrTrunc(NewSRL, DL, VT); in foldMaskAndShiftToScale()
2249 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewExt, NewSHLAmt); in foldMaskAndShiftToScale()
2252 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskAndShiftToScale()
2254 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskAndShiftToScale()
2303 MVT VT = N.getSimpleValueType(); in foldMaskedShiftToBEXTR() local
2309 SDValue NewExt = DAG.getZExtOrTrunc(NewAnd, DL, VT); in foldMaskedShiftToBEXTR()
2311 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewExt, NewSHLAmt); in foldMaskedShiftToBEXTR()
2314 // a valid topological ordering as nothing is going to go back and re-sort in foldMaskedShiftToBEXTR()
2316 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no in foldMaskedShiftToBEXTR()
2346 EVT VT = N.getValueType(); in matchIndexRecursively() local
2349 // index: add(x,c) -> index: x, disp + c in matchIndexRecursively()
2350 if (CurDAG->isBaseWithConstantOffset(N)) { in matchIndexRecursively()
2352 uint64_t Offset = (uint64_t)AddVal->getSExtValue() * AM.Scale; in matchIndexRecursively()
2357 // index: add(x,x) -> index: x, scale * 2 in matchIndexRecursively()
2365 // index: shl(x,i) -> index: x, scale * (1 << i) in matchIndexRecursively()
2375 // index: sext(add_nsw(x,c)) -> index: sext(x), disp + sext(c) in matchIndexRecursively()
2377 if (Opc == ISD::SIGN_EXTEND && !VT.isVector() && N.hasOneUse()) { in matchIndexRecursively()
2379 if (Src.getOpcode() == ISD::ADD && Src->getFlags().hasNoSignedWrap() && in matchIndexRecursively()
2381 if (CurDAG->isBaseWithConstantOffset(Src)) { in matchIndexRecursively()
2384 uint64_t Offset = (uint64_t)AddVal->getSExtValue(); in matchIndexRecursively()
2387 SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc); in matchIndexRecursively()
2388 SDValue ExtVal = CurDAG->getConstant(Offset, DL, VT); in matchIndexRecursively()
2389 SDValue ExtAdd = CurDAG->getNode(ISD::ADD, DL, VT, ExtSrc, ExtVal); in matchIndexRecursively()
2393 CurDAG->ReplaceAllUsesWith(N, ExtAdd); in matchIndexRecursively()
2394 CurDAG->RemoveDeadNode(N.getNode()); in matchIndexRecursively()
2401 // index: zext(add_nuw(x,c)) -> index: zext(x), disp + zext(c) in matchIndexRecursively()
2402 // index: zext(addlike(x,c)) -> index: zext(x), disp + zext(c) in matchIndexRecursively()
2404 if (Opc == ISD::ZERO_EXTEND && !VT.isVector() && N.hasOneUse()) { in matchIndexRecursively()
2407 if (((SrcOpc == ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) || in matchIndexRecursively()
2408 CurDAG->isADDLike(Src, /*NoWrap=*/true)) && in matchIndexRecursively()
2410 if (CurDAG->isBaseWithConstantOffset(Src)) { in matchIndexRecursively()
2425 (AddSrc->getFlags().hasNoUnsignedWrap() || in matchIndexRecursively()
2426 CurDAG->MaskedValueIsZero(ShVal, HiBits))) { in matchIndexRecursively()
2428 SDValue ExtShVal = CurDAG->getNode(Opc, DL, VT, ShVal); in matchIndexRecursively()
2429 SDValue ExtShift = CurDAG->getNode(ISD::SHL, DL, VT, ExtShVal, in matchIndexRecursively()
2437 SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc); in matchIndexRecursively()
2438 SDValue ExtVal = CurDAG->getConstant(Offset, DL, VT); in matchIndexRecursively()
2439 SDValue ExtAdd = CurDAG->getNode(SrcOpc, DL, VT, ExtSrc, ExtVal); in matchIndexRecursively()
2443 CurDAG->ReplaceAllUsesWith(N, ExtAdd); in matchIndexRecursively()
2444 CurDAG->RemoveDeadNode(N.getNode()); in matchIndexRecursively()
2468 // RIP relative addressing: %rip + 32-bit displacement! in matchAddressRecursively()
2473 if (!(AM.ES || AM.MCSym) && AM.JT != -1) in matchAddressRecursively()
2477 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) in matchAddressRecursively()
2488 AM.MCSym = ESNode->getMCSymbol(); in matchAddressRecursively()
2494 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); in matchAddressRecursively()
2514 (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { in matchAddressRecursively()
2516 AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); in matchAddressRecursively()
2526 unsigned Val = CN->getZExtValue(); in matchAddressRecursively()
2529 // the base doesn't end up getting used, a post-processing step in matchAddressRecursively()
2544 // We only handle up to 64-bit values here as those are what matter for in matchAddressRecursively()
2553 // The mask used for the transform is expected to be post-shift, but we in matchAddressRecursively()
2575 // X*[3,5,9] -> X+X*[2,4,8] in matchAddressRecursively()
2580 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || in matchAddressRecursively()
2581 CN->getZExtValue() == 9) { in matchAddressRecursively()
2582 AM.Scale = unsigned(CN->getZExtValue())-1; in matchAddressRecursively()
2590 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && in matchAddressRecursively()
2594 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); in matchAddressRecursively()
2608 // Given A-B, if A can be completely folded into the address and in matchAddressRecursively()
2609 // the index field with the index field unused, use -B as the index. in matchAddressRecursively()
2612 // other uses, since it avoids a two-address sub instruction, however in matchAddressRecursively()
2638 if (!RHS.getNode()->hasOneUse() || in matchAddressRecursively()
2639 RHS.getNode()->getOpcode() == ISD::CopyFromReg || in matchAddressRecursively()
2640 RHS.getNode()->getOpcode() == ISD::TRUNCATE || in matchAddressRecursively()
2641 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || in matchAddressRecursively()
2642 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && in matchAddressRecursively()
2648 !AM.Base_Reg.getNode()->hasOneUse()) || in matchAddressRecursively()
2650 --Cost; in matchAddressRecursively()
2656 --Cost; in matchAddressRecursively()
2675 if (!CurDAG->isADDLike(N)) in matchAddressRecursively()
2684 // Perform some heroic transforms on an and of a constant-count shift in matchAddressRecursively()
2690 // We only handle up to 64-bit values here as those are what matter for in matchAddressRecursively()
2745 Mask = MaskC->getAPIntValue(); in matchAddressRecursively()
2749 if (Src.getOpcode() == ISD::SHL && Src.hasOneUse() && N->hasOneUse()) { in matchAddressRecursively()
2756 unsigned ShAmtV = ShAmtC->getZExtValue(); in matchAddressRecursively()
2764 if (!Src->getFlags().hasNoUnsignedWrap() && in matchAddressRecursively()
2765 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask)) in matchAddressRecursively()
2769 // --> shl (zext i8 %x to i32), (zext C1) in matchAddressRecursively()
2771 // --> shl (zext i8 (and %x, C2 >> C1) to i32), (zext C1) in matchAddressRecursively()
2773 MVT VT = N.getSimpleValueType(); in matchAddressRecursively() local
2778 Res = CurDAG->getConstant(Mask.lshr(ShAmtV), DL, SrcVT); in matchAddressRecursively()
2780 Res = CurDAG->getNode(ISD::AND, DL, SrcVT, ShlSrc, Res); in matchAddressRecursively()
2783 SDValue Zext = CurDAG->getNode(ISD::ZERO_EXTEND, DL, VT, Res); in matchAddressRecursively()
2785 SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, ShlAmt); in matchAddressRecursively()
2787 CurDAG->ReplaceAllUsesWith(N, NewShl); in matchAddressRecursively()
2788 CurDAG->RemoveDeadNode(N.getNode()); in matchAddressRecursively()
2860 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); in matchVectorAddressRecursively()
2910 AM.Scale = ScaleOp->getAsZExtVal(); in selectVectorAddr()
2913 // sign-extension, which is performed BEFORE scale. in selectVectorAddr()
2919 unsigned AddrSpace = Parent->getPointerInfo().getAddrSpace(); in selectVectorAddr()
2921 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in selectVectorAddr()
2923 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in selectVectorAddr()
2925 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); in selectVectorAddr()
2928 MVT VT = BasePtr.getSimpleValueType(); in selectVectorAddr() local
2934 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); in selectVectorAddr()
2953 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme in selectAddr()
2954 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores in selectAddr()
2955 Parent->getOpcode() != X86ISD::TLSCALL && // Fixme in selectAddr()
2956 Parent->getOpcode() != X86ISD::ENQCMD && // Fixme in selectAddr()
2957 Parent->getOpcode() != X86ISD::ENQCMDS && // Fixme in selectAddr()
2958 Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp in selectAddr()
2959 Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp in selectAddr()
2961 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); in selectAddr()
2963 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); in selectAddr()
2965 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); in selectAddr()
2967 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); in selectAddr()
2970 // Save the DL and VT before calling matchAddress, it can invalidate N. in selectAddr()
2972 MVT VT = N.getSimpleValueType(); in selectAddr() local
2977 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); in selectAddr()
2990 if (N->getOpcode() != X86ISD::Wrapper) in selectMOV64Imm32()
2997 if (N->getOpcode() == ISD::TargetGlobalTLSAddress) in selectMOV64Imm32()
3001 // Small/medium code model can reference non-TargetGlobalAddress objects with in selectMOV64Imm32()
3003 if (N->getOpcode() != ISD::TargetGlobalAddress) { in selectMOV64Imm32()
3008 const GlobalValue *GV = cast<GlobalAddressSDNode>(N)->getGlobal(); in selectMOV64Imm32()
3009 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange()) in selectMOV64Imm32()
3010 return CR->getUnsignedMax().ult(1ull << 32); in selectMOV64Imm32()
3025 if (RN && RN->getReg() == 0) in selectLEA64_32Addr()
3026 Base = CurDAG->getRegister(0, MVT::i64); in selectLEA64_32Addr()
3029 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL, in selectLEA64_32Addr()
3031 Base = CurDAG->getTargetInsertSubreg(X86::sub_32bit, DL, MVT::i64, ImplDef, in selectLEA64_32Addr()
3036 if (RN && RN->getReg() == 0) in selectLEA64_32Addr()
3037 Index = CurDAG->getRegister(0, MVT::i64); in selectLEA64_32Addr()
3040 "Expect to be extending 32-bit registers for use in LEA"); in selectLEA64_32Addr()
3041 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL, in selectLEA64_32Addr()
3043 Index = CurDAG->getTargetInsertSubreg(X86::sub_32bit, DL, MVT::i64, ImplDef, in selectLEA64_32Addr()
3058 // Save the DL and VT before calling matchAddress, it can invalidate N. in selectLEAAddr()
3060 MVT VT = N.getSimpleValueType(); in selectLEAAddr() local
3065 SDValue T = CurDAG->getRegister(0, MVT::i32); in selectLEAAddr()
3089 // its three-address nature. Tweak the cost function again when we can run in selectLEAAddr()
3092 // For X86-64, always use LEA to materialize RIP-relative addresses. in selectLEAAddr()
3093 if (Subtarget->is64Bit()) in selectLEAAddr()
3101 // duplicating flag-producing instructions later in the pipeline. in selectLEAAddr()
3112 their inclusion for different reasons (better for reg-alloc). in selectLEAAddr()
3117 // Value 1 is the flag output of the node - verify it's not dead. in selectLEAAddr()
3136 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); in selectLEAAddr()
3149 AM.GV = GA->getGlobal(); in selectTLSADDRAddr()
3150 AM.Disp += GA->getOffset(); in selectTLSADDRAddr()
3151 AM.SymbolFlags = GA->getTargetFlags(); in selectTLSADDRAddr()
3154 AM.ES = SA->getSymbol(); in selectTLSADDRAddr()
3155 AM.SymbolFlags = SA->getTargetFlags(); in selectTLSADDRAddr()
3158 if (Subtarget->is32Bit()) { in selectTLSADDRAddr()
3160 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); in selectTLSADDRAddr()
3163 MVT VT = N.getSimpleValueType(); in selectTLSADDRAddr() local
3164 getAddressOperands(AM, SDLoc(N), VT, Base, Scale, Index, Disp, Segment); in selectTLSADDRAddr()
3170 // truncated. If we see a truncation from pointer type to VT that truncates in selectRelocImm()
3172 EVT VT = N.getValueType(); in selectRelocImm() local
3182 // We can only use non-GlobalValues as immediates if they were not truncated, in selectRelocImm()
3185 unsigned Opc = N.getOperand(0)->getOpcode(); in selectRelocImm()
3193 // Check that the global's range fits into VT. in selectRelocImm()
3195 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange(); in selectRelocImm()
3196 if (!CR || CR->getUnsignedMax().uge(1ull << VT.getSizeInBits())) in selectRelocImm()
3200 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N), VT, in selectRelocImm()
3201 GA->getOffset(), GA->getTargetFlags()); in selectRelocImm()
3224 if (N->getOpcode() != X86ISD::VBROADCAST_LOAD || in tryFoldBroadcast()
3237 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); in getGlobalBaseReg()
3238 auto &DL = MF->getDataLayout(); in getGlobalBaseReg()
3239 return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode(); in getGlobalBaseReg()
3243 if (N->getOpcode() == ISD::TRUNCATE) in isSExtAbsoluteSymbolRef()
3244 N = N->getOperand(0).getNode(); in isSExtAbsoluteSymbolRef()
3245 if (N->getOpcode() != X86ISD::Wrapper) in isSExtAbsoluteSymbolRef()
3248 auto *GA = dyn_cast<GlobalAddressSDNode>(N->getOperand(0)); in isSExtAbsoluteSymbolRef()
3252 auto *GV = GA->getGlobal(); in isSExtAbsoluteSymbolRef()
3253 std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange(); in isSExtAbsoluteSymbolRef()
3255 return CR->getSignedMin().sge(-1ull << Width) && in isSExtAbsoluteSymbolRef()
3256 CR->getSignedMax().slt(1ull << Width); in isSExtAbsoluteSymbolRef()
3258 // space, so globals can be a sign extended 32-bit immediate. in isSExtAbsoluteSymbolRef()
3265 assert(N->isMachineOpcode() && "Unexpected node"); in getCondFromNode()
3266 unsigned Opc = N->getMachineOpcode(); in getCondFromNode()
3267 const MCInstrDesc &MCID = getInstrInfo()->get(Opc); in getCondFromNode()
3272 return static_cast<X86::CondCode>(N->getConstantOperandVal(CondNo)); in getCondFromNode()
3279 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); in onlyUsesZeroFlag()
3285 if (UI->getOpcode() != ISD::CopyToReg || in onlyUsesZeroFlag()
3286 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) in onlyUsesZeroFlag()
3289 for (SDNode::use_iterator FlagUI = UI->use_begin(), in onlyUsesZeroFlag()
3290 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { in onlyUsesZeroFlag()
3294 if (!FlagUI->isMachineOpcode()) return false; in onlyUsesZeroFlag()
3315 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); in hasNoSignFlagUses()
3321 if (UI->getOpcode() != ISD::CopyToReg || in hasNoSignFlagUses()
3322 cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) in hasNoSignFlagUses()
3325 for (SDNode::use_iterator FlagUI = UI->use_begin(), in hasNoSignFlagUses()
3326 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { in hasNoSignFlagUses()
3330 if (!FlagUI->isMachineOpcode()) return false; in hasNoSignFlagUses()
3371 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); in hasNoCarryFlagUses()
3377 unsigned UIOpc = UI->getOpcode(); in hasNoCarryFlagUses()
3381 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) in hasNoCarryFlagUses()
3384 for (SDNode::use_iterator FlagUI = UI->use_begin(), FlagUE = UI->use_end(); in hasNoCarryFlagUses()
3390 if (!FlagUI->isMachineOpcode()) in hasNoCarryFlagUses()
3403 // This might be an unselected node. So look for the pre-isel opcodes that in hasNoCarryFlagUses()
3416 X86::CondCode CC = (X86::CondCode)UI->getConstantOperandVal(CCOpNo); in hasNoCarryFlagUses()
3434 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false; in isFusableLoadOpStorePattern()
3436 // Is the store non-extending and non-indexed? in isFusableLoadOpStorePattern()
3437 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) in isFusableLoadOpStorePattern()
3440 SDValue Load = StoredVal->getOperand(LoadOpNo); in isFusableLoadOpStorePattern()
3441 // Is the stored value a non-extending and non-indexed load? in isFusableLoadOpStorePattern()
3452 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || in isFusableLoadOpStorePattern()
3453 LoadNode->getOffset() != StoreNode->getOffset()) in isFusableLoadOpStorePattern()
3462 // Visualization of Load-Op-Store fusion: in isFusableLoadOpStorePattern()
3463 // ------------------------- in isFusableLoadOpStorePattern()
3465 // *-lines = Chain operand dependencies. in isFusableLoadOpStorePattern()
3466 // |-lines = Normal operand dependencies. in isFusableLoadOpStorePattern()
3467 // Dependencies flow down and right. n-suffix references multiple nodes. in isFusableLoadOpStorePattern()
3472 // Xn A-LD Yn TF Yn in isFusableLoadOpStorePattern()
3475 // * * \ | => A--LD_OP_ST in isFusableLoadOpStorePattern()
3480 // A-ST Zn in isFusableLoadOpStorePattern()
3483 // This merge induced dependences from: #1: Xn -> LD, OP, Zn in isFusableLoadOpStorePattern()
3484 // #2: Yn -> LD in isFusableLoadOpStorePattern()
3485 // #3: ST -> Zn in isFusableLoadOpStorePattern()
3499 SDValue Chain = StoreNode->getChain(); in isFusableLoadOpStorePattern()
3523 for (SDValue Op : StoredVal->ops()) in isFusableLoadOpStorePattern()
3533 CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ChainOps); in isFusableLoadOpStorePattern()
3548 // [(store (add (loadi64 addr:$dst), -1), addr:$dst),
3552 // [(store (add (loadi64 addr:$dst), -1), addr:$dst),
3559 SDValue StoredVal = StoreNode->getOperand(1); in foldLoadStoreIntoMemOperand()
3560 unsigned Opc = StoredVal->getOpcode(); in foldLoadStoreIntoMemOperand()
3565 EVT MemVT = StoreNode->getMemoryVT(); in foldLoadStoreIntoMemOperand()
3605 if (!selectAddr(LoadNode, LoadNode->getBasePtr(), Base, Scale, Index, Disp, in foldLoadStoreIntoMemOperand()
3633 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, in foldLoadStoreIntoMemOperand()
3640 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) { in foldLoadStoreIntoMemOperand()
3643 // ADD/SUB with 1/-1 and carry flag isn't used can use inc/dec. in foldLoadStoreIntoMemOperand()
3650 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, in foldLoadStoreIntoMemOperand()
3716 SDValue Operand = StoredVal->getOperand(1-LoadOpNo); in foldLoadStoreIntoMemOperand()
3721 int64_t OperandV = OperandC->getSExtValue(); in foldLoadStoreIntoMemOperand()
3727 ((MemVT != MVT::i8 && !isInt<8>(OperandV) && isInt<8>(-OperandV)) || in foldLoadStoreIntoMemOperand()
3729 isInt<32>(-OperandV))) && in foldLoadStoreIntoMemOperand()
3731 OperandV = -OperandV; in foldLoadStoreIntoMemOperand()
3736 Operand = CurDAG->getTargetConstant(OperandV, SDLoc(Node), MemVT); in foldLoadStoreIntoMemOperand()
3743 CurDAG->getCopyToReg(InputChain, SDLoc(Node), X86::EFLAGS, in foldLoadStoreIntoMemOperand()
3748 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other, in foldLoadStoreIntoMemOperand()
3753 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other, in foldLoadStoreIntoMemOperand()
3762 MachineMemOperand *MemOps[] = {StoreNode->getMemOperand(), in foldLoadStoreIntoMemOperand()
3763 LoadNode->getMemOperand()}; in foldLoadStoreIntoMemOperand()
3764 CurDAG->setNodeMemRefs(Result, MemOps); in foldLoadStoreIntoMemOperand()
3770 CurDAG->RemoveDeadNode(Node); in foldLoadStoreIntoMemOperand()
3776 // a) x & (1 << nbits) - 1
3777 // b) x & ~(-1 << nbits)
3778 // c) x & (-1 >> (32 - y))
3779 // d) x << (32 - y) >> (32 - y)
3780 // e) (1 << nbits) - 1
3783 (Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::AND || in matchBitExtract()
3784 Node->getOpcode() == ISD::SRL) && in matchBitExtract()
3785 "Should be either an and-mask, or right-shift after clearing high bits."); in matchBitExtract()
3788 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2()) in matchBitExtract()
3791 MVT NVT = Node->getSimpleValueType(0); in matchBitExtract()
3800 // If we have BMI2's BZHI, we are ok with muti-use patterns. in matchBitExtract()
3801 // Else, if we only have BMI1's BEXTR, we require one-use. in matchBitExtract()
3802 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2(); in matchBitExtract()
3807 Op.getNode()->hasNUsesOfValue(NUses, Op.getResNo()); in matchBitExtract()
3821 if (V->getOpcode() == ISD::TRUNCATE && checkOneUse(V)) { in matchBitExtract()
3824 "Expected i64 -> i32 truncation"); in matchBitExtract()
3830 // a) x & ((1 << nbits) + (-1)) in matchBitExtract()
3832 &NegateNBits](SDValue Mask) -> bool { in matchBitExtract()
3834 if (Mask->getOpcode() != ISD::ADD || !checkOneUse(Mask)) in matchBitExtract()
3836 // We should be adding all-ones constant (i.e. subtracting one.) in matchBitExtract()
3837 if (!isAllOnesConstant(Mask->getOperand(1))) in matchBitExtract()
3840 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0)); in matchBitExtract()
3841 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0)) in matchBitExtract()
3843 if (!isOneConstant(M0->getOperand(0))) in matchBitExtract()
3845 NBits = M0->getOperand(1); in matchBitExtract()
3852 return CurDAG->MaskedValueIsAllOnes( in matchBitExtract()
3857 // b) x & ~(-1 << nbits) in matchBitExtract()
3859 &NBits, &NegateNBits](SDValue Mask) -> bool { in matchBitExtract()
3863 // The -1 only has to be all-ones for the final Node's NVT. in matchBitExtract()
3864 if (!isAllOnes(Mask->getOperand(1))) in matchBitExtract()
3866 // Match `-1 << nbits`. Might be truncated. Must only have one use! in matchBitExtract()
3867 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0)); in matchBitExtract()
3868 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0)) in matchBitExtract()
3870 // The -1 only has to be all-ones for the final Node's NVT. in matchBitExtract()
3871 if (!isAllOnes(M0->getOperand(0))) in matchBitExtract()
3873 NBits = M0->getOperand(1); in matchBitExtract()
3878 // Try to match potentially-truncated shift amount as `(bitwidth - y)`, in matchBitExtract()
3879 // or leave the shift amount as-is, but then we'll have to negate it. in matchBitExtract()
3887 // Try to match the shift amount as (bitwidth - y). It should go away, too. in matchBitExtract()
3892 if (!V0 || V0->getZExtValue() != Bitwidth) in matchBitExtract()
3898 // c) x & (-1 >> z) but then we'll have to subtract z from bitwidth in matchBitExtract()
3900 // c) x & (-1 >> (32 - y)) in matchBitExtract()
3902 canonicalizeShiftAmt](SDValue Mask) -> bool { in matchBitExtract()
3909 // We should be shifting truly all-ones constant. in matchBitExtract()
3917 // Pattern c. is non-canonical, and is expanded into pattern d. iff there in matchBitExtract()
3928 // d) x << (32 - y) >> (32 - y) in matchBitExtract()
3931 &X](SDNode *Node) -> bool { in matchBitExtract()
3932 if (Node->getOpcode() != ISD::SRL) in matchBitExtract()
3934 SDValue N0 = Node->getOperand(0); in matchBitExtract()
3935 if (N0->getOpcode() != ISD::SHL) in matchBitExtract()
3938 SDValue N1 = Node->getOperand(1); in matchBitExtract()
3939 SDValue N01 = N0->getOperand(1); in matchBitExtract()
3950 X = N0->getOperand(0); in matchBitExtract()
3955 matchPatternC](SDValue Mask) -> bool { in matchBitExtract()
3959 if (Node->getOpcode() == ISD::AND) { in matchBitExtract()
3960 X = Node->getOperand(0); in matchBitExtract()
3961 SDValue Mask = Node->getOperand(1); in matchBitExtract()
3971 X = CurDAG->getAllOnesConstant(SDLoc(Node), NVT); in matchBitExtract()
3977 if (NegateNBits && !Subtarget->hasBMI2()) in matchBitExtract()
3983 NBits = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NBits); in matchBitExtract()
3986 // Insert 8-bit NBits into lowest 8 bits of 32-bit register. in matchBitExtract()
3989 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i32), 0); in matchBitExtract()
3992 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit, DL, MVT::i32); in matchBitExtract()
3994 NBits = SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, DL, in matchBitExtract()
3999 // We might have matched the amount of high bits to be cleared, in matchBitExtract()
4002 SDValue BitWidthC = CurDAG->getConstant(NVT.getSizeInBits(), DL, MVT::i32); in matchBitExtract()
4005 NBits = CurDAG->getNode(ISD::SUB, DL, MVT::i32, BitWidthC, NBits); in matchBitExtract()
4009 if (Subtarget->hasBMI2()) { in matchBitExtract()
4012 // But have to place the bit count into the wide-enough register first. in matchBitExtract()
4013 NBits = CurDAG->getNode(ISD::ANY_EXTEND, DL, NVT, NBits); in matchBitExtract()
4017 SDValue Extract = CurDAG->getNode(X86ISD::BZHI, DL, NVT, X, NBits); in matchBitExtract()
4024 // *logically* shifted (potentially with one-use trunc inbetween), in matchBitExtract()
4026 // and if so look past one-use truncation. in matchBitExtract()
4029 // FIXME: only if the shift is one-use? in matchBitExtract()
4044 SDValue C8 = CurDAG->getConstant(8, DL, MVT::i8); in matchBitExtract()
4046 SDValue Control = CurDAG->getNode(ISD::SHL, DL, MVT::i32, NBits, C8); in matchBitExtract()
4050 // FIXME: only if the shift is one-use? in matchBitExtract()
4058 // Now, *zero*-extend the shift amount. The bits 8...15 *must* be zero! in matchBitExtract()
4061 ShiftAmt = CurDAG->getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShiftAmt); in matchBitExtract()
4065 Control = CurDAG->getNode(ISD::OR, DL, MVT::i32, Control, ShiftAmt); in matchBitExtract()
4069 // But have to place the 'control' into the wide-enough register first. in matchBitExtract()
4071 Control = CurDAG->getNode(ISD::ANY_EXTEND, DL, XVT, Control); in matchBitExtract()
4076 SDValue Extract = CurDAG->getNode(X86ISD::BEXTR, DL, XVT, X, Control); in matchBitExtract()
4081 Extract = CurDAG->getNode(ISD::TRUNCATE, DL, NVT, Extract); in matchBitExtract()
4092 MVT NVT = Node->getSimpleValueType(0); in matchBEXTRFromAndImm()
4095 SDValue N0 = Node->getOperand(0); in matchBEXTRFromAndImm()
4096 SDValue N1 = Node->getOperand(1); in matchBEXTRFromAndImm()
4101 // TODO: Maybe load folding, greater than 32-bit masks, or a guarantee of LICM in matchBEXTRFromAndImm()
4105 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR()); in matchBEXTRFromAndImm()
4106 if (!PreferBEXTR && !Subtarget->hasBMI2()) in matchBEXTRFromAndImm()
4110 if (N0->getOpcode() != ISD::SRL && N0->getOpcode() != ISD::SRA) in matchBEXTRFromAndImm()
4114 if (!N0->hasOneUse()) in matchBEXTRFromAndImm()
4123 auto *ShiftCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); in matchBEXTRFromAndImm()
4128 uint64_t Mask = MaskCst->getZExtValue(); in matchBEXTRFromAndImm()
4132 uint64_t Shift = ShiftCst->getZExtValue(); in matchBEXTRFromAndImm()
4154 #define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC) in matchBEXTRFromAndImm()
4156 assert(Subtarget->hasBMI2() && "We must have BMI2's BZHI then."); in matchBEXTRFromAndImm()
4160 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT); in matchBEXTRFromAndImm()
4166 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0); in matchBEXTRFromAndImm()
4172 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT); in matchBEXTRFromAndImm()
4173 if (Subtarget->hasTBM()) { in matchBEXTRFromAndImm()
4177 assert(Subtarget->hasBMI() && "We must have BMI1's BEXTR then."); in matchBEXTRFromAndImm()
4184 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0); in matchBEXTRFromAndImm()
4189 SDValue Input = N0->getOperand(0); in matchBEXTRFromAndImm()
4194 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other); in matchBEXTRFromAndImm()
4195 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in matchBEXTRFromAndImm()
4198 // Record the mem-refs in matchBEXTRFromAndImm()
4199 CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()}); in matchBEXTRFromAndImm()
4201 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control); in matchBEXTRFromAndImm()
4206 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT); in matchBEXTRFromAndImm()
4210 CurDAG->getMachineNode(NewOpc, dl, NVT, SDValue(NewNode, 0), ShAmt); in matchBEXTRFromAndImm()
4219 MVT VT, SDNode *Node) { in emitPCMPISTR() argument
4220 SDValue N0 = Node->getOperand(0); in emitPCMPISTR()
4221 SDValue N1 = Node->getOperand(1); in emitPCMPISTR()
4222 SDValue Imm = Node->getOperand(2); in emitPCMPISTR()
4223 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue(); in emitPCMPISTR()
4224 Imm = CurDAG->getTargetConstant(*Val, SDLoc(Node), Imm.getValueType()); in emitPCMPISTR()
4231 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other); in emitPCMPISTR()
4232 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in emitPCMPISTR()
4235 // Record the mem-refs in emitPCMPISTR()
4236 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); in emitPCMPISTR()
4241 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32); in emitPCMPISTR()
4242 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops); in emitPCMPISTR()
4251 MVT VT, SDNode *Node, in emitPCMPESTR() argument
4253 SDValue N0 = Node->getOperand(0); in emitPCMPESTR()
4254 SDValue N2 = Node->getOperand(2); in emitPCMPESTR()
4255 SDValue Imm = Node->getOperand(4); in emitPCMPESTR()
4256 auto *Val = cast<ConstantSDNode>(Imm)->getConstantIntValue(); in emitPCMPESTR()
4257 Imm = CurDAG->getTargetConstant(*Val, SDLoc(Node), Imm.getValueType()); in emitPCMPESTR()
4264 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue); in emitPCMPESTR()
4265 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in emitPCMPESTR()
4269 // Record the mem-refs in emitPCMPESTR()
4270 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N2)->getMemOperand()}); in emitPCMPESTR()
4275 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue); in emitPCMPESTR()
4276 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops); in emitPCMPESTR()
4282 EVT VT = N->getValueType(0); in tryShiftAmountMod() local
4285 if (VT.isVector()) in tryShiftAmountMod()
4289 unsigned Size = VT == MVT::i64 ? 64 : 32; in tryShiftAmountMod()
4291 SDValue OrigShiftAmt = N->getOperand(1); in tryShiftAmountMod()
4296 if (ShiftAmt->getOpcode() == ISD::TRUNCATE) in tryShiftAmountMod()
4297 ShiftAmt = ShiftAmt->getOperand(0); in tryShiftAmountMod()
4303 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB || in tryShiftAmountMod()
4304 ShiftAmt->getOpcode() == ISD::XOR) { in tryShiftAmountMod()
4305 SDValue Add0 = ShiftAmt->getOperand(0); in tryShiftAmountMod()
4306 SDValue Add1 = ShiftAmt->getOperand(1); in tryShiftAmountMod()
4309 // If we are shifting by X+/-/^N where N == 0 mod Size, then just shift by X in tryShiftAmountMod()
4311 if (Add1C && Add1C->getAPIntValue().urem(Size) == 0) { in tryShiftAmountMod()
4314 } else if (ShiftAmt->getOpcode() != ISD::ADD && ShiftAmt.hasOneUse() && in tryShiftAmountMod()
4315 ((Add0C && Add0C->getAPIntValue().urem(Size) == Size - 1) || in tryShiftAmountMod()
4316 (Add1C && Add1C->getAPIntValue().urem(Size) == Size - 1))) { in tryShiftAmountMod()
4317 // If we are doing a NOT on just the lower bits with (Size*N-1) -/^ X in tryShiftAmountMod()
4322 // We can only do N-X, not X-N in tryShiftAmountMod()
4323 if (ShiftAmt->getOpcode() == ISD::SUB && Add0C == nullptr) in tryShiftAmountMod()
4328 SDValue AllOnes = CurDAG->getAllOnesConstant(DL, OpVT); in tryShiftAmountMod()
4329 NewShiftAmt = CurDAG->getNode(ISD::XOR, DL, OpVT, in tryShiftAmountMod()
4333 // If we are shifting by N-X where N == 0 mod Size, then just shift by in tryShiftAmountMod()
4334 // -X to generate a NEG instead of a SUB of a constant. in tryShiftAmountMod()
4335 } else if (ShiftAmt->getOpcode() == ISD::SUB && Add0C && in tryShiftAmountMod()
4336 Add0C->getZExtValue() != 0) { in tryShiftAmountMod()
4339 if (Add0C->getZExtValue() % Size == 0) in tryShiftAmountMod()
4342 Add0C->getZExtValue() % 32 == 0) { in tryShiftAmountMod()
4343 // We have a 64-bit shift by (n*32-x), turn it into -(x+n*32). in tryShiftAmountMod()
4350 Add0 = CurDAG->getZExtOrTrunc(Add0, DL, SubVT); in tryShiftAmountMod()
4354 X = CurDAG->getNode(ISD::ADD, DL, SubVT, Add1, Add0); in tryShiftAmountMod()
4361 SDValue Zero = CurDAG->getConstant(0, DL, SubVT); in tryShiftAmountMod()
4362 SDValue Neg = CurDAG->getNode(ISD::SUB, DL, SubVT, Zero, X); in tryShiftAmountMod()
4376 NewShiftAmt = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NewShiftAmt); in tryShiftAmountMod()
4383 NewShiftAmt = CurDAG->getNode(ISD::AND, DL, MVT::i8, NewShiftAmt, in tryShiftAmountMod()
4384 CurDAG->getConstant(Size - 1, DL, MVT::i8)); in tryShiftAmountMod()
4388 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(N, N->getOperand(0), in tryShiftAmountMod()
4399 if (OrigShiftAmt.getNode()->use_empty()) in tryShiftAmountMod()
4400 CurDAG->RemoveDeadNode(OrigShiftAmt.getNode()); in tryShiftAmountMod()
4409 MVT NVT = N->getSimpleValueType(0); in tryShrinkShlLogicImm()
4410 unsigned Opcode = N->getOpcode(); in tryShrinkShlLogicImm()
4415 SDValue Shift = N->getOperand(0); in tryShrinkShlLogicImm()
4416 SDValue N1 = N->getOperand(1); in tryShrinkShlLogicImm()
4422 int64_t Val = Cst->getSExtValue(); in tryShrinkShlLogicImm()
4446 uint64_t ShAmt = ShlCst->getZExtValue(); in tryShrinkShlLogicImm()
4450 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1; in tryShrinkShlLogicImm()
4491 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits(); in tryShrinkShlLogicImm()
4497 NeededMask &= ~Cst->getAPIntValue(); in tryShrinkShlLogicImm()
4499 if (CurDAG->MaskedValueIsZero(N->getOperand(0), NeededMask)) in tryShrinkShlLogicImm()
4505 SDValue NewX = CurDAG->getNode(ISD::ANY_EXTEND, dl, NVT, X); in tryShrinkShlLogicImm()
4510 SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT); in tryShrinkShlLogicImm()
4512 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst); in tryShrinkShlLogicImm()
4514 SDValue NewSHL = CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp, in tryShrinkShlLogicImm()
4545 unsigned Size = MemIntr->getMemoryVT().getSizeInBits(); in matchVPTERNLOG()
4582 SDValue TImm = CurDAG->getTargetConstant(Imm, DL, MVT::i8); in matchVPTERNLOG()
4584 MVT NVT = Root->getSimpleValueType(0); in matchVPTERNLOG()
4588 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other); in matchVPTERNLOG()
4593 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits(); in matchVPTERNLOG()
4618 MNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops); in matchVPTERNLOG()
4622 // Record the mem-refs in matchVPTERNLOG()
4623 CurDAG->setNodeMemRefs(MNode, {cast<MemSDNode>(C)->getMemOperand()}); in matchVPTERNLOG()
4636 MNode = CurDAG->getMachineNode(Opc, DL, NVT, {A, B, C, TImm}); in matchVPTERNLOG()
4640 CurDAG->RemoveDeadNode(Root); in matchVPTERNLOG()
4647 MVT NVT = N->getSimpleValueType(0); in tryVPTERNLOG()
4650 if (!NVT.isVector() || !Subtarget->hasAVX512() || in tryVPTERNLOG()
4654 // We need VLX for 128/256-bit. in tryVPTERNLOG()
4655 if (!(Subtarget->hasVLX() || NVT.is512BitVector())) in tryVPTERNLOG()
4658 SDValue N0 = N->getOperand(0); in tryVPTERNLOG()
4659 SDValue N1 = N->getOperand(1); in tryVPTERNLOG()
4722 switch (N->getOpcode()) { in tryVPTERNLOG()
4738 /// If the high bits of an 'and' operand are known zero, try setting the
4739 /// high bits of an 'and' constant operand to produce a smaller encoding by
4740 /// creating a small, sign-extended negative immediate rather than a large
4743 /// the 'and' mask can be made -1, so the 'and' itself is unnecessary. In that
4748 MVT VT = And->getSimpleValueType(0); in shrinkAndImmediate() local
4749 if (VT != MVT::i32 && VT != MVT::i64) in shrinkAndImmediate()
4752 auto *And1C = dyn_cast<ConstantSDNode>(And->getOperand(1)); in shrinkAndImmediate()
4758 // patterns to use a 32-bit and instead of a 64-bit and by relying on the in shrinkAndImmediate()
4761 APInt MaskVal = And1C->getAPIntValue(); in shrinkAndImmediate()
4763 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32)) in shrinkAndImmediate()
4767 if (VT == MVT::i64 && MaskLZ >= 32) { in shrinkAndImmediate()
4768 MaskLZ -= 32; in shrinkAndImmediate()
4772 SDValue And0 = And->getOperand(0); in shrinkAndImmediate()
4783 if (VT == MVT::i64 && MaskVal.getBitWidth() < 64) { in shrinkAndImmediate()
4790 if (!CurDAG->MaskedValueIsZero(And0, HighZeros)) in shrinkAndImmediate()
4793 // Check if the mask is -1. In that case, this is an unnecessary instruction in shrinkAndImmediate()
4801 SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT); in shrinkAndImmediate()
4803 SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask); in shrinkAndImmediate()
4811 #define VPTESTM_CASE(VT, SUFFIX) \ in getVPTESTMOpc() argument
4812 case MVT::VT: \ in getVPTESTMOpc()
4819 default: llvm_unreachable("Unexpected VT!"); \ in getVPTESTMOpc()
4861 assert(Subtarget->hasAVX512() && "Expected AVX512!"); in tryVPTESTM()
4863 "Unexpected VT!"); in tryVPTESTM()
4866 ISD::CondCode CC = cast<CondCodeSDNode>(Setcc.getOperand(2))->get(); in tryVPTESTM()
4904 bool Widen = !Subtarget->hasVLX() && !CmpVT.is512BitVector(); in tryVPTESTM()
4929 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.getSizeInBits()) in tryVPTESTM()
4967 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl, in tryVPTESTM()
4969 Src0 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src0); in tryVPTESTM()
4972 Src1 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src1); in tryVPTESTM()
4976 unsigned RegClass = TLI->getRegClassFor(MaskVT)->getID(); in tryVPTESTM()
4977 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32); in tryVPTESTM()
4978 InMask = SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, in tryVPTESTM()
4989 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other); in tryVPTESTM()
4994 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in tryVPTESTM()
4998 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in tryVPTESTM()
5003 // Record the mem-refs in tryVPTESTM()
5004 CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Src1)->getMemOperand()}); in tryVPTESTM()
5007 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1); in tryVPTESTM()
5009 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1); in tryVPTESTM()
5012 // If we widened, we need to shrink the mask VT. in tryVPTESTM()
5014 unsigned RegClass = TLI->getRegClassFor(ResVT)->getID(); in tryVPTESTM()
5015 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32); in tryVPTESTM()
5016 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, in tryVPTESTM()
5021 CurDAG->RemoveDeadNode(Root); in tryVPTESTM()
5028 assert(N->getOpcode() == ISD::OR && "Unexpected opcode!"); in tryMatchBitSelect()
5030 MVT NVT = N->getSimpleValueType(0); in tryMatchBitSelect()
5033 if (!NVT.isVector() || !Subtarget->hasAVX512()) in tryMatchBitSelect()
5036 // We need VLX for 128/256-bit. in tryMatchBitSelect()
5037 if (!(Subtarget->hasVLX() || NVT.is512BitVector())) in tryMatchBitSelect()
5040 SDValue N0 = N->getOperand(0); in tryMatchBitSelect()
5041 SDValue N1 = N->getOperand(1); in tryMatchBitSelect()
5067 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8); in tryMatchBitSelect()
5068 SDValue Ternlog = CurDAG->getNode(X86ISD::VPTERNLOG, dl, NVT, A, B, C, Imm); in tryMatchBitSelect()
5076 MVT NVT = Node->getSimpleValueType(0); in Select()
5077 unsigned Opcode = Node->getOpcode(); in Select()
5080 if (Node->isMachineOpcode()) { in Select()
5081 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); in Select()
5082 Node->setNodeId(-1); in Select()
5089 unsigned IntNo = Node->getConstantOperandVal(1); in Select()
5094 if (!Subtarget->hasKL()) in Select()
5108 SDValue Chain = Node->getOperand(0); in Select()
5109 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(3), in Select()
5112 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(4), in Select()
5115 MachineSDNode *Res = CurDAG->getMachineNode( in Select()
5116 Opcode, dl, Node->getVTList(), in Select()
5117 {Node->getOperand(2), Chain, Chain.getValue(1)}); in Select()
5123 if (!Subtarget->hasAMXTILE()) in Select()
5126 CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); in Select()
5127 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA); in Select()
5132 SDValue Base = Node->getOperand(4); in Select()
5134 SDValue Index = Node->getOperand(5); in Select()
5135 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32); in Select()
5136 SDValue Segment = CurDAG->getRegister(0, MVT::i16); in Select()
5137 SDValue Chain = Node->getOperand(0); in Select()
5139 SDValue Ops[] = {Node->getOperand(2), in Select()
5140 Node->getOperand(3), in Select()
5147 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops); in Select()
5155 unsigned IntNo = Node->getConstantOperandVal(1); in Select()
5161 bool Use64BitPtr = Node->getOperand(2).getValueType() == MVT::i64; in Select()
5167 if (!Subtarget->hasSSE3()) in Select()
5172 if (!Subtarget->hasMWAITX()) in Select()
5177 if (!Subtarget->hasCLZERO()) in Select()
5185 SDValue Chain = CurDAG->getCopyToReg(Node->getOperand(0), dl, PtrReg, in Select()
5186 Node->getOperand(2), SDValue()); in Select()
5192 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX, Node->getOperand(3), in Select()
5195 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX, Node->getOperand(4), in Select()
5200 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, in Select()
5210 CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); in Select()
5211 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA); in Select()
5214 SDValue Base = Node->getOperand(4); in Select()
5216 SDValue Index = Node->getOperand(5); in Select()
5217 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32); in Select()
5218 SDValue Segment = CurDAG->getRegister(0, MVT::i16); in Select()
5219 SDValue Chain = Node->getOperand(0); in Select()
5221 SDValue Ops[] = {Node->getOperand(2), in Select()
5222 Node->getOperand(3), in Select()
5228 Node->getOperand(6), in Select()
5230 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in Select()
5237 if (!Subtarget->hasAMXTILE()) in Select()
5240 CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); in Select()
5241 MFI->setAMXProgModel(AMXProgModelEnum::DirectReg); in Select()
5250 unsigned TIndex = Node->getConstantOperandVal(2); in Select()
5252 SDValue Base = Node->getOperand(3); in Select()
5254 SDValue Index = Node->getOperand(4); in Select()
5255 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32); in Select()
5256 SDValue Segment = CurDAG->getRegister(0, MVT::i16); in Select()
5257 SDValue Chain = Node->getOperand(0); in Select()
5261 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in Select()
5264 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); in Select()
5274 if (Subtarget->isTargetNaCl()) in Select()
5278 if (Subtarget->isTarget64BitILP32()) { in Select()
5279 // Converts a 32-bit register to a 64-bit, zero-extended version of in Select()
5280 // it. This is needed because x86-64 can do many things, but jmp %r32 in Select()
5282 SDValue Target = Node->getOperand(1); in Select()
5283 assert(Target.getValueType() == MVT::i32 && "Unexpected VT!"); in Select()
5284 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, MVT::i64); in Select()
5285 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other, in Select()
5286 Node->getOperand(0), ZextTarget); in Select()
5299 // Just drop all 128/256/512-bit bitcasts. in Select()
5302 ReplaceUses(SDValue(Node, 0), Node->getOperand(0)); in Select()
5303 CurDAG->RemoveDeadNode(Node); in Select()
5319 uint8_t Imm = Node->getConstantOperandVal(3); in Select()
5320 if (matchVPTERNLOG(Node, Node, Node, Node, Node->getOperand(0), in Select()
5321 Node->getOperand(1), Node->getOperand(2), Imm)) in Select()
5334 SDValue N0 = Node->getOperand(0); in Select()
5335 SDValue N1 = Node->getOperand(1); in Select()
5346 CurDAG->RemoveDeadNode(Node); in Select()
5375 // unavailable to the fast-isel table. in Select()
5376 if (!CurDAG->shouldOptForSize()) in Select()
5383 SDValue N0 = Node->getOperand(0); in Select()
5384 SDValue N1 = Node->getOperand(1); in Select()
5390 int64_t Val = Cst->getSExtValue(); in Select()
5393 // FIXME: Handle unsigned 32 bit immediates for 64-bit AND. in Select()
5398 if (Opcode == ISD::ADD && (Val == 1 || Val == -1)) in Select()
5408 default: llvm_unreachable("Unexpected VT!"); in Select()
5518 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other); in Select()
5519 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
5522 // Record the mem-refs in Select()
5523 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N0)->getMemOperand()}); in Select()
5525 CurDAG->RemoveDeadNode(Node); in Select()
5530 CurDAG->SelectNodeTo(Node, ROpc, NVT, MVT::i32, N0, N1); in Select()
5540 SDValue N0 = Node->getOperand(0); in Select()
5541 SDValue N1 = Node->getOperand(1); in Select()
5545 default: llvm_unreachable("Unsupported VT!"); in Select()
5577 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, in Select()
5582 // i16/i32/i64 use an instruction that produces a low and high result even in Select()
5586 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other); in Select()
5588 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other); in Select()
5592 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
5596 // Record the mem-refs in Select()
5597 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); in Select()
5599 // i16/i32/i64 use an instruction that produces a low and high result even in Select()
5603 VTs = CurDAG->getVTList(NVT, MVT::i32); in Select()
5605 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); in Select()
5607 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue}); in Select()
5612 CurDAG->RemoveDeadNode(Node); in Select()
5618 SDValue N0 = Node->getOperand(0); in Select()
5619 SDValue N1 = Node->getOperand(1); in Select()
5624 bool UseMULX = !IsSigned && Subtarget->hasBMI2(); in Select()
5627 default: llvm_unreachable("Unsupported VT!"); in Select()
5663 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, in Select()
5672 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other); in Select()
5673 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
5677 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other); in Select()
5678 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
5683 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); in Select()
5684 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops); in Select()
5691 // Record the mem-refs in Select()
5692 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); in Select()
5696 SDVTList VTs = CurDAG->getVTList(NVT); in Select()
5697 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
5700 SDVTList VTs = CurDAG->getVTList(NVT, NVT); in Select()
5701 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
5705 SDVTList VTs = CurDAG->getVTList(MVT::Glue); in Select()
5706 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops); in Select()
5715 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, in Select()
5720 LLVM_DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG); in Select()
5723 // Copy the high half of the result, if it is needed. in Select()
5726 assert(HiReg && "Register for high half is not defined!"); in Select()
5727 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, in Select()
5732 LLVM_DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG); in Select()
5736 CurDAG->RemoveDeadNode(Node); in Select()
5742 SDValue N0 = Node->getOperand(0); in Select()
5743 SDValue N1 = Node->getOperand(1); in Select()
5749 default: llvm_unreachable("Unsupported VT!"); in Select()
5757 default: llvm_unreachable("Unsupported VT!"); in Select()
5768 default: llvm_unreachable("Unsupported VT!"); in Select()
5790 bool signBitIsZero = CurDAG->SignBitIsZero(N0); in Select()
5802 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops); in Select()
5805 // Record the mem-refs in Select()
5806 CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()}); in Select()
5810 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0); in Select()
5811 Chain = CurDAG->getEntryNode(); in Select()
5813 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, SDValue(Move, 0), in Select()
5818 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, in Select()
5821 // Sign extend the low part into the high part. in Select()
5823 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0); in Select()
5825 // Zero out the high part, effectively zero extending the input. in Select()
5826 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32); in Select()
5828 CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, std::nullopt), 0); in Select()
5832 SDValue(CurDAG->getMachineNode( in Select()
5834 CurDAG->getTargetConstant(X86::sub_16bit, dl, in Select()
5842 SDValue(CurDAG->getMachineNode( in Select()
5844 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode, in Select()
5845 CurDAG->getTargetConstant(X86::sub_32bit, dl, in Select()
5853 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, in Select()
5862 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops); in Select()
5866 // Record the mem-refs in Select()
5867 CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); in Select()
5870 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0); in Select()
5881 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8); in Select()
5885 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32, in Select()
5891 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result); in Select()
5894 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); in Select()
5899 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, in Select()
5903 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); in Select()
5906 // Copy the remainder (high) result, if it is needed. in Select()
5908 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, in Select()
5912 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); in Select()
5915 CurDAG->RemoveDeadNode(Node); in Select()
5922 bool IsStrictCmp = Node->getOpcode() == X86ISD::STRICT_FCMP || in Select()
5923 Node->getOpcode() == X86ISD::STRICT_FCMPS; in Select()
5924 SDValue N0 = Node->getOperand(IsStrictCmp ? 1 : 0); in Select()
5925 SDValue N1 = Node->getOperand(IsStrictCmp ? 2 : 1); in Select()
5927 // Save the original VT of the compare. in Select()
5931 if (Subtarget->canUseCMOV()) in Select()
5934 bool IsSignaling = Node->getOpcode() == X86ISD::STRICT_FCMPS; in Select()
5951 IsStrictCmp ? Node->getOperand(0) : CurDAG->getEntryNode(); in Select()
5954 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue); in Select()
5955 Chain = SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0); in Select()
5958 Glue = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0); in Select()
5963 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0); in Select()
5965 // Extract upper 8-bits of AX. in Select()
5967 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW); in Select()
5970 // Some 64-bit targets lack SAHF support, but they do support FCOMI. in Select()
5971 assert(Subtarget->canUseLAHFSAHF() && in Select()
5973 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract, SDValue()); in Select()
5976 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.getValue(1)), 0); in Select()
5982 CurDAG->RemoveDeadNode(Node); in Select()
5987 SDValue N0 = Node->getOperand(0); in Select()
5988 SDValue N1 = Node->getOperand(1); in Select()
5994 // Save the original VT of the compare. in Select()
6006 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR); in Select()
6008 CurDAG->RemoveDeadNode(Node); in Select()
6020 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && in Select()
6028 uint64_t Mask = MaskC->getZExtValue(); in Select()
6045 // eliminate a movabsq or shrink a 32-bit immediate to 8-bit without in Select()
6064 } else if (MaskC->hasOneUse() && !isInt<32>(Mask)) { in Select()
6065 // If the shifted mask extends into the high half and is 8/16/32 bits in Select()
6067 unsigned PopCount = 64 - LeadingZeros - TrailingZeros; in Select()
6089 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64); in Select()
6091 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32, in Select()
6096 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift); in Select()
6099 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift); in Select()
6105 MVT VT; in Select() local
6118 VT = MVT::i8; in Select()
6129 VT = MVT::i16; in Select()
6135 // Without minsize 16-bit Cmps can get here so we need to in Select()
6145 VT = MVT::i32; in Select()
6154 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT); in Select()
6162 if (!LoadN->isSimple()) { in Select()
6163 unsigned NumVolBits = LoadN->getValueType(0).getSizeInBits(); in Select()
6172 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops); in Select()
6175 // Record the mem-refs in Select()
6176 CurDAG->setNodeMemRefs(NewNode, in Select()
6177 {cast<LoadSDNode>(Reg)->getMemOperand()}); in Select()
6180 if (N0.getValueType() != VT) in Select()
6181 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg); in Select()
6183 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm); in Select()
6192 if (!Subtarget->hasSSE42()) in Select()
6203 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri; in Select()
6205 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi; in Select()
6211 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri; in Select()
6213 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi; in Select()
6220 CurDAG->RemoveDeadNode(Node); in Select()
6224 if (!Subtarget->hasSSE42()) in Select()
6228 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX, in Select()
6229 Node->getOperand(1), in Select()
6231 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX, in Select()
6232 Node->getOperand(3), InGlue).getValue(1); in Select()
6242 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri; in Select()
6244 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi; in Select()
6251 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri; in Select()
6253 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi; in Select()
6259 CurDAG->RemoveDeadNode(Node); in Select()
6276 MVT VT = Node->getSimpleValueType(0); in Select() local
6278 if (Subtarget->hasSBBDepBreaking()) { in Select()
6283 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS, in Select()
6284 Node->getOperand(1), SDValue()); in Select()
6286 // Create a 64-bit instruction if the result is 64-bits otherwise use the in Select()
6287 // 32-bit version. in Select()
6288 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r; in Select()
6289 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32; in Select()
6291 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.getValue(1)), in Select()
6295 // no-source idiom, so we explicitly zero the input values. in Select()
6299 // For less than 32-bits we need to extract from the 32-bit node. in Select()
6300 if (VT == MVT::i8 || VT == MVT::i16) { in Select()
6301 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit; in Select()
6302 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result); in Select()
6306 CurDAG->RemoveDeadNode(Node); in Select()
6310 if (isNullConstant(Node->getOperand(0)) && in Select()
6311 isNullConstant(Node->getOperand(1))) { in Select()
6319 // For less than 32-bits we need to extract from the 32-bit node. in Select()
6320 MVT VT = Node->getSimpleValueType(0); in Select() local
6321 if (VT == MVT::i8 || VT == MVT::i16) { in Select()
6322 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit; in Select()
6323 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result); in Select()
6328 CurDAG->RemoveDeadNode(Node); in Select()
6335 SDValue IndexOp = Mgt->getIndex(); in Select()
6336 SDValue Mask = Mgt->getMask(); in Select()
6338 MVT ValueVT = Node->getSimpleValueType(0); in Select()
6382 "Unexpected mask VT!"); in Select()
6405 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(), in Select()
6409 SDValue PassThru = Mgt->getPassThru(); in Select()
6410 SDValue Chain = Mgt->getChain(); in Select()
6412 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other); in Select()
6418 NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops); in Select()
6422 NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops); in Select()
6424 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()}); in Select()
6427 CurDAG->RemoveDeadNode(Node); in Select()
6432 SDValue Value = Sc->getValue(); in Select()
6433 SDValue IndexOp = Sc->getIndex(); in Select()
6478 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(), in Select()
6482 SDValue Mask = Sc->getMask(); in Select()
6483 SDValue Chain = Sc->getChain(); in Select()
6485 SDVTList VTs = CurDAG->getVTList(Mask.getValueType(), MVT::Other); in Select()
6488 MachineSDNode *NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops); in Select()
6489 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()}); in Select()
6491 CurDAG->RemoveDeadNode(Node); in Select()
6495 auto *MFI = CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); in Select()
6496 auto CallId = MFI->getPreallocatedIdForCallSite( in Select()
6497 cast<SrcValueSDNode>(Node->getOperand(1))->getValue()); in Select()
6498 SDValue Chain = Node->getOperand(0); in Select()
6499 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32); in Select()
6500 MachineSDNode *New = CurDAG->getMachineNode( in Select()
6503 CurDAG->RemoveDeadNode(Node); in Select()
6507 auto *MFI = CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>(); in Select()
6508 auto CallId = MFI->getPreallocatedIdForCallSite( in Select()
6509 cast<SrcValueSDNode>(Node->getOperand(1))->getValue()); in Select()
6510 SDValue Chain = Node->getOperand(0); in Select()
6511 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32); in Select()
6512 SDValue ArgIndex = Node->getOperand(2); in Select()
6517 MachineSDNode *New = CurDAG->getMachineNode( in Select()
6519 CurDAG->getVTList(TLI->getPointerTy(CurDAG->getDataLayout()), in Select()
6524 CurDAG->RemoveDeadNode(Node); in Select()
6531 if (!Subtarget->hasWIDEKL()) in Select()
6535 switch (Node->getOpcode()) { in Select()
6552 SDValue Chain = Node->getOperand(0); in Select()
6553 SDValue Addr = Node->getOperand(1); in Select()
6559 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(2), in Select()
6561 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(3), in Select()
6563 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2, Node->getOperand(4), in Select()
6565 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3, Node->getOperand(5), in Select()
6567 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4, Node->getOperand(6), in Select()
6569 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5, Node->getOperand(7), in Select()
6571 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6, Node->getOperand(8), in Select()
6573 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7, Node->getOperand(9), in Select()
6576 MachineSDNode *Res = CurDAG->getMachineNode( in Select()
6577 Opcode, dl, Node->getVTList(), in Select()
6579 CurDAG->setNodeMemRefs(Res, cast<MemSDNode>(Node)->getMemOperand()); in Select()
6617 /// This pass converts a legalized DAG into a X86-specific DAG,