Lines Matching +full:fiq +full:- +full:based
1 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
122 #define DEBUG_TYPE "arm-isel"
131 ARMInterworking("arm-interworking", cl::Hidden,
136 "arm-promote-constant", cl::Hidden,
141 "arm-promote-constant-max-size", cl::Hidden,
145 "arm-promote-constant-max-total", cl::Hidden,
150 MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
403 // MVE integer-only / float support. Only doing FP data processing on the FP in addMVEVectorTypes()
404 // vector types is inhibited at integer-only level. in addMVEVectorTypes()
499 RegInfo = Subtarget->getRegisterInfo(); in ARMTargetLowering()
500 Itins = Subtarget->getInstrItineraryData(); in ARMTargetLowering()
505 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() && in ARMTargetLowering()
506 !Subtarget->isTargetWatchOS() && !Subtarget->isTargetDriverKit()) { in ARMTargetLowering()
514 if (Subtarget->isTargetMachO()) { in ARMTargetLowering()
516 if (Subtarget->isThumb() && Subtarget->hasVFP2Base() && in ARMTargetLowering()
517 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) { in ARMTargetLowering()
523 // Single-precision floating-point arithmetic. in ARMTargetLowering()
529 // Double-precision floating-point arithmetic. in ARMTargetLowering()
535 // Single-precision comparisons. in ARMTargetLowering()
544 // Double-precision comparisons. in ARMTargetLowering()
553 // Floating-point to integer conversions. in ARMTargetLowering()
565 // Integer to floating-point conversions. in ARMTargetLowering()
585 if (Subtarget->isAAPCS_ABI() && in ARMTargetLowering()
586 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() || in ARMTargetLowering()
587 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) { in ARMTargetLowering()
594 // Double-precision floating-point arithmetic helper functions in ARMTargetLowering()
601 // Double-precision floating-point comparison helper functions in ARMTargetLowering()
611 // Single-precision floating-point arithmetic helper functions in ARMTargetLowering()
618 // Single-precision floating-point comparison helper functions in ARMTargetLowering()
628 // Floating-point to integer conversions. in ARMTargetLowering()
645 // Integer to floating-point conversions. in ARMTargetLowering()
707 if (Subtarget->isTargetWindows()) { in ARMTargetLowering()
729 // Use divmod compiler-rt calls for iOS 5.0 and later. in ARMTargetLowering()
730 if (Subtarget->isTargetMachO() && in ARMTargetLowering()
731 !(Subtarget->isTargetIOS() && in ARMTargetLowering()
732 Subtarget->getTargetTriple().isOSVersionLT(5, 0))) { in ARMTargetLowering()
737 // The half <-> float conversion functions are always soft-float on in ARMTargetLowering()
738 // non-watchos platforms, but are needed for some targets which use a in ARMTargetLowering()
739 // hard-float calling convention by default. in ARMTargetLowering()
740 if (!Subtarget->isTargetWatchABI()) { in ARMTargetLowering()
741 if (Subtarget->isAAPCS_ABI()) { in ARMTargetLowering()
754 if (Subtarget->isTargetAEABI()) { in ARMTargetLowering()
771 if (Subtarget->isThumb1Only()) in ARMTargetLowering()
776 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() && in ARMTargetLowering()
777 Subtarget->hasFPRegs()) { in ARMTargetLowering()
786 if (!Subtarget->hasVFP2Base()) in ARMTargetLowering()
788 if (!Subtarget->hasFP64()) in ARMTargetLowering()
792 if (Subtarget->hasFullFP16()) { in ARMTargetLowering()
801 if (Subtarget->hasBF16()) { in ARMTargetLowering()
804 if (!Subtarget->hasFullFP16()) in ARMTargetLowering()
826 if (Subtarget->hasMVEIntegerOps()) in ARMTargetLowering()
827 addMVEVectorTypes(Subtarget->hasMVEFloatOps()); in ARMTargetLowering()
829 // Combine low-overhead loop intrinsics so that we can lower i1 types. in ARMTargetLowering()
830 if (Subtarget->hasLOB()) { in ARMTargetLowering()
834 if (Subtarget->hasNEON()) { in ARMTargetLowering()
848 if (Subtarget->hasFullFP16()) { in ARMTargetLowering()
853 if (Subtarget->hasBF16()) { in ARMTargetLowering()
859 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) { in ARMTargetLowering()
899 if (Subtarget->hasNEON()) { in ARMTargetLowering()
939 // Custom handling for some quad-vector types to detect VMULL. in ARMTargetLowering()
965 // types wider than 8-bits. However, custom lowering can leverage the in ARMTargetLowering()
1004 if (!Subtarget->hasVFP4Base()) { in ARMTargetLowering()
1031 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) { in ARMTargetLowering()
1039 if (Subtarget->hasMVEIntegerOps()) { in ARMTargetLowering()
1044 if (Subtarget->hasMVEFloatOps()) { in ARMTargetLowering()
1048 if (!Subtarget->hasFP64()) { in ARMTargetLowering()
1049 // When targeting a floating-point unit with only single-precision in ARMTargetLowering()
1050 // operations, f64 is legal for the few double-precision instructions which in ARMTargetLowering()
1051 // are present However, no double-precision operations other than moves, in ARMTargetLowering()
1092 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) { in ARMTargetLowering()
1095 if (Subtarget->hasFullFP16()) { in ARMTargetLowering()
1101 if (!Subtarget->hasFP16()) { in ARMTargetLowering()
1106 computeRegisterProperties(Subtarget->getRegisterInfo()); in ARMTargetLowering()
1108 // ARM does not have floating-point extending loads. in ARMTargetLowering()
1124 if (!Subtarget->isThumb1Only()) { in ARMTargetLowering()
1137 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}. in ARMTargetLowering()
1149 if (Subtarget->hasDSP()) { in ARMTargetLowering()
1159 if (Subtarget->hasBaseDSP()) { in ARMTargetLowering()
1167 if (Subtarget->isThumb1Only()) { in ARMTargetLowering()
1171 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops() in ARMTargetLowering()
1172 || (Subtarget->isThumb2() && !Subtarget->hasDSP())) in ARMTargetLowering()
1187 if (Subtarget->hasMVEIntegerOps()) in ARMTargetLowering()
1191 if (Subtarget->isThumb1Only()) { in ARMTargetLowering()
1197 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) in ARMTargetLowering()
1208 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) { in ARMTargetLowering()
1216 // implementation-specific ways of obtaining this information. in ARMTargetLowering()
1217 if (Subtarget->hasPerfMon()) in ARMTargetLowering()
1221 if (!Subtarget->hasV6Ops()) in ARMTargetLowering()
1224 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode() in ARMTargetLowering()
1225 : Subtarget->hasDivideInARMMode(); in ARMTargetLowering()
1232 if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) { in ARMTargetLowering()
1243 // Register based DivRem for AEABI (RTABI 4.2) in ARMTargetLowering()
1244 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || in ARMTargetLowering()
1245 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || in ARMTargetLowering()
1246 Subtarget->isTargetWindows()) { in ARMTargetLowering()
1251 if (Subtarget->isTargetWindows()) { in ARMTargetLowering()
1320 if (Subtarget->isTargetWindows()) in ARMTargetLowering()
1328 if (Subtarget->hasAnyDataBarrier() && in ARMTargetLowering()
1329 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) { in ARMTargetLowering()
1333 if (!Subtarget->isThumb() || !Subtarget->isMClass()) in ARMTargetLowering()
1338 if (!Subtarget->hasAcquireRelease() || in ARMTargetLowering()
1347 if (Subtarget->hasDataBarrier()) in ARMTargetLowering()
1351 Subtarget->hasAnyDataBarrier() ? Custom : Expand); in ARMTargetLowering()
1375 if (Subtarget->isTargetLinux() || in ARMTargetLowering()
1376 (!Subtarget->isMClass() && Subtarget->hasV6Ops())) { in ARMTargetLowering()
1380 // ARM Linux always supports 64-bit atomics through kernel-assisted atomic in ARMTargetLowering()
1381 // routines (kernel 3.1 or later). FIXME: Not with compiler-rt? in ARMTargetLowering()
1388 } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) || in ARMTargetLowering()
1389 Subtarget->hasForced32BitAtomics()) { in ARMTargetLowering()
1390 // Cortex-M (besides Cortex-M0) have 32-bit atomics. in ARMTargetLowering()
1403 if (!Subtarget->hasV6Ops()) { in ARMTargetLowering()
1409 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() && in ARMTargetLowering()
1410 !Subtarget->isThumb1Only()) { in ARMTargetLowering()
1411 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR in ARMTargetLowering()
1429 if (Subtarget->useSjLjEH()) in ARMTargetLowering()
1441 if (Subtarget->hasFullFP16()) { in ARMTargetLowering()
1451 if (Subtarget->hasFullFP16()) in ARMTargetLowering()
1466 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() && in ARMTargetLowering()
1467 !Subtarget->isThumb1Only()) { in ARMTargetLowering()
1474 if (!Subtarget->hasVFP4Base()) { in ARMTargetLowering()
1480 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) { in ARMTargetLowering()
1481 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded. in ARMTargetLowering()
1482 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) { in ARMTargetLowering()
1487 // fp16 is a special v7 extension that adds f16 <-> f32 conversions. in ARMTargetLowering()
1488 if (!Subtarget->hasFP16()) { in ARMTargetLowering()
1493 // Strict floating-point comparisons need custom lowering. in ARMTargetLowering()
1509 // FP-ARMv8 implements a lot of rounding-like FP operations. in ARMTargetLowering()
1510 if (Subtarget->hasFPARMv8Base()) { in ARMTargetLowering()
1519 if (Subtarget->hasNEON()) { in ARMTargetLowering()
1526 if (Subtarget->hasFP64()) { in ARMTargetLowering()
1539 if (Subtarget->hasFullFP16()) { in ARMTargetLowering()
1558 if (Subtarget->hasNEON()) { in ARMTargetLowering()
1570 if (Subtarget->hasFullFP16()) { in ARMTargetLowering()
1583 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has in ARMTargetLowering()
1585 if (Subtarget->isTargetWindows()) { in ARMTargetLowering()
1597 // We have target-specific dag combine patterns for the following nodes: in ARMTargetLowering()
1598 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine in ARMTargetLowering()
1602 if (Subtarget->hasMVEIntegerOps()) in ARMTargetLowering()
1605 if (Subtarget->hasV6Ops()) in ARMTargetLowering()
1607 if (Subtarget->isThumb1Only()) in ARMTargetLowering()
1610 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || in ARMTargetLowering()
1611 Subtarget->isThumb2()) { in ARMTargetLowering()
1617 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() || in ARMTargetLowering()
1618 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize()) in ARMTargetLowering()
1623 //// temporary - rewrite interface to use type in ARMTargetLowering()
1626 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores in ARMTargetLowering()
1628 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores in ARMTargetLowering()
1635 // Prefer likely predicted branches to selects on out-of-order cores. in ARMTargetLowering()
1636 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder(); in ARMTargetLowering()
1638 setPrefLoopAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment())); in ARMTargetLowering()
1639 setPrefFunctionAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment())); in ARMTargetLowering()
1641 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4)); in ARMTargetLowering()
1645 return Subtarget->useSoftFloat(); in useSoftFloat()
1649 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1650 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1674 // to the VFP2 class (D0-D15). We currently model this constraint prior to in findRepresentativeClass()
1675 // coalescing by double-counting the SP regs. See the FIXME above. in findRepresentativeClass()
1676 if (Subtarget->useNEONForSinglePrecisionFP()) in findRepresentativeClass()
1916 if ((Subtarget->hasMVEIntegerOps() && in getSetCCResultType()
1919 (Subtarget->hasMVEFloatOps() && in getSetCCResultType()
1925 /// getRegClassFor - Return the register class that should be used for the
1934 if (Subtarget->hasNEON()) { in getRegClassFor()
1940 if (Subtarget->hasMVEIntegerOps()) { in getRegClassFor()
1957 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1 in shouldAlignPointerArgs()
1958 // cycle faster than 4-byte aligned LDM. in shouldAlignPointerArgs()
1960 (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4)); in shouldAlignPointerArgs()
1972 unsigned NumVals = N->getNumValues(); in getSchedulingPreference()
1977 EVT VT = N->getValueType(i); in getSchedulingPreference()
1984 if (!N->isMachineOpcode()) in getSchedulingPreference()
1989 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in getSchedulingPreference()
1990 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); in getSchedulingPreference()
1994 if (!Itins->isEmpty() && in getSchedulingPreference()
1995 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2U) in getSchedulingPreference()
2001 //===----------------------------------------------------------------------===//
2003 //===----------------------------------------------------------------------===//
2009 return Const->getZExtValue() == 16; in isSRL16()
2017 return Const->getZExtValue() == 16; in isSRA16()
2025 return Const->getZExtValue() == 16; in isSHL16()
2029 // Check for a signed 16-bit value. We special case SRA because it makes it
2039 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
2056 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
2085 //===----------------------------------------------------------------------===//
2087 //===----------------------------------------------------------------------===//
2089 /// getEffectiveCallingConv - Get the effective calling convention, taking into
2113 if (!Subtarget->isAAPCS_ABI()) in getEffectiveCallingConv()
2115 else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && in getEffectiveCallingConv()
2123 if (!Subtarget->isAAPCS_ABI()) { in getEffectiveCallingConv()
2124 if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() && !isVarArg) in getEffectiveCallingConv()
2127 } else if (Subtarget->hasVFP2Base() && in getEffectiveCallingConv()
2128 !Subtarget->isThumb1Only() && !isVarArg) in getEffectiveCallingConv()
2145 /// CCAssignFnForNode - Selects the correct CCAssignFn for the given
2176 if (Subtarget->hasFullFP16()) { in MoveToHPR()
2189 if (Subtarget->hasFullFP16()) { in MoveFromHPR()
2201 /// LowerCallResult - Lower the result values of a call into the
2240 if (!Subtarget->isLittle()) in LowerCallResult()
2257 if (!Subtarget->isLittle()) in LowerCallResult()
2279 // had been copied to the LSBs of a 32-bit register. in LowerCallResult()
2285 // On CMSE Non-secure Calls, call results (returned values) whose bitwidth in LowerCallResult()
2286 // is less than 32 bits must be sign- or zero-extended after the call for in LowerCallResult()
2337 unsigned id = Subtarget->isLittle() ? 0 : 1; in PassF64ArgInRegs()
2341 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id))); in PassF64ArgInRegs()
2353 DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo)); in PassF64ArgInRegs()
2362 /// LowerCall - Lowering a call into a callseq_start <-
2363 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2396 // Lower 'returns_twice' calls to a pseudo-instruction. in LowerCall()
2397 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) && in LowerCall()
2398 !Subtarget->noBTIAtReturnTwice()) in LowerCall()
2399 GuardWithBTI = AFI->branchTargetEnforcement(); in LowerCall()
2401 // Determine whether this is a non-secure function call. in LowerCall()
2402 if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call")) in LowerCall()
2406 if (!Subtarget->supportsTailCall()) in LowerCall()
2409 // For both the non-secure calls and the returns from a CMSE entry function, in LowerCall()
2412 if (isCmseNSCall || AFI->isCmseNSEntryFunction()) in LowerCall()
2418 // as BLXr has a 16-bit encoding. in LowerCall()
2419 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal(); in LowerCall()
2421 auto *BB = CLI.CB->getParent(); in LowerCall()
2422 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() && in LowerCall()
2423 count_if(GV->users(), [&BB](const User *U) { in LowerCall()
2425 cast<Instruction>(U)->getParent() == BB; in LowerCall()
2444 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall()) in LowerCall()
2455 // arguments to begin at SP+0. Completely unused for non-tail calls. in LowerCall()
2460 unsigned NumReusableBytes = FuncInfo->getArgumentStackSize(); in LowerCall()
2463 // popped size 16-byte aligned. in LowerCall()
2470 SPDiff = NumReusableBytes - NumBytes; in LowerCall()
2474 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff) in LowerCall()
2475 AFI->setArgRegsSaveSize(-SPDiff); in LowerCall()
2534 // had been copied to the LSBs of a 32-bit register. in LowerCall()
2621 offset = RegEnd - RegBegin; in LowerCall()
2634 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl, in LowerCall()
2659 // Build a sequence of copy-to-reg nodes chained together with token chain in LowerCall()
2676 GVal = G->getGlobal(); in LowerCall()
2677 bool isStub = !TM.shouldAssumeDSOLocal(GVal) && Subtarget->isTargetMachO(); in LowerCall()
2679 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass()); in LowerCall()
2683 if (Subtarget->genLongCalls()) { in LowerCall()
2684 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) && in LowerCall()
2685 "long-calls codegen is not position independent!"); in LowerCall()
2690 if (Subtarget->genExecuteOnly()) { in LowerCall()
2691 if (Subtarget->useMovt()) in LowerCall()
2697 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerCall()
2709 const char *Sym = S->getSymbol(); in LowerCall()
2711 if (Subtarget->genExecuteOnly()) { in LowerCall()
2712 if (Subtarget->useMovt()) in LowerCall()
2718 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerCall()
2733 bool isDef = GVal->isStrongDefinitionForLinker(); in LowerCall()
2736 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking); in LowerCall()
2738 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { in LowerCall()
2739 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?"); in LowerCall()
2748 } else if (Subtarget->isTargetCOFF()) { in LowerCall()
2749 assert(Subtarget->isTargetWindows() && in LowerCall()
2752 if (GVal->hasDLLImportStorageClass()) in LowerCall()
2770 const char *Sym = S->getSymbol(); in LowerCall()
2771 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { in LowerCall()
2772 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerCall()
2793 "call to non-secure function would " in LowerCall()
2796 DAG.getContext()->diagnose(Diag); in LowerCall()
2801 "call to non-secure function would return value through pointer", in LowerCall()
2803 DAG.getContext()->diagnose(Diag); in LowerCall()
2809 if (Subtarget->isThumb()) { in LowerCall()
2814 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps()) in LowerCall()
2819 if (!isDirect && !Subtarget->hasV5TOps()) in LowerCall()
2821 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() && in LowerCall()
2823 !Subtarget->hasMinSize()) in LowerCall()
2830 // We don't usually want to end the call-sequence here because we would tidy in LowerCall()
2831 // the frame up *after* the call, however in the ABI-changing tail-call case in LowerCall()
2853 // Add a register mask operand representing the call-preserved registers. in LowerCall()
2855 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo(); in LowerCall()
2857 // For 'this' returns, use the R0-preserving mask if applicable in LowerCall()
2858 Mask = ARI->getThisReturnPreservedMask(MF, CallConv); in LowerCall()
2864 Mask = ARI->getCallPreservedMask(MF, CallConv); in LowerCall()
2867 Mask = ARI->getCallPreservedMask(MF, CallConv); in LowerCall()
2890 // If we're guaranteeing tail-calls will be honoured, the callee must in LowerCall()
2892 // we need to undo that after it returns to restore the status-quo. in LowerCall()
2895 canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1ULL; in LowerCall()
2908 /// HandleByVal - Every parameter *after* a byval parameter is passed
2917 unsigned Reg = State->AllocateReg(GPRArgRegs); in HandleByVal()
2922 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs; in HandleByVal()
2924 Reg = State->AllocateReg(GPRArgRegs); in HandleByVal()
2929 unsigned Excess = 4 * (ARM::R4 - Reg); in HandleByVal()
2935 const unsigned NSAAOffset = State->getStackSize(); in HandleByVal()
2937 while (State->AllocateReg(GPRArgRegs)) in HandleByVal()
2945 // the end (first after last) register would be reg + param-size-in-regs, in HandleByVal()
2950 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd); in HandleByVal()
2954 State->AllocateReg(GPRArgRegs); in HandleByVal()
2959 Size = std::max<int>(Size - Excess, 0); in HandleByVal()
2962 /// MatchingStackOffset - Return true if the given stack call argument is
2972 Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); in MatchingStackOffset()
2975 MachineInstr *Def = MRI->getVRegDef(VR); in MatchingStackOffset()
2979 if (!TII->isLoadFromStackSlot(*Def, FI)) in MatchingStackOffset()
2992 SDValue Ptr = Ld->getBasePtr(); in MatchingStackOffset()
2996 FI = FINode->getIndex(); in MatchingStackOffset()
3006 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
3025 assert(Subtarget->supportsTailCall()); in IsEligibleForTailCallOptimization()
3028 // to the call take up r0-r3. The reason is that there are no legal registers in IsEligibleForTailCallOptimization()
3035 if (Subtarget->isThumb1Only()) in IsEligibleForTailCallOptimization()
3038 if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true)) in IsEligibleForTailCallOptimization()
3045 // Exception-handling functions need a special set of instructions to indicate in IsEligibleForTailCallOptimization()
3046 // a return to the hardware. Tail-calling another function would probably in IsEligibleForTailCallOptimization()
3061 // Externally-defined functions with weak linkage should not be in IsEligibleForTailCallOptimization()
3062 // tail-called on ARM when the OS does not support dynamic in IsEligibleForTailCallOptimization()
3063 // pre-emption of symbols, as the AAELF spec requires normal calls in IsEligibleForTailCallOptimization()
3066 // situation (as used for tail calls) is implementation-defined, so we in IsEligibleForTailCallOptimization()
3069 const GlobalValue *GV = G->getGlobal(); in IsEligibleForTailCallOptimization()
3071 if (GV->hasExternalWeakLinkage() && in IsEligibleForTailCallOptimization()
3085 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); in IsEligibleForTailCallOptimization()
3086 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); in IsEligibleForTailCallOptimization()
3088 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); in IsEligibleForTailCallOptimization()
3089 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) in IsEligibleForTailCallOptimization()
3097 if (AFI_Caller->getArgRegsSaveSize()) in IsEligibleForTailCallOptimization()
3108 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in IsEligibleForTailCallOptimization()
3120 // register/stack-slot combinations. The types will not match in IsEligibleForTailCallOptimization()
3169 // IRQ/FIQ: +4 "subs pc, lr, #4" in LowerInterruptReturn()
3177 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" || in LowerInterruptReturn()
3184 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF"); in LowerInterruptReturn()
3198 // CCValAssign - represent the assignment of the return value to a location. in LowerReturn()
3201 // CCState - Info about the registers and stack slots. in LowerReturn()
3211 bool isLittleEndian = Subtarget->isLittle(); in LowerReturn()
3215 AFI->setReturnRegsCount(RVLocs.size()); in LowerReturn()
3218 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) { in LowerReturn()
3225 DAG.getContext()->diagnose(Diag); in LowerReturn()
3238 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) { in LowerReturn()
3239 // Half-precision return values can be returned like this: in LowerReturn()
3272 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) { in LowerReturn()
3312 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is in LowerReturn()
3332 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); in LowerReturn()
3334 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); in LowerReturn()
3351 // CPUs which aren't M-class use a special sequence to return from in LowerReturn()
3355 // M-class CPUs actually use a normal return sequence with a special in LowerReturn()
3356 // (hardware-provided) value in LR, so the normal code path works. in LowerReturn()
3358 !Subtarget->isMClass()) { in LowerReturn()
3359 if (Subtarget->isThumb1Only()) in LowerReturn()
3364 ARMISD::NodeType RetNode = AFI->isCmseNSEntryFunction() ? ARMISD::SERET_GLUE : in LowerReturn()
3370 if (N->getNumValues() != 1) in isUsedByReturnOnly()
3372 if (!N->hasNUsesOfValue(1, 0)) in isUsedByReturnOnly()
3376 SDNode *Copy = *N->use_begin(); in isUsedByReturnOnly()
3377 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
3380 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
3382 TCChain = Copy->getOperand(0); in isUsedByReturnOnly()
3383 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) { in isUsedByReturnOnly()
3387 for (SDNode *U : VMov->uses()) { in isUsedByReturnOnly()
3388 if (U->getOpcode() != ISD::CopyToReg) in isUsedByReturnOnly()
3395 for (SDNode *U : VMov->uses()) { in isUsedByReturnOnly()
3396 SDValue UseChain = U->getOperand(0); in isUsedByReturnOnly()
3404 if (U->getOperand(U->getNumOperands() - 1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
3410 } else if (Copy->getOpcode() == ISD::BITCAST) { in isUsedByReturnOnly()
3412 if (!Copy->hasOneUse()) in isUsedByReturnOnly()
3414 Copy = *Copy->use_begin(); in isUsedByReturnOnly()
3415 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0)) in isUsedByReturnOnly()
3419 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
3421 TCChain = Copy->getOperand(0); in isUsedByReturnOnly()
3427 for (const SDNode *U : Copy->uses()) { in isUsedByReturnOnly()
3428 if (U->getOpcode() != ARMISD::RET_GLUE && in isUsedByReturnOnly()
3429 U->getOpcode() != ARMISD::INTRET_GLUE) in isUsedByReturnOnly()
3442 if (!Subtarget->supportsTailCall()) in mayBeEmittedAsTailCall()
3445 if (!CI->isTailCall()) in mayBeEmittedAsTailCall()
3455 SDValue WriteValue = Op->getOperand(2); in LowerWRITE_REGISTER()
3459 && "LowerWRITE_REGISTER called for non-i64 type argument."); in LowerWRITE_REGISTER()
3463 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi }; in LowerWRITE_REGISTER()
3481 // When generating execute-only code Constant Pools must be promoted to the in LowerConstantPool()
3483 // blocks, but this way we guarantee that execute-only behaves correct with in LowerConstantPool()
3484 // position-independent addressing modes. in LowerConstantPool()
3485 if (Subtarget->genExecuteOnly()) { in LowerConstantPool()
3487 auto T = const_cast<Type*>(CP->getType()); in LowerConstantPool()
3488 auto C = const_cast<Constant*>(CP->getConstVal()); in LowerConstantPool()
3495 Twine(AFI->createPICLabelUId()) in LowerConstantPool()
3502 // The 16-bit ADR instruction can only encode offsets that are multiples of 4, in LowerConstantPool()
3503 // so we need to align to at least 4 bytes when we don't have 32-bit ADR. in LowerConstantPool()
3504 Align CPAlign = CP->getAlign(); in LowerConstantPool()
3505 if (Subtarget->isThumb1Only()) in LowerConstantPool()
3507 if (CP->isMachineConstantPoolEntry()) in LowerConstantPool()
3509 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CPAlign); in LowerConstantPool()
3511 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CPAlign); in LowerConstantPool()
3516 // If we don't have a 32-bit pc-relative branch instruction then the jump in getJumpTableEncoding()
3518 // execute-only it must be placed out-of-line. in getJumpTableEncoding()
3519 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps()) in getJumpTableEncoding()
3531 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); in LowerBlockAddress()
3533 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI(); in LowerBlockAddress()
3537 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerBlockAddress()
3538 ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerBlockAddress()
3581 assert(Subtarget->isTargetDarwin() && in LowerGlobalTLSAddressDarwin()
3607 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo(); in LowerGlobalTLSAddressDarwin()
3609 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction()); in LowerGlobalTLSAddressDarwin()
3625 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering"); in LowerGlobalTLSAddressWindows()
3668 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL); in LowerGlobalTLSAddressWindows()
3684 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerToTLSGeneralDynamicModel()
3687 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerToTLSGeneralDynamicModel()
3689 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, in LowerToTLSGeneralDynamicModel()
3724 const GlobalValue *GV = GA->getGlobal(); in LowerToTLSExecModels()
3735 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerToTLSExecModels()
3737 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerToTLSExecModels()
3739 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, in LowerToTLSExecModels()
3778 if (Subtarget->isTargetDarwin()) in LowerGlobalTLSAddress()
3781 if (Subtarget->isTargetWindows()) in LowerGlobalTLSAddress()
3785 assert(Subtarget->isTargetELF() && "Only ELF implemented here"); in LowerGlobalTLSAddress()
3786 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal()); in LowerGlobalTLSAddress()
3802 SmallVector<const User*,4> Worklist(V->users()); in allUsersAreInFunction()
3806 append_range(Worklist, U->users()); in allUsersAreInFunction()
3811 if (!I || I->getParent()->getParent() != F) in allUsersAreInFunction()
3830 // use-site. We know that if we inline a variable at one use site, we'll in promoteToConstantPool()
3831 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel in promoteToConstantPool()
3834 // the GV from fast-isel generated code. in promoteToConstantPool()
3840 if (!GVar || !GVar->hasInitializer() || in promoteToConstantPool()
3841 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() || in promoteToConstantPool()
3842 !GVar->hasLocalLinkage()) in promoteToConstantPool()
3846 // from .data to .text. This is not allowed in position-independent code. in promoteToConstantPool()
3847 auto *Init = GVar->getInitializer(); in promoteToConstantPool()
3848 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) && in promoteToConstantPool()
3849 Init->needsDynamicRelocation()) in promoteToConstantPool()
3859 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType()); in promoteToConstantPool()
3861 unsigned RequiredPadding = 4 - (Size % 4); in promoteToConstantPool()
3863 RequiredPadding == 4 || (CDAInit && CDAInit->isString()); in promoteToConstantPool()
3876 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4) in promoteToConstantPool()
3877 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >= in promoteToConstantPool()
3893 StringRef S = CDAInit->getAsString(); in promoteToConstantPool()
3897 while (RequiredPadding--) in promoteToConstantPool()
3904 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) { in promoteToConstantPool()
3905 AFI->markGlobalAsPromotedToConstantPool(GVar); in promoteToConstantPool()
3906 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() + in promoteToConstantPool()
3907 PaddedSize - 4); in promoteToConstantPool()
3915 if (!(GV = GA->getAliaseeObject())) in isReadOnly()
3918 return V->isConstant(); in isReadOnly()
3924 switch (Subtarget->getTargetTriple().getObjectFormat()) { in LowerGlobalAddress()
3939 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); in LowerGlobalAddressELF()
3943 if (GV->isDSOLocal() && !Subtarget->genExecuteOnly()) in LowerGlobalAddressELF()
3949 GV, dl, PtrVT, 0, GV->isDSOLocal() ? 0 : ARMII::MO_GOT); in LowerGlobalAddressELF()
3951 if (!GV->isDSOLocal()) in LowerGlobalAddressELF()
3956 } else if (Subtarget->isROPI() && IsRO) { in LowerGlobalAddressELF()
3957 // PC-relative. in LowerGlobalAddressELF()
3961 } else if (Subtarget->isRWPI() && !IsRO) { in LowerGlobalAddressELF()
3962 // SB-relative. in LowerGlobalAddressELF()
3964 if (Subtarget->useMovt()) { in LowerGlobalAddressELF()
3986 if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) { in LowerGlobalAddressELF()
3987 if (Subtarget->useMovt()) in LowerGlobalAddressELF()
4004 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && in LowerGlobalAddressDarwin()
4008 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); in LowerGlobalAddressDarwin()
4010 if (Subtarget->useMovt()) in LowerGlobalAddressDarwin()
4021 if (Subtarget->isGVIndirectSymbol(GV)) in LowerGlobalAddressDarwin()
4029 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported"); in LowerGlobalAddressWindows()
4030 assert(Subtarget->useMovt() && in LowerGlobalAddressWindows()
4032 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && in LowerGlobalAddressWindows()
4036 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); in LowerGlobalAddressWindows()
4038 if (GV->hasDLLImportStorageClass()) in LowerGlobalAddressWindows()
4095 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo(); in LowerINTRINSIC_VOID()
4097 ARI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C); in LowerINTRINSIC_VOID()
4099 // Mark LR an implicit live-in. in LowerINTRINSIC_VOID()
4107 if (Subtarget->isThumb()) in LowerINTRINSIC_VOID()
4176 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerINTRINSIC_WO_CHAIN()
4180 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0; in LowerINTRINSIC_WO_CHAIN()
4258 return DAG.getNode(ARMISD::LSLL, SDLoc(Op), Op->getVTList(), in LowerINTRINSIC_WO_CHAIN()
4261 return DAG.getNode(ARMISD::ASRL, SDLoc(Op), Op->getVTList(), in LowerINTRINSIC_WO_CHAIN()
4273 if (!Subtarget->hasDataBarrier()) { in LowerATOMIC_FENCE()
4275 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get in LowerATOMIC_FENCE()
4277 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() && in LowerATOMIC_FENCE()
4286 if (Subtarget->isMClass()) { in LowerATOMIC_FENCE()
4287 // Only a full system barrier exists in the M-class architectures. in LowerATOMIC_FENCE()
4289 } else if (Subtarget->preferISHSTBarriers() && in LowerATOMIC_FENCE()
4305 if (!(Subtarget->isThumb2() || in LowerPREFETCH()
4306 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps()))) in LowerPREFETCH()
4313 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension())) in LowerPREFETCH()
4318 if (Subtarget->isThumb()) { in LowerPREFETCH()
4337 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); in LowerVASTART()
4338 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); in LowerVASTART()
4352 if (AFI->isThumb1OnlyFunction()) in GetF64FormalArgument()
4375 if (!Subtarget->isLittle()) in GetF64FormalArgument()
4380 // The remaining GPRs hold either the beginning of variable-argument
4393 // Currently, two use-cases possible: in StoreByValRegs()
4394 // Case #1. Non-var-args function, and we meet first byval parameter. in StoreByValRegs()
4399 // "store-reg" instructions. in StoreByValRegs()
4400 // Case #2. Var-args function, that doesn't contain byval parameters. in StoreByValRegs()
4417 ArgOffset = -4 * (ARM::R4 - RBegin); in StoreByValRegs()
4425 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass; in StoreByValRegs()
4458 AFI->setVarArgsFrameIndex(FrameIndex); in VarArgStyleRegisters()
4514 AFI->setArgRegsSaveSize(0); in LowerFormalArguments()
4540 int lastInsIndex = -1; in LowerFormalArguments()
4547 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin); in LowerFormalArguments()
4548 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize); in LowerFormalArguments()
4555 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx); in LowerFormalArguments()
4600 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass in LowerFormalArguments()
4612 AFI->setPreservesR0(); in LowerFormalArguments()
4616 // If this is an 8 or 16-bit value, it is really passed promoted in LowerFormalArguments()
4628 // had been copied to the LSBs of a 32-bit register. in LowerFormalArguments()
4635 // less than 32 bits must be sign- or zero-extended in the callee for in LowerFormalArguments()
4639 if (AFI->isCmseNSEntryFunction() && Arg.ArgVT.isScalarInteger() && in LowerFormalArguments()
4691 if (AFI->isCmseNSEntryFunction()) { in LowerFormalArguments()
4695 DAG.getContext()->diagnose(Diag); in LowerFormalArguments()
4707 AFI->setArgumentStackToRestore(StackArgSize); in LowerFormalArguments()
4709 AFI->setArgumentStackSize(StackArgSize); in LowerFormalArguments()
4711 if (CCInfo.getStackSize() > 0 && AFI->isCmseNSEntryFunction()) { in LowerFormalArguments()
4715 DAG.getContext()->diagnose(Diag); in LowerFormalArguments()
4721 /// isFloatingPointZero - Return true if this is +0.0.
4724 return CFP->getValueAPF().isPosZero(); in isFloatingPointZero()
4730 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) in isFloatingPointZero()
4731 return CFP->getValueAPF().isPosZero(); in isFloatingPointZero()
4733 } else if (Op->getOpcode() == ISD::BITCAST && in isFloatingPointZero()
4734 Op->getValueType(0) == MVT::f64) { in isFloatingPointZero()
4737 SDValue BitcastOp = Op->getOperand(0); in isFloatingPointZero()
4738 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM && in isFloatingPointZero()
4739 isNullConstant(BitcastOp->getOperand(0))) in isFloatingPointZero()
4751 unsigned C = RHSC->getZExtValue(); in getARMCmp()
4758 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) { in getARMCmp()
4760 RHS = DAG.getConstant(C - 1, dl, MVT::i32); in getARMCmp()
4765 if (C != 0 && isLegalICmpImmediate(C-1)) { in getARMCmp()
4767 RHS = DAG.getConstant(C - 1, dl, MVT::i32); in getARMCmp()
4788 // In ARM and Thumb-2, the compare instructions can shift their second in getARMCmp()
4815 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND && in getARMCmp()
4816 LHS->hasOneUse() && isa<ConstantSDNode>(LHS.getOperand(1)) && in getARMCmp()
4821 uint64_t RHSV = RHSC->getZExtValue(); in getARMCmp()
4836 // some tweaks to the heuristics for the previous and->shift transform. in getARMCmp()
4838 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL && in getARMCmp()
4839 isa<ConstantSDNode>(RHS) && RHS->getAsZExtVal() == 0x80000000U && in getARMCmp()
4889 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64); in getVFPCmp()
4900 /// duplicateCmp - Glue values can have only one use, so this function
5069 // value. So compute 1 - C. in LowerUnsignedALUO()
5082 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->isThumb1Only()) in LowerADDSUBSAT()
5092 switch (Op->getOpcode()) { in LowerADDSUBSAT()
5108 switch (Op->getOpcode()) { in LowerADDSUBSAT()
5128 DAG.getSExtOrTrunc(Op->getOperand(0), dl, MVT::i32), in LowerADDSUBSAT()
5129 DAG.getSExtOrTrunc(Op->getOperand(1), dl, MVT::i32)); in LowerADDSUBSAT()
5143 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) in LowerSELECT()
5158 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond) in LowerSELECT()
5159 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond) in LowerSELECT()
5168 unsigned CMOVTrueVal = CMOVTrue->getZExtValue(); in LowerSELECT()
5169 unsigned CMOVFalseVal = CMOVFalse->getZExtValue(); in LowerSELECT()
5193 // undefined bits before doing a full-word comparison with zero. in LowerSELECT()
5255 if (!Subtarget->hasFP64() && VT == MVT::f64) { in getCMOV()
5286 // See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5306 // x < -k ? -k : (x > k ? k : x)
5307 // x < -k ? -k : (x < k ? x : k)
5308 // x > -k ? (x > k ? k : x) : -k
5309 // x < k ? (x < -k ? -k : x) : k
5324 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSaturatingConditional()
5334 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get(); in LowerSaturatingConditional()
5346 // Check that the constant in the lower-bound check is in LowerSaturatingConditional()
5347 // the opposite of the constant in the upper-bound check in LowerSaturatingConditional()
5352 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue(); in LowerSaturatingConditional()
5353 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue(); in LowerSaturatingConditional()
5379 // - The conditions and values match up
5380 // - k is 0 or -1 (all ones)
5389 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in isLowerSaturatingConditional()
5420 return !Subtarget->hasVFP2Base(); in isUnsupportedFloatingType()
5422 return !Subtarget->hasFP64(); in isUnsupportedFloatingType()
5424 return !Subtarget->hasFullFP16(); in isUnsupportedFloatingType()
5433 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2()) in LowerSELECT_CC()
5438 // into more efficient bit operations, which is possible when k is 0 or -1 in LowerSELECT_CC()
5439 // On ARM and Thumb-2 which have flexible operand 2 this will result in in LowerSELECT_CC()
5442 // Only allow this transformation on full-width (32-bit) operations in LowerSELECT_CC()
5459 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSELECT_CC()
5465 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal && in LowerSELECT_CC()
5467 unsigned TVal = CTVal->getZExtValue(); in LowerSELECT_CC()
5468 unsigned FVal = CFVal->getZExtValue(); in LowerSELECT_CC()
5496 // -(-a) == a, but (a+1)+1 != a). in LowerSELECT_CC()
5536 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 || in LowerSELECT_CC()
5551 if (ARMcc->getAsZExtVal() == ARMCC::PL) in LowerSELECT_CC()
5563 if (Subtarget->hasFPARMv8Base() && in LowerSELECT_CC()
5594 /// canChangeToInt - Given the fp compare operand, return true if it is suitable
5599 if (!N->hasOneUse()) in canChangeToInt()
5602 if (!N->getNumValues()) in canChangeToInt()
5605 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow()) in canChangeToInt()
5607 // vmrs are very slow, e.g. cortex-a8. in canChangeToInt()
5622 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(), in bitcastf32Toi32()
5623 Ld->getPointerInfo(), Ld->getAlign(), in bitcastf32Toi32()
5624 Ld->getMemOperand()->getFlags()); in bitcastf32Toi32()
5640 SDValue Ptr = Ld->getBasePtr(); in expandf64Toi32()
5642 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(), in expandf64Toi32()
5643 Ld->getAlign(), Ld->getMemOperand()->getFlags()); in expandf64Toi32()
5648 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr, in expandf64Toi32()
5649 Ld->getPointerInfo().getWithOffset(4), in expandf64Toi32()
5650 commonAlignment(Ld->getAlign(), 4), in expandf64Toi32()
5651 Ld->getMemOperand()->getFlags()); in expandf64Toi32()
5658 /// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some
5663 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in OptimizeVFPBrcond()
5721 !Subtarget->isThumb1Only(); in LowerBRCOND()
5726 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) in LowerBRCOND()
5736 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue(); in LowerBRCOND()
5750 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in LowerBR_CC()
5772 !Subtarget->isThumb1Only(); in LowerBR_CC()
5778 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0))) in LowerBR_CC()
5789 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue(); in LowerBR_CC()
5839 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy); in LowerBR_JT()
5843 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) { in LowerBR_JT()
5844 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table in LowerBR_JT()
5851 if (isPositionIndependent() || Subtarget->isROPI()) { in LowerBR_JT()
5902 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_TO_INT()
5939 EVT ToVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); in LowerFP_TO_INT_SAT()
5945 Subtarget->hasFP64()) in LowerFP_TO_INT_SAT()
5948 Subtarget->hasFullFP16()) in LowerFP_TO_INT_SAT()
5951 Subtarget->hasMVEFloatOps()) in LowerFP_TO_INT_SAT()
5954 Subtarget->hasMVEFloatOps()) in LowerFP_TO_INT_SAT()
5962 unsigned BW = ToVT.getScalarSizeInBits() - IsSigned; in LowerFP_TO_INT_SAT()
5966 DAG.getConstant((1 << BW) - 1, DL, VT)); in LowerFP_TO_INT_SAT()
5969 DAG.getConstant(-(1 << BW), DL, VT)); in LowerFP_TO_INT_SAT()
6046 bool UseNEON = !InGPR && Subtarget->hasNEON(); in LowerFCOPYSIGN()
6138 // Return LR, which contains the return address. Mark it an implicit live-in. in LowerRETURNADDR()
6155 while (Depth--) in LowerFRAMEADDR()
6181 assert(N->getValueType(0) == MVT::i64 in ExpandREAD_REGISTER()
6182 && "ExpandREAD_REGISTER called for non-i64 type result."); in ExpandREAD_REGISTER()
6186 N->getOperand(0), in ExpandREAD_REGISTER()
6187 N->getOperand(1)); in ExpandREAD_REGISTER()
6203 SDValue Op = BC->getOperand(0); in CombineVMOVDRRCandidateWithVecOp()
6204 EVT DstVT = BC->getValueType(0); in CombineVMOVDRRCandidateWithVecOp()
6225 const APInt &APIntIndex = Index->getAPIntValue(); in CombineVMOVDRRCandidateWithVecOp()
6232 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) -> in CombineVMOVDRRCandidateWithVecOp()
6244 /// ExpandBITCAST - If the target supports VFP, this function is called to
6246 /// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
6253 SDValue Op = N->getOperand(0); in ExpandBITCAST()
6258 EVT DstVT = N->getValueType(0); in ExpandBITCAST()
6274 // Turn i64->f64 into VMOVDRR. in ExpandBITCAST()
6286 // Turn f64->i64 into VMOVRRD. in ExpandBITCAST()
6304 /// getZeroVector - Returns a vector of specified type with all zero elements.
6319 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6323 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); in LowerShiftRightParts()
6352 DAG.getConstant(VTBits - 1, dl, VT)) in LowerShiftRightParts()
6363 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6367 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); in LowerShiftLeftParts()
6405 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0 in LowerGET_ROUNDING()
6428 SDValue Chain = Op->getOperand(0); in LowerSET_ROUNDING()
6429 SDValue RMValue = Op->getOperand(1); in LowerSET_ROUNDING()
6433 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is in LowerSET_ROUNDING()
6434 // ((arg - 1) & 3) << 22). in LowerSET_ROUNDING()
6470 SDValue Chain = Op->getOperand(0); in LowerSET_FPMODE()
6471 SDValue Mode = Op->getOperand(1); in LowerSET_FPMODE()
6498 SDValue Chain = Op->getOperand(0); in LowerRESET_FPMODE()
6521 EVT VT = N->getValueType(0); in LowerCTTZ()
6522 if (VT.isVector() && ST->hasNEON()) { in LowerCTTZ()
6524 // Compute the least significant set bit: LSB = X & -X in LowerCTTZ()
6525 SDValue X = N->getOperand(0); in LowerCTTZ()
6532 // Compute with: cttz(x) = ctpop(lsb - 1) in LowerCTTZ()
6540 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) { in LowerCTTZ()
6541 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0 in LowerCTTZ()
6545 DAG.getTargetConstant(NumBits - 1, dl, ElemTy)); in LowerCTTZ()
6550 // Compute with: cttz(x) = ctpop(lsb - 1) in LowerCTTZ()
6552 // Compute LSB - 1. in LowerCTTZ()
6567 if (!ST->hasV6T2Ops()) in LowerCTTZ()
6570 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); in LowerCTTZ()
6576 EVT VT = N->getValueType(0); in LowerCTPOP()
6579 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON."); in LowerCTPOP()
6586 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0)); in LowerCTPOP()
6607 /// Getvshiftimm - Check if this is a valid build_vector for the immediate
6619 !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, in getVShiftImm()
6627 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
6636 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits); in isVShiftLImm()
6639 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
6653 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) { in isVShiftRImm()
6654 Cnt = -Cnt; in isVShiftRImm()
6662 EVT VT = N->getValueType(0); in LowerShift()
6675 if (N->getOpcode() == ISD::SHL) { in LowerShift()
6676 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) in LowerShift()
6677 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0), in LowerShift()
6679 return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0), in LowerShift()
6680 N->getOperand(1)); in LowerShift()
6683 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && in LowerShift()
6686 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { in LowerShift()
6688 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM); in LowerShift()
6689 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), in LowerShift()
6695 EVT ShiftVT = N->getOperand(1).getValueType(); in LowerShift()
6697 ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1)); in LowerShift()
6699 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu); in LowerShift()
6700 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount); in LowerShift()
6705 EVT VT = N->getValueType(0); in Expand64BitShift()
6712 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA || in Expand64BitShift()
6713 N->getOpcode() == ISD::SHL) && in Expand64BitShift()
6716 unsigned ShOpc = N->getOpcode(); in Expand64BitShift()
6717 if (ST->hasMVEIntegerOps()) { in Expand64BitShift()
6718 SDValue ShAmt = N->getOperand(1); in Expand64BitShift()
6724 if ((!Con && ShAmt->getValueType(0).getSizeInBits() > 64) || in Expand64BitShift()
6725 (Con && (Con->getAPIntValue() == 0 || Con->getAPIntValue().uge(32)))) in Expand64BitShift()
6729 if (ShAmt->getValueType(0) != MVT::i32) in Expand64BitShift()
6747 DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32); in Expand64BitShift()
6757 if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL) in Expand64BitShift()
6761 if (ST->isThumb1Only()) in Expand64BitShift()
6764 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr. in Expand64BitShift()
6766 std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32); in Expand64BitShift()
6770 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_GLUE:ARMISD::SRA_GLUE; in Expand64BitShift()
6790 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerVSETCC()
6794 if (ST->hasNEON()) in LowerVSETCC()
6797 assert(ST->hasMVEIntegerOps() && in LowerVSETCC()
6805 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps()) in LowerVSETCC()
6813 // Special-case integer 64-bit equality comparisons. They aren't legal, in LowerVSETCC()
6831 // 64-bit comparisons are not legal in general. in LowerVSETCC()
6839 if (ST->hasMVEFloatOps()) { in LowerVSETCC()
6888 if (ST->hasMVEIntegerOps()) { in LowerVSETCC()
6905 if (ST->hasNEON() && Opc == ARMCC::EQ) { in LowerVSETCC()
6931 // comparison to a specialized compare-against-zero form. in LowerVSETCC()
6982 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32); in LowerSETCCCARRY()
6990 /// isVMOVModifiedImm - Check if the specified splat value corresponds to a
7002 // immediate instructions others than VMOV do not support the 8-bit encoding in isVMOVModifiedImm()
7004 // 32-bit version. in isVMOVModifiedImm()
7012 // Any 1-byte value is OK. Op=0, Cmode=1110. in isVMOVModifiedImm()
7020 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero. in isVMOVModifiedImm()
7037 // NEON's 32-bit VMOV supports splat values where: in isVMOVModifiedImm()
7090 // Note: there are a few 32-bit splat values (specifically: 00ffff00, in isVMOVModifiedImm()
7093 // and fall through here to test for a valid 64-bit splat. But, then the in isVMOVModifiedImm()
7100 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff. in isVMOVModifiedImm()
7117 unsigned Mask = (1 << BytesPerElem) - 1; in isVMOVModifiedImm()
7122 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem; in isVMOVModifiedImm()
7146 const APFloat &FPVal = CFP->getValueAPF(); in LowerConstantFP()
7148 // Prevent floating-point constants from using literal loads in LowerConstantFP()
7149 // when execute-only is enabled. in LowerConstantFP()
7150 if (ST->genExecuteOnly()) { in LowerConstantFP()
7151 // We shouldn't trigger this for v6m execute-only in LowerConstantFP()
7152 assert((!ST->isThumb1Only() || ST->hasV8MBaselineOps()) && in LowerConstantFP()
7176 if (!ST->hasVFP3Base()) in LowerConstantFP()
7180 // an SP-only FPU in LowerConstantFP()
7181 if (IsDouble && !Subtarget->hasFP64()) in LowerConstantFP()
7187 if (ImmVal != -1) { in LowerConstantFP()
7188 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) { in LowerConstantFP()
7206 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP())) in LowerConstantFP()
7316 Imm -= NumElts; in isVEXTMask()
7341 // WhichResult gives the offset for each element in the mask based on which
7368 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only in isVTRNMask()
7385 /// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
7440 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVUZPMask()
7447 /// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
7476 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVUZP_v_undef_Mask()
7514 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVZIPMask()
7521 /// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
7547 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVZIP_v_undef_Mask()
7554 /// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7585 // Look for <15, ..., 3, -1, 1, 0>. in isReverseMask()
7587 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i)) in isReverseMask()
7599 // Half-width truncation patterns (e.g. v4i32 -> v8i16): in isTruncMask()
7670 if (!ST->hasMVEFloatOps()) in LowerBuildVectorOfFPTrunc()
7723 if (!ST->hasMVEFloatOps()) in LowerBuildVectorOfFPExt()
7767 Val = N->getAsZExtVal(); in IsSingleInstrConstant()
7769 if (ST->isThumb1Only()) { in IsSingleInstrConstant()
7773 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1) in IsSingleInstrConstant()
7784 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!"); in LowerBUILD_VECTOR_i1()
7808 llvm::all_of(llvm::drop_begin(Op->ops()), [&FirstOp](const SDUse &U) { in LowerBUILD_VECTOR_i1()
7822 bool BitSet = V.isUndef() ? false : V->getAsZExtVal(); in LowerBUILD_VECTOR_i1()
7843 if (!ST->hasMVEIntegerOps()) in LowerBUILD_VECTORToVIDUP()
7877 switch (N->getOpcode()) { in IsQRMVEInstruction()
7886 return N->getOperand(1).getNode() == Op; in IsQRMVEInstruction()
7888 switch (N->getConstantOperandVal(0)) { in IsQRMVEInstruction()
7905 return N->getOperand(2).getNode() == Op; in IsQRMVEInstruction()
7922 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) in LowerBUILD_VECTOR()
7931 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { in LowerBUILD_VECTOR()
7937 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize && in LowerBUILD_VECTOR()
7939 all_of(BVN->uses(), in LowerBUILD_VECTOR()
7949 if ((ST->hasNEON() && SplatBitSize <= 64) || in LowerBUILD_VECTOR()
7950 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) { in LowerBUILD_VECTOR()
7966 VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm); in LowerBUILD_VECTOR()
7975 if (ImmVal != -1) { in LowerBUILD_VECTOR()
7983 if (ST->hasMVEIntegerOps() && in LowerBUILD_VECTOR()
8033 Value = ValueCounts.begin()->first; in LowerBUILD_VECTOR()
8045 // Use VDUP for non-constant splats. For f32 constant splats, reduce to in LowerBUILD_VECTOR()
8055 // constant-index forms. in LowerBUILD_VECTOR()
8057 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in LowerBUILD_VECTOR()
8058 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) { in LowerBUILD_VECTOR()
8063 if (VT != Value->getOperand(0).getValueType()) { in LowerBUILD_VECTOR()
8064 unsigned index = constIndex->getAPIntValue().getLimitedValue() % in LowerBUILD_VECTOR()
8072 Value->getOperand(0), Value->getOperand(1)); in LowerBUILD_VECTOR()
8132 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) { in LowerBUILD_VECTOR()
8133 // If we haven't found an efficient lowering, try splitting a 128-bit vector in LowerBUILD_VECTOR()
8134 // into two 64-bit vectors; we might discover a better way to lower it. in LowerBUILD_VECTOR()
8135 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts); in LowerBUILD_VECTOR()
8149 // Vectors with 32- or 64-bit elements can be built by directly assigning in LowerBUILD_VECTOR()
8153 // Do the expansion with floating-point types, since that is what the VFP in LowerBUILD_VECTOR()
8166 // worse. For a vector with one or two non-undef values, that's in LowerBUILD_VECTOR()
8239 Source->MinElt = std::min(Source->MinElt, EltNo); in ReconstructShuffle()
8240 Source->MaxElt = std::max(Source->MaxElt, EltNo); in ReconstructShuffle()
8292 if (Src.MaxElt - Src.MinElt >= NumSrcElts) { in ReconstructShuffle()
8302 Src.WindowBase = -NumSrcElts; in ReconstructShuffle()
8320 Src.WindowBase = -Src.MinElt; in ReconstructShuffle()
8343 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1); in ReconstructShuffle()
8351 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue(); in ReconstructShuffle()
8365 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase; in ReconstructShuffle()
8366 ExtractBase += NumElts * (Src - Sources.begin()); in ReconstructShuffle()
8419 /// isShuffleMaskLegal - Targets can use this to indicate that they only
8440 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry))) in isShuffleMaskLegal()
8455 else if (Subtarget->hasNEON() && in isShuffleMaskLegal()
8463 else if (Subtarget->hasMVEIntegerOps() && in isShuffleMaskLegal()
8467 else if (Subtarget->hasMVEIntegerOps() && in isShuffleMaskLegal()
8476 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8482 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); in GeneratePerfectShuffle()
8483 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); in GeneratePerfectShuffle()
8502 // vrev <4 x i16> -> VREV32 in GeneratePerfectShuffle()
8505 // vrev <4 x i8> -> VREV16 in GeneratePerfectShuffle()
8513 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32)); in GeneratePerfectShuffle()
8519 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32)); in GeneratePerfectShuffle()
8523 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL); in GeneratePerfectShuffle()
8527 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL); in GeneratePerfectShuffle()
8531 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL); in GeneratePerfectShuffle()
8547 if (V2.getNode()->isUndef()) in LowerVECTOR_SHUFFLEv8i8()
8592 // of all ones or all zeroes and selecting the lanes based upon the real in PromoteMVEPredVector()
8619 // Recast our new predicate-as-integer v16i8 vector into something in PromoteMVEPredVector()
8628 ArrayRef<int> ShuffleMask = SVN->getMask(); in LowerVECTOR_SHUFFLE_i1()
8630 assert(ST->hasMVEIntegerOps() && in LowerVECTOR_SHUFFLE_i1()
8646 // to essentially promote the boolean predicate to a 8-bit integer, where in LowerVECTOR_SHUFFLE_i1()
8650 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit in LowerVECTOR_SHUFFLE_i1()
8651 // fields in a register into 8 other arbitrary 2-bit fields! in LowerVECTOR_SHUFFLE_i1()
8697 // Detect which mov lane this would be from the first non-undef element. in LowerVECTOR_SHUFFLEUsingMovs()
8698 int MovIdx = -1; in LowerVECTOR_SHUFFLEUsingMovs()
8702 return -1; in LowerVECTOR_SHUFFLEUsingMovs()
8708 if (MovIdx == -1) in LowerVECTOR_SHUFFLEUsingMovs()
8709 return -1; in LowerVECTOR_SHUFFLEUsingMovs()
8715 return -1; in LowerVECTOR_SHUFFLEUsingMovs()
8723 if (Elt != -1) { in LowerVECTOR_SHUFFLEUsingMovs()
8724 SDValue Input = Op->getOperand(0); in LowerVECTOR_SHUFFLEUsingMovs()
8726 Input = Op->getOperand(1); in LowerVECTOR_SHUFFLEUsingMovs()
8727 Elt -= 4; in LowerVECTOR_SHUFFLEUsingMovs()
8746 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]); in LowerVECTOR_SHUFFLEUsingMovs()
8748 VT, dl, Op->getOperand(0), Op->getOperand(1), NewShuffleMask); in LowerVECTOR_SHUFFLEUsingMovs()
8770 // An One-Off Identity mask is one that is mostly an identity mask from as in LowerVECTOR_SHUFFLEUsingOneOff()
8771 // single source but contains a single element out-of-place, either from a in LowerVECTOR_SHUFFLEUsingOneOff()
8777 OffElement = -1; in LowerVECTOR_SHUFFLEUsingOneOff()
8780 if (Mask[i] == -1) in LowerVECTOR_SHUFFLEUsingOneOff()
8784 if (OffElement == -1) in LowerVECTOR_SHUFFLEUsingOneOff()
8790 return NonUndef > 2 && OffElement != -1; in LowerVECTOR_SHUFFLEUsingOneOff()
8822 if (ST->hasMVEIntegerOps() && EltSize == 1) in LowerVECTOR_SHUFFLE()
8825 // Convert shuffles that are directly supported on NEON to target-specific in LowerVECTOR_SHUFFLE()
8829 // FIXME: floating-point vectors should be canonicalized to integer vectors in LowerVECTOR_SHUFFLE()
8831 ArrayRef<int> ShuffleMask = SVN->getMask(); in LowerVECTOR_SHUFFLE()
8834 if (SVN->isSplat()) { in LowerVECTOR_SHUFFLE()
8835 int Lane = SVN->getSplatIndex(); in LowerVECTOR_SHUFFLE()
8837 if (Lane == -1) Lane = 0; in LowerVECTOR_SHUFFLE()
8863 if (ST->hasNEON() && isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) { in LowerVECTOR_SHUFFLE()
8877 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
8889 if (ST->hasNEON()) { in LowerVECTOR_SHUFFLE()
8898 if (ST->hasMVEIntegerOps()) { in LowerVECTOR_SHUFFLE()
8913 // -> in LowerVECTOR_SHUFFLE()
8918 // native shuffles produce larger results: the two-result ops. in LowerVECTOR_SHUFFLE()
8922 // -> in LowerVECTOR_SHUFFLE()
8925 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) { in LowerVECTOR_SHUFFLE()
8926 SDValue SubV1 = V1->getOperand(0); in LowerVECTOR_SHUFFLE()
8927 SDValue SubV2 = V1->getOperand(1); in LowerVECTOR_SHUFFLE()
8930 // We expect these to have been canonicalized to -1. in LowerVECTOR_SHUFFLE()
8940 "In-place shuffle of concat can only have one result!"); in LowerVECTOR_SHUFFLE()
8949 if (ST->hasMVEIntegerOps() && EltSize <= 32) { in LowerVECTOR_SHUFFLE()
8973 // the PerfectShuffle-generated table to synthesize it from other shuffles. in LowerVECTOR_SHUFFLE()
8991 if (ST->hasNEON()) in LowerVECTOR_SHUFFLE()
8994 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); in LowerVECTOR_SHUFFLE()
8995 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); in LowerVECTOR_SHUFFLE()
9004 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs. in LowerVECTOR_SHUFFLE()
9006 // Do the expansion with floating-point types, since that is what the VFP in LowerVECTOR_SHUFFLE()
9019 DAG.getConstant(ShuffleMask[i] & (NumElts-1), in LowerVECTOR_SHUFFLE()
9030 if (ST->hasNEON() && VT == MVT::v8i8) in LowerVECTOR_SHUFFLE()
9034 if (ST->hasMVEIntegerOps()) in LowerVECTOR_SHUFFLE()
9046 assert(ST->hasMVEIntegerOps() && in LowerINSERT_VECTOR_ELT_i1()
9050 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0)); in LowerINSERT_VECTOR_ELT_i1()
9054 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth; in LowerINSERT_VECTOR_ELT_i1()
9072 if (Subtarget->hasMVEIntegerOps() && in LowerINSERT_VECTOR_ELT()
9080 // Reinterpret any such vector-element insertion as one with the in LowerINSERT_VECTOR_ELT()
9109 assert(ST->hasMVEIntegerOps() && in LowerEXTRACT_VECTOR_ELT_i1()
9113 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0)); in LowerEXTRACT_VECTOR_ELT_i1()
9132 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) in LowerEXTRACT_VECTOR_ELT()
9150 assert(ST->hasMVEIntegerOps() && in LowerCONCAT_VECTORS_i1()
9213 SmallVector<SDValue> ConcatOps(Op->op_begin(), Op->op_end()); in LowerCONCAT_VECTORS_i1()
9227 EVT VT = Op->getValueType(0); in LowerCONCAT_VECTORS()
9228 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) in LowerCONCAT_VECTORS()
9232 // two 64-bit vectors are concatenated to a 128-bit vector. in LowerCONCAT_VECTORS()
9258 unsigned Index = V2->getAsZExtVal(); in LowerEXTRACT_SUBVECTOR()
9262 assert(ST->hasMVEIntegerOps() && in LowerEXTRACT_SUBVECTOR()
9306 assert(ST->hasMVEIntegerOps() && "Expected MVE!"); in LowerTruncatei1()
9307 EVT VT = N->getValueType(0); in LowerTruncatei1()
9310 SDValue Op = N->getOperand(0); in LowerTruncatei1()
9322 if (!Subtarget->hasMVEIntegerOps()) in LowerTruncate()
9325 EVT ToVT = N->getValueType(0); in LowerTruncate()
9340 // instructions, but that doesn't extend to v8i32->v8i16 where the lanes need in LowerTruncate()
9345 // - Wherever possible combine them into an instruction that makes them in LowerTruncate()
9349 // - Lane Interleaving to transform blocks surrounded by ext/trunc. So in LowerTruncate()
9354 // - Otherwise we have an option. By default we would expand the in LowerTruncate()
9358 // - The other option is to use the fact that loads/store can extend/truncate in LowerTruncate()
9360 // becomes 3 back-to-back memory operations, but at least that is less than in LowerTruncate()
9370 EVT FromVT = N->getOperand(0).getValueType(); in LowerTruncate()
9382 if (!Subtarget->hasMVEIntegerOps()) in LowerVectorExtend()
9387 EVT ToVT = N->getValueType(0); in LowerVectorExtend()
9390 SDValue Op = N->getOperand(0); in LowerVectorExtend()
9401 N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT; in LowerVectorExtend()
9406 Ext = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext); in LowerVectorExtend()
9407 Ext1 = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext1); in LowerVectorExtend()
9413 /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
9414 /// element has been zero/sign-extended, depending on the isSigned parameter,
9419 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR()
9420 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { in isExtendedBUILD_VECTOR()
9421 SDNode *BVN = N->getOperand(0).getNode(); in isExtendedBUILD_VECTOR()
9422 if (BVN->getValueType(0) != MVT::v4i32 || in isExtendedBUILD_VECTOR()
9423 BVN->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
9426 unsigned HiElt = 1 - LoElt; in isExtendedBUILD_VECTOR()
9427 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt)); in isExtendedBUILD_VECTOR()
9428 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt)); in isExtendedBUILD_VECTOR()
9429 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2)); in isExtendedBUILD_VECTOR()
9430 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2)); in isExtendedBUILD_VECTOR()
9434 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 && in isExtendedBUILD_VECTOR()
9435 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32) in isExtendedBUILD_VECTOR()
9438 if (Hi0->isZero() && Hi1->isZero()) in isExtendedBUILD_VECTOR()
9444 if (N->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
9447 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { in isExtendedBUILD_VECTOR()
9448 SDNode *Elt = N->getOperand(i).getNode(); in isExtendedBUILD_VECTOR()
9453 if (!isIntN(HalfSize, C->getSExtValue())) in isExtendedBUILD_VECTOR()
9456 if (!isUIntN(HalfSize, C->getZExtValue())) in isExtendedBUILD_VECTOR()
9467 /// isSignExtended - Check if a node is a vector value that is sign-extended
9468 /// or a constant BUILD_VECTOR with sign-extended elements.
9470 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N)) in isSignExtended()
9477 /// isZeroExtended - Check if a node is a vector value that is zero-extended (or
9478 /// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
9480 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND || in isZeroExtended()
9505 /// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
9506 /// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
9513 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than in AddRequiredExtensionForVMULL()
9514 // 64-bits we need to insert a new extension so that it will be 64-bits. in AddRequiredExtensionForVMULL()
9525 /// SkipLoadExtensionForVMULL - return a load of the original vector size that
9531 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT()); in SkipLoadExtensionForVMULL()
9534 if (ExtendedTy == LD->getMemoryVT()) in SkipLoadExtensionForVMULL()
9535 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(), in SkipLoadExtensionForVMULL()
9536 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(), in SkipLoadExtensionForVMULL()
9537 LD->getMemOperand()->getFlags()); in SkipLoadExtensionForVMULL()
9542 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy, in SkipLoadExtensionForVMULL()
9543 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(), in SkipLoadExtensionForVMULL()
9544 LD->getMemoryVT(), LD->getAlign(), in SkipLoadExtensionForVMULL()
9545 LD->getMemOperand()->getFlags()); in SkipLoadExtensionForVMULL()
9548 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
9555 if (N->getOpcode() == ISD::SIGN_EXTEND || in SkipExtensionForVMULL()
9556 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND) in SkipExtensionForVMULL()
9557 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG, in SkipExtensionForVMULL()
9558 N->getOperand(0)->getValueType(0), in SkipExtensionForVMULL()
9559 N->getValueType(0), in SkipExtensionForVMULL()
9560 N->getOpcode()); in SkipExtensionForVMULL()
9570 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad); in SkipExtensionForVMULL()
9578 if (N->getOpcode() == ISD::BITCAST) { in SkipExtensionForVMULL()
9579 SDNode *BVN = N->getOperand(0).getNode(); in SkipExtensionForVMULL()
9580 assert(BVN->getOpcode() == ISD::BUILD_VECTOR && in SkipExtensionForVMULL()
9581 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR"); in SkipExtensionForVMULL()
9585 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)}); in SkipExtensionForVMULL()
9588 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); in SkipExtensionForVMULL()
9589 EVT VT = N->getValueType(0); in SkipExtensionForVMULL()
9596 const APInt &CInt = N->getConstantOperandAPInt(i); in SkipExtensionForVMULL()
9605 unsigned Opcode = N->getOpcode(); in isAddSubSExt()
9607 SDNode *N0 = N->getOperand(0).getNode(); in isAddSubSExt()
9608 SDNode *N1 = N->getOperand(1).getNode(); in isAddSubSExt()
9609 return N0->hasOneUse() && N1->hasOneUse() && in isAddSubSExt()
9616 unsigned Opcode = N->getOpcode(); in isAddSubZExt()
9618 SDNode *N0 = N->getOperand(0).getNode(); in isAddSubZExt()
9619 SDNode *N1 = N->getOperand(1).getNode(); in isAddSubZExt()
9620 return N0->hasOneUse() && N1->hasOneUse() && in isAddSubZExt()
9627 // Multiplications are only custom-lowered for 128-bit vectors so that in LowerMUL()
9631 "unexpected type for custom-lowering ISD::MUL"); in LowerMUL()
9684 // isel lowering to take advantage of no-stall back to back vmul + vmla. in LowerMUL()
9691 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG); in LowerMUL()
9692 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG); in LowerMUL()
9694 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
9703 // TODO: Should this propagate fast-math-flags? in LowerSDIV_v4i8()
9734 // TODO: Should this propagate fast-math-flags? in LowerSDIV_v4i16()
9775 "unexpected type for custom-lowering ISD::SDIV"); in LowerSDIV()
9809 // TODO: Should this propagate fast-math-flags? in LowerUDIV()
9812 "unexpected type for custom-lowering ISD::UDIV"); in LowerUDIV()
9886 EVT VT = N->getValueType(0); in LowerUADDSUBO_CARRY()
9919 // by ISD::USUBO_CARRY, so compute 1 - C. in LowerUADDSUBO_CARRY()
9925 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry); in LowerUADDSUBO_CARRY()
9929 assert(Subtarget->isTargetDarwin()); in LowerFSINCOS()
9947 bool ShouldUseSRet = Subtarget->isAPCS_ABI(); in LowerFSINCOS()
9958 Entry.Ty = PointerType::getUnqual(RetTy->getContext()); in LowerFSINCOS()
10049 if (N->getOpcode() != ISD::SDIV) in BuildSDIVPow2()
10059 if (N->getOperand(0).getValueType().isVector()) in BuildSDIVPow2()
10073 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV, in BuildSDIVPow2()
10097 SDValue Op = N->getOperand(1); in WinDBZCheckDenominator()
10098 if (N->getValueType(0) == MVT::i32) in WinDBZCheckDenominator()
10130 EVT MemVT = LD->getMemoryVT(); in LowerPredicateLoad()
10135 assert(LD->getExtensionType() == ISD::NON_EXTLOAD && in LowerPredicateLoad()
10136 "Expected a non-extending load"); in LowerPredicateLoad()
10137 assert(LD->isUnindexed() && "Expected a unindexed load"); in LowerPredicateLoad()
10151 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(), in LowerPredicateLoad()
10153 LD->getMemOperand()); in LowerPredicateLoad()
10158 DAG.getConstant(32 - MemVT.getSizeInBits(), dl, MVT::i32)); in LowerPredicateLoad()
10169 EVT MemVT = LD->getMemoryVT(); in LowerLOAD()
10170 assert(LD->isUnindexed() && "Loads should be unindexed at this point."); in LowerLOAD()
10172 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() && in LowerLOAD()
10173 !Subtarget->isThumb1Only() && LD->isVolatile() && in LowerLOAD()
10174 LD->getAlign() >= Subtarget->getDualLoadStoreAlignment()) { in LowerLOAD()
10178 {LD->getChain(), LD->getBasePtr()}, MemVT, LD->getMemOperand()); in LowerLOAD()
10188 EVT MemVT = ST->getMemoryVT(); in LowerPredicateStore()
10192 assert(MemVT == ST->getValue().getValueType()); in LowerPredicateStore()
10193 assert(!ST->isTruncatingStore() && "Expected a non-extending store"); in LowerPredicateStore()
10194 assert(ST->isUnindexed() && "Expected a unindexed store"); in LowerPredicateStore()
10199 SDValue Build = ST->getValue(); in LowerPredicateStore()
10204 ? MemVT.getVectorNumElements() - I - 1 in LowerPredicateStore()
10219 ST->getChain(), dl, GRP, ST->getBasePtr(), in LowerPredicateStore()
10221 ST->getMemOperand()); in LowerPredicateStore()
10227 EVT MemVT = ST->getMemoryVT(); in LowerSTORE()
10228 assert(ST->isUnindexed() && "Stores should be unindexed at this point."); in LowerSTORE()
10230 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() && in LowerSTORE()
10231 !Subtarget->isThumb1Only() && ST->isVolatile() && in LowerSTORE()
10232 ST->getAlign() >= Subtarget->getDualLoadStoreAlignment()) { in LowerSTORE()
10237 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(), in LowerSTORE()
10241 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(), in LowerSTORE()
10246 {ST->getChain(), Lo, Hi, ST->getBasePtr()}, in LowerSTORE()
10247 MemVT, ST->getMemOperand()); in LowerSTORE()
10248 } else if (Subtarget->hasMVEIntegerOps() && in LowerSTORE()
10259 (N->getOpcode() == ARMISD::VMOVIMM && in isZeroVector()
10260 isNullConstant(N->getOperand(0)))); in isZeroVector()
10266 SDValue Mask = N->getMask(); in LowerMLOAD()
10267 SDValue PassThru = N->getPassThru(); in LowerMLOAD()
10278 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, ZeroVec, in LowerMLOAD()
10279 N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), in LowerMLOAD()
10280 N->getExtensionType(), N->isExpandingLoad()); in LowerMLOAD()
10284 isZeroVector(PassThru->getOperand(0)); in LowerMLOAD()
10292 if (!ST->hasMVEIntegerOps()) in LowerVecReduce()
10297 switch (Op->getOpcode()) { in LowerVecReduce()
10309 SDValue Op0 = Op->getOperand(0); in LowerVecReduce()
10339 SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags()); in LowerVecReduce()
10340 SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags()); in LowerVecReduce()
10341 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res0, Res1, Op->getFlags()); in LowerVecReduce()
10347 Res = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags()); in LowerVecReduce()
10351 if (EltVT != Op->getValueType(0)) in LowerVecReduce()
10352 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res); in LowerVecReduce()
10358 if (!ST->hasMVEFloatOps()) in LowerVecReduceF()
10365 if (!ST->hasNEON()) in LowerVecReduceMinMax()
10369 SDValue Op0 = Op->getOperand(0); in LowerVecReduceMinMax()
10374 switch (Op->getOpcode()) { in LowerVecReduceMinMax()
10399 // Split 128-bit vectors, since vpmin/max takes 2 64-bit vectors. in LowerVecReduceMinMax()
10420 switch (Op->getOpcode()) { in LowerVecReduceMinMax()
10438 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getSuccessOrdering())) in LowerAtomicLoadStore()
10452 // Under Power Management extensions, the cycle-count is: in ReplaceREADCYCLECOUNTER()
10454 SDValue Ops[] = { N->getOperand(0), // Chain in ReplaceREADCYCLECOUNTER()
10488 assert(N->getValueType(0) == MVT::i64 && in ReplaceCMP_SWAP_64Results()
10490 SDValue Ops[] = {N->getOperand(1), in ReplaceCMP_SWAP_64Results()
10491 createGPRPairNode(DAG, N->getOperand(2)), in ReplaceCMP_SWAP_64Results()
10492 createGPRPairNode(DAG, N->getOperand(3)), in ReplaceCMP_SWAP_64Results()
10493 N->getOperand(0)}; in ReplaceCMP_SWAP_64Results()
10498 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in ReplaceCMP_SWAP_64Results()
10519 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get(); in LowerFSETCC()
10633 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) in LowerOperation()
10637 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector()) in LowerOperation()
10681 if (Subtarget->isTargetWindows()) in LowerOperation()
10698 unsigned IntNo = N->getConstantOperandVal(0); in ReplaceLongIntrinsic()
10713 std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(3), dl, MVT::i32, MVT::i32); in ReplaceLongIntrinsic()
10717 N->getOperand(1), N->getOperand(2), in ReplaceLongIntrinsic()
10723 /// ReplaceNodeResults - Replace the results of node with an illegal result
10729 switch (N->getOpcode()) { in ReplaceNodeResults()
10765 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows"); in ReplaceNodeResults()
10766 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV, in ReplaceNodeResults()
10792 //===----------------------------------------------------------------------===//
10794 //===----------------------------------------------------------------------===//
10796 /// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10802 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() && in SetupEntryBlockForSjLj()
10804 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in SetupEntryBlockForSjLj()
10806 MachineFunction *MF = MBB->getParent(); in SetupEntryBlockForSjLj()
10807 MachineRegisterInfo *MRI = &MF->getRegInfo(); in SetupEntryBlockForSjLj()
10808 MachineConstantPool *MCP = MF->getConstantPool(); in SetupEntryBlockForSjLj()
10809 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>(); in SetupEntryBlockForSjLj()
10810 const Function &F = MF->getFunction(); in SetupEntryBlockForSjLj()
10812 bool isThumb = Subtarget->isThumb(); in SetupEntryBlockForSjLj()
10813 bool isThumb2 = Subtarget->isThumb2(); in SetupEntryBlockForSjLj()
10815 unsigned PCLabelId = AFI->createPICLabelUId(); in SetupEntryBlockForSjLj()
10819 unsigned CPI = MCP->getConstantPoolIndex(CPV, Align(4)); in SetupEntryBlockForSjLj()
10826 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF), in SetupEntryBlockForSjLj()
10830 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI), in SetupEntryBlockForSjLj()
10840 Register NewVReg1 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10841 BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1) in SetupEntryBlockForSjLj()
10846 Register NewVReg2 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10847 BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2) in SetupEntryBlockForSjLj()
10852 Register NewVReg3 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10853 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3) in SetupEntryBlockForSjLj()
10856 BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12)) in SetupEntryBlockForSjLj()
10870 Register NewVReg1 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10871 BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1) in SetupEntryBlockForSjLj()
10875 Register NewVReg2 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10876 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2) in SetupEntryBlockForSjLj()
10880 Register NewVReg3 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10881 BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3) in SetupEntryBlockForSjLj()
10885 Register NewVReg4 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10886 BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4) in SetupEntryBlockForSjLj()
10891 Register NewVReg5 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10892 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5) in SetupEntryBlockForSjLj()
10895 BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi)) in SetupEntryBlockForSjLj()
10906 Register NewVReg1 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10907 BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1) in SetupEntryBlockForSjLj()
10912 Register NewVReg2 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
10913 BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2) in SetupEntryBlockForSjLj()
10917 BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12)) in SetupEntryBlockForSjLj()
10928 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitSjLjDispatchBlock()
10930 MachineFunction *MF = MBB->getParent(); in EmitSjLjDispatchBlock()
10931 MachineRegisterInfo *MRI = &MF->getRegInfo(); in EmitSjLjDispatchBlock()
10932 MachineFrameInfo &MFI = MF->getFrameInfo(); in EmitSjLjDispatchBlock()
10935 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass in EmitSjLjDispatchBlock()
10953 if (!MF->hasCallSiteLandingPad(Sym)) continue; in EmitSjLjDispatchBlock()
10955 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym); in EmitSjLjDispatchBlock()
10972 InvokeBBs.insert(MBB->pred_begin(), MBB->pred_end()); in EmitSjLjDispatchBlock()
10981 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline); in EmitSjLjDispatchBlock()
10982 unsigned MJTI = JTI->createJumpTableIndex(LPadList); in EmitSjLjDispatchBlock()
10987 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
10988 DispatchBB->setIsEHPad(); in EmitSjLjDispatchBlock()
10990 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
10992 if (Subtarget->isThumb()) in EmitSjLjDispatchBlock()
10995 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP; in EmitSjLjDispatchBlock()
10997 BuildMI(TrapBB, dl, TII->get(trap_opcode)); in EmitSjLjDispatchBlock()
10998 DispatchBB->addSuccessor(TrapBB); in EmitSjLjDispatchBlock()
11000 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
11001 DispatchBB->addSuccessor(DispContBB); in EmitSjLjDispatchBlock()
11004 MF->insert(MF->end(), DispatchBB); in EmitSjLjDispatchBlock()
11005 MF->insert(MF->end(), DispContBB); in EmitSjLjDispatchBlock()
11006 MF->insert(MF->end(), TrapBB); in EmitSjLjDispatchBlock()
11012 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand( in EmitSjLjDispatchBlock()
11017 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup)); in EmitSjLjDispatchBlock()
11020 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo(); in EmitSjLjDispatchBlock()
11030 if (Subtarget->isThumb2()) { in EmitSjLjDispatchBlock()
11031 Register NewVReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11032 BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1) in EmitSjLjDispatchBlock()
11039 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri)) in EmitSjLjDispatchBlock()
11044 Register VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11045 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1) in EmitSjLjDispatchBlock()
11051 VReg2 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11052 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2) in EmitSjLjDispatchBlock()
11058 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr)) in EmitSjLjDispatchBlock()
11064 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc)) in EmitSjLjDispatchBlock()
11069 Register NewVReg3 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11070 BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT), NewVReg3) in EmitSjLjDispatchBlock()
11074 Register NewVReg4 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11075 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4) in EmitSjLjDispatchBlock()
11082 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT)) in EmitSjLjDispatchBlock()
11086 } else if (Subtarget->isThumb()) { in EmitSjLjDispatchBlock()
11087 Register NewVReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11088 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1) in EmitSjLjDispatchBlock()
11095 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8)) in EmitSjLjDispatchBlock()
11100 MachineConstantPool *ConstantPool = MF->getConstantPool(); in EmitSjLjDispatchBlock()
11101 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); in EmitSjLjDispatchBlock()
11105 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty); in EmitSjLjDispatchBlock()
11106 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment); in EmitSjLjDispatchBlock()
11108 Register VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11109 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci)) in EmitSjLjDispatchBlock()
11113 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr)) in EmitSjLjDispatchBlock()
11119 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc)) in EmitSjLjDispatchBlock()
11124 Register NewVReg2 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11125 BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2) in EmitSjLjDispatchBlock()
11131 Register NewVReg3 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11132 BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3) in EmitSjLjDispatchBlock()
11136 Register NewVReg4 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11137 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4) in EmitSjLjDispatchBlock()
11144 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF), in EmitSjLjDispatchBlock()
11147 Register NewVReg5 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11148 BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5) in EmitSjLjDispatchBlock()
11156 NewVReg6 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11157 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6) in EmitSjLjDispatchBlock()
11164 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr)) in EmitSjLjDispatchBlock()
11168 Register NewVReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11169 BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1) in EmitSjLjDispatchBlock()
11176 BuildMI(DispatchBB, dl, TII->get(ARM::CMPri)) in EmitSjLjDispatchBlock()
11180 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) { in EmitSjLjDispatchBlock()
11181 Register VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11182 BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1) in EmitSjLjDispatchBlock()
11188 VReg2 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11189 BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2) in EmitSjLjDispatchBlock()
11195 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) in EmitSjLjDispatchBlock()
11200 MachineConstantPool *ConstantPool = MF->getConstantPool(); in EmitSjLjDispatchBlock()
11201 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); in EmitSjLjDispatchBlock()
11205 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty); in EmitSjLjDispatchBlock()
11206 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment); in EmitSjLjDispatchBlock()
11208 Register VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11209 BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp)) in EmitSjLjDispatchBlock()
11214 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) in EmitSjLjDispatchBlock()
11220 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc)) in EmitSjLjDispatchBlock()
11225 Register NewVReg3 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11226 BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3) in EmitSjLjDispatchBlock()
11231 Register NewVReg4 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11232 BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4) in EmitSjLjDispatchBlock()
11237 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF), in EmitSjLjDispatchBlock()
11239 Register NewVReg5 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
11240 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5) in EmitSjLjDispatchBlock()
11248 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd)) in EmitSjLjDispatchBlock()
11253 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr)) in EmitSjLjDispatchBlock()
11263 DispContBB->addSuccessor(CurMBB); in EmitSjLjDispatchBlock()
11273 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors()); in EmitSjLjDispatchBlock()
11276 if (SMBB->isEHPad()) { in EmitSjLjDispatchBlock()
11277 BB->removeSuccessor(SMBB); in EmitSjLjDispatchBlock()
11282 BB->addSuccessor(DispatchBB, BranchProbability::getZero()); in EmitSjLjDispatchBlock()
11283 BB->normalizeSuccProbs(); in EmitSjLjDispatchBlock()
11285 // Find the invoke call and mark all of the callee-saved registers as in EmitSjLjDispatchBlock()
11290 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) { in EmitSjLjDispatchBlock()
11291 if (!II->isCall()) continue; in EmitSjLjDispatchBlock()
11295 OI = II->operands_begin(), OE = II->operands_end(); in EmitSjLjDispatchBlock()
11297 if (!OI->isReg()) continue; in EmitSjLjDispatchBlock()
11298 DefRegs[OI->getReg()] = true; in EmitSjLjDispatchBlock()
11305 if (Subtarget->isThumb2() && in EmitSjLjDispatchBlock()
11309 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg)) in EmitSjLjDispatchBlock()
11311 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg)) in EmitSjLjDispatchBlock()
11321 // Mark all former landing pads as non-landing pads. The dispatch is the only in EmitSjLjDispatchBlock()
11324 MBBLPad->setIsEHPad(false); in EmitSjLjDispatchBlock()
11332 for (MachineBasicBlock *S : MBB->successors()) in OtherSucc()
11376 /// Emit a post-increment load operation with given size. The instructions
11385 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
11392 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
11396 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut) in emitPostLd()
11402 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
11408 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
11417 /// Emit a post-increment store operation with given size. The instructions
11426 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) in emitPostSt()
11433 BuildMI(*BB, Pos, dl, TII->get(StOpc)) in emitPostSt()
11438 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut) in emitPostSt()
11444 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) in emitPostSt()
11450 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) in emitPostSt()
11463 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold(). in EmitStructByval()
11465 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitStructByval()
11466 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitStructByval()
11467 MachineFunction::iterator It = ++BB->getIterator(); in EmitStructByval()
11475 MachineFunction *MF = BB->getParent(); in EmitStructByval()
11476 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitStructByval()
11481 bool IsThumb1 = Subtarget->isThumb1Only(); in EmitStructByval()
11482 bool IsThumb2 = Subtarget->isThumb2(); in EmitStructByval()
11483 bool IsThumb = Subtarget->isThumb(); in EmitStructByval()
11491 if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) && in EmitStructByval()
11492 Subtarget->hasNEON()) { in EmitStructByval()
11512 unsigned LoopSize = SizeVal - BytesLeft; in EmitStructByval()
11514 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) { in EmitStructByval()
11553 // movw varEnd, # --> with thumb2 in EmitStructByval()
11555 // ldrcp varEnd, idx --> without thumb2 in EmitStructByval()
11556 // fallthrough --> loopMBB in EmitStructByval()
11565 // fallthrough --> exitMBB in EmitStructByval()
11567 // epilogue to handle left-over bytes in EmitStructByval()
11570 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitStructByval()
11571 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitStructByval()
11572 MF->insert(It, loopMBB); in EmitStructByval()
11573 MF->insert(It, exitMBB); in EmitStructByval()
11576 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI); in EmitStructByval()
11577 loopMBB->setCallFrameSize(CallFrameSize); in EmitStructByval()
11578 exitMBB->setCallFrameSize(CallFrameSize); in EmitStructByval()
11581 exitMBB->splice(exitMBB->begin(), BB, in EmitStructByval()
11582 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitStructByval()
11583 exitMBB->transferSuccessorsAndUpdatePHIs(BB); in EmitStructByval()
11587 if (Subtarget->useMovt()) { in EmitStructByval()
11588 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm), in EmitStructByval()
11591 } else if (Subtarget->genExecuteOnly()) { in EmitStructByval()
11592 assert(IsThumb && "Non-thumb expected to have used movt"); in EmitStructByval()
11593 BuildMI(BB, dl, TII->get(ARM::tMOVi32imm), varEnd).addImm(LoopSize); in EmitStructByval()
11595 MachineConstantPool *ConstantPool = MF->getConstantPool(); in EmitStructByval()
11596 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext()); in EmitStructByval()
11600 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty); in EmitStructByval()
11601 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment); in EmitStructByval()
11603 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF), in EmitStructByval()
11607 BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci)) in EmitStructByval()
11613 BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp)) in EmitStructByval()
11620 BB->addSuccessor(loopMBB); in EmitStructByval()
11635 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi) in EmitStructByval()
11638 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi) in EmitStructByval()
11641 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi) in EmitStructByval()
11648 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop, in EmitStructByval()
11650 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop, in EmitStructByval()
11655 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop) in EmitStructByval()
11662 BuildMI(*BB, BB->end(), dl, in EmitStructByval()
11663 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop); in EmitStructByval()
11668 MIB->getOperand(5).setReg(ARM::CPSR); in EmitStructByval()
11669 MIB->getOperand(5).setIsDef(true); in EmitStructByval()
11671 BuildMI(*BB, BB->end(), dl, in EmitStructByval()
11672 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc)) in EmitStructByval()
11676 BB->addSuccessor(loopMBB); in EmitStructByval()
11677 BB->addSuccessor(exitMBB); in EmitStructByval()
11681 auto StartOfExit = exitMBB->begin(); in EmitStructByval()
11707 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); in EmitLowered__chkstk()
11710 assert(Subtarget->isTargetWindows() && in EmitLowered__chkstk()
11712 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode"); in EmitLowered__chkstk()
11720 // thumb-2 environment, so there is no interworking required. As a result, we in EmitLowered__chkstk()
11728 // branches for Thumb), we can generate the long-call version via in EmitLowered__chkstk()
11729 // -mcmodel=large, alleviating the need for the trampoline which may clobber in EmitLowered__chkstk()
11749 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); in EmitLowered__chkstk()
11754 BuildMI(*MBB, MI, DL, TII.get(gettBLXrOpcode(*MBB->getParent()))) in EmitLowered__chkstk()
11782 MachineFunction *MF = MBB->getParent(); in EmitLowered__dbzchk()
11783 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitLowered__dbzchk()
11785 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock(); in EmitLowered__dbzchk()
11786 MF->insert(++MBB->getIterator(), ContBB); in EmitLowered__dbzchk()
11787 ContBB->splice(ContBB->begin(), MBB, in EmitLowered__dbzchk()
11788 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); in EmitLowered__dbzchk()
11789 ContBB->transferSuccessorsAndUpdatePHIs(MBB); in EmitLowered__dbzchk()
11790 MBB->addSuccessor(ContBB); in EmitLowered__dbzchk()
11792 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); in EmitLowered__dbzchk()
11793 BuildMI(TrapBB, DL, TII->get(ARM::t__brkdiv0)); in EmitLowered__dbzchk()
11794 MF->push_back(TrapBB); in EmitLowered__dbzchk()
11795 MBB->addSuccessor(TrapBB); in EmitLowered__dbzchk()
11797 BuildMI(*MBB, MI, DL, TII->get(ARM::tCMPi8)) in EmitLowered__dbzchk()
11801 BuildMI(*MBB, MI, DL, TII->get(ARM::t2Bcc)) in EmitLowered__dbzchk()
11820 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) { in checkAndUpdateCPSRKill()
11825 break; // Should have kill-flag - update below. in checkAndUpdateCPSRKill()
11830 if (miI == BB->end()) { in checkAndUpdateCPSRKill()
11831 for (MachineBasicBlock *Succ : BB->successors()) in checkAndUpdateCPSRKill()
11832 if (Succ->isLiveIn(ARM::CPSR)) in checkAndUpdateCPSRKill()
11838 SelectItr->addRegisterKilled(ARM::CPSR, TRI); in checkAndUpdateCPSRKill()
11851 BuildMI(TpEntry, Dl, TII->get(ARM::t2ADDri), AddDestReg) in genTPEntry()
11858 BuildMI(TpEntry, Dl, TII->get(ARM::t2LSRri), LsrDestReg) in genTPEntry()
11865 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg) in genTPEntry()
11868 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopStart)) in genTPEntry()
11872 BuildMI(TpEntry, Dl, TII->get(ARM::t2B)) in genTPEntry()
11896 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), SrcPhiReg) in genTPLoopBody()
11906 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), DestPhiReg) in genTPLoopBody()
11916 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), LoopCounterPhiReg) in genTPLoopBody()
11925 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), PredCounterPhiReg) in genTPLoopBody()
11933 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VCTP8), VccrReg) in genTPLoopBody()
11939 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2SUBri), RemainingElementsReg) in genTPLoopBody()
11949 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VLDRBU8_post)) in genTPLoopBody()
11960 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VSTRBU8_post)) in genTPLoopBody()
11971 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg) in genTPLoopBody()
11975 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopEnd)) in genTPLoopBody()
11979 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2B)) in genTPLoopBody()
11987 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitInstrWithCustomInserter()
11989 bool isThumb2 = Subtarget->isThumb2(); in EmitInstrWithCustomInserter()
11996 // Thumb1 post-indexed loads are really just single-register LDMs. in EmitInstrWithCustomInserter()
11999 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD)) in EmitInstrWithCustomInserter()
12028 // |-----------------| in EmitInstrWithCustomInserter()
12031 // | TP loop Body MBB<--| in EmitInstrWithCustomInserter()
12037 MachineFunction *MF = BB->getParent(); in EmitInstrWithCustomInserter()
12038 MachineFunctionProperties &Properties = MF->getProperties(); in EmitInstrWithCustomInserter()
12039 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitInstrWithCustomInserter()
12047 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock(); in EmitInstrWithCustomInserter()
12050 MF->push_back(TpLoopBody); in EmitInstrWithCustomInserter()
12062 TpExit = BB->splitAt(MI, false); in EmitInstrWithCustomInserter()
12064 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the " in EmitInstrWithCustomInserter()
12066 TpExit = BB->getFallThrough(); in EmitInstrWithCustomInserter()
12067 BuildMI(BB, dl, TII->get(ARM::t2B)) in EmitInstrWithCustomInserter()
12070 TpExit = BB->splitAt(MI, false); in EmitInstrWithCustomInserter()
12086 TpEntry->addSuccessor(TpLoopBody); in EmitInstrWithCustomInserter()
12087 TpLoopBody->addSuccessor(TpLoopBody); in EmitInstrWithCustomInserter()
12088 TpLoopBody->addSuccessor(TpExit); in EmitInstrWithCustomInserter()
12091 TpLoopBody->moveAfter(TpEntry); in EmitInstrWithCustomInserter()
12092 TpExit->moveAfter(TpLoopBody); in EmitInstrWithCustomInserter()
12102 // The Thumb2 pre-indexed stores have the same MI operands, they just in EmitInstrWithCustomInserter()
12106 MI.setDesc(TII->get(ARM::t2STR_PRE)); in EmitInstrWithCustomInserter()
12109 MI.setDesc(TII->get(ARM::t2STRB_PRE)); in EmitInstrWithCustomInserter()
12112 MI.setDesc(TII->get(ARM::t2STRH_PRE)); in EmitInstrWithCustomInserter()
12124 Offset = -Offset; in EmitInstrWithCustomInserter()
12127 BuildMI(*BB, MI, dl, TII->get(NewOpc)) in EmitInstrWithCustomInserter()
12148 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc)); in EmitInstrWithCustomInserter()
12157 // diamond control-flow pattern. The incoming instruction knows the in EmitInstrWithCustomInserter()
12160 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitInstrWithCustomInserter()
12161 MachineFunction::iterator It = ++BB->getIterator(); in EmitInstrWithCustomInserter()
12168 // fallthrough --> copy0MBB in EmitInstrWithCustomInserter()
12170 MachineFunction *F = BB->getParent(); in EmitInstrWithCustomInserter()
12171 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
12172 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
12173 F->insert(It, copy0MBB); in EmitInstrWithCustomInserter()
12174 F->insert(It, sinkMBB); in EmitInstrWithCustomInserter()
12177 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI); in EmitInstrWithCustomInserter()
12178 copy0MBB->setCallFrameSize(CallFrameSize); in EmitInstrWithCustomInserter()
12179 sinkMBB->setCallFrameSize(CallFrameSize); in EmitInstrWithCustomInserter()
12182 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); in EmitInstrWithCustomInserter()
12185 copy0MBB->addLiveIn(ARM::CPSR); in EmitInstrWithCustomInserter()
12186 sinkMBB->addLiveIn(ARM::CPSR); in EmitInstrWithCustomInserter()
12190 sinkMBB->splice(sinkMBB->begin(), BB, in EmitInstrWithCustomInserter()
12191 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitInstrWithCustomInserter()
12192 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); in EmitInstrWithCustomInserter()
12194 BB->addSuccessor(copy0MBB); in EmitInstrWithCustomInserter()
12195 BB->addSuccessor(sinkMBB); in EmitInstrWithCustomInserter()
12197 BuildMI(BB, dl, TII->get(ARM::tBcc)) in EmitInstrWithCustomInserter()
12207 // Update machine-CFG edges in EmitInstrWithCustomInserter()
12208 BB->addSuccessor(sinkMBB); in EmitInstrWithCustomInserter()
12214 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg()) in EmitInstrWithCustomInserter()
12227 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitInstrWithCustomInserter()
12236 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) in EmitInstrWithCustomInserter()
12240 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) in EmitInstrWithCustomInserter()
12246 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) in EmitInstrWithCustomInserter()
12250 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) in EmitInstrWithCustomInserter()
12260 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)) in EmitInstrWithCustomInserter()
12263 BuildMI(BB, dl, TII->get(ARM::t2B)) in EmitInstrWithCustomInserter()
12267 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB); in EmitInstrWithCustomInserter()
12287 // diamond control-flow pattern. The incoming instruction knows the in EmitInstrWithCustomInserter()
12298 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitInstrWithCustomInserter()
12299 MachineFunction::iterator BBI = ++BB->getIterator(); in EmitInstrWithCustomInserter()
12300 MachineFunction *Fn = BB->getParent(); in EmitInstrWithCustomInserter()
12301 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
12302 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
12303 Fn->insert(BBI, RSBBB); in EmitInstrWithCustomInserter()
12304 Fn->insert(BBI, SinkBB); in EmitInstrWithCustomInserter()
12309 bool isThumb2 = Subtarget->isThumb2(); in EmitInstrWithCustomInserter()
12310 MachineRegisterInfo &MRI = Fn->getRegInfo(); in EmitInstrWithCustomInserter()
12317 SinkBB->splice(SinkBB->begin(), BB, in EmitInstrWithCustomInserter()
12318 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitInstrWithCustomInserter()
12319 SinkBB->transferSuccessorsAndUpdatePHIs(BB); in EmitInstrWithCustomInserter()
12321 BB->addSuccessor(RSBBB); in EmitInstrWithCustomInserter()
12322 BB->addSuccessor(SinkBB); in EmitInstrWithCustomInserter()
12325 RSBBB->addSuccessor(SinkBB); in EmitInstrWithCustomInserter()
12328 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) in EmitInstrWithCustomInserter()
12335 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB) in EmitInstrWithCustomInserter()
12340 // by if-conversion pass in EmitInstrWithCustomInserter()
12341 BuildMI(*RSBBB, RSBBB->begin(), dl, in EmitInstrWithCustomInserter()
12342 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg) in EmitInstrWithCustomInserter()
12350 BuildMI(*SinkBB, SinkBB->begin(), dl, in EmitInstrWithCustomInserter()
12351 TII->get(ARM::PHI), ABSDstReg) in EmitInstrWithCustomInserter()
12372 /// when it is expanded into LDM/STM. This is done as a post-isel lowering
12376 bool isThumb1 = Subtarget->isThumb1Only(); in attachMEMCPYScratchRegs()
12379 MachineFunction *MF = MI.getParent()->getParent(); in attachMEMCPYScratchRegs()
12380 MachineRegisterInfo &MRI = MF->getRegInfo(); in attachMEMCPYScratchRegs()
12384 if (!Node->hasAnyUseOfValue(0)) { in attachMEMCPYScratchRegs()
12387 if (!Node->hasAnyUseOfValue(1)) { in attachMEMCPYScratchRegs()
12412 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR). in AdjustInstrPostInstrSelection()
12418 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo(); in AdjustInstrPostInstrSelection()
12419 MCID = &TII->get(NewOpc); in AdjustInstrPostInstrSelection()
12421 assert(MCID->getNumOperands() == in AdjustInstrPostInstrSelection()
12422 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() in AdjustInstrPostInstrSelection()
12432 if (Subtarget->isThumb1Only()) { in AdjustInstrPostInstrSelection()
12433 for (unsigned c = MCID->getNumOperands() - 4; c--;) { in AdjustInstrPostInstrSelection()
12439 for (unsigned i = MI.getNumOperands(); i--;) { in AdjustInstrPostInstrSelection()
12442 int DefIdx = MCID->getOperandConstraint(i, MCOI::TIED_TO); in AdjustInstrPostInstrSelection()
12443 if (DefIdx != -1) in AdjustInstrPostInstrSelection()
12452 ccOutIdx = MCID->getNumOperands() - 1; in AdjustInstrPostInstrSelection()
12454 ccOutIdx = MCID->getNumOperands() - 1; in AdjustInstrPostInstrSelection()
12458 if (!MI.hasOptionalDef() || !MCID->operands()[ccOutIdx].isOptionalDef()) { in AdjustInstrPostInstrSelection()
12466 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e; in AdjustInstrPostInstrSelection()
12481 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag"); in AdjustInstrPostInstrSelection()
12486 if (!Subtarget->isThumb1Only()) in AdjustInstrPostInstrSelection()
12497 //===----------------------------------------------------------------------===//
12499 //===----------------------------------------------------------------------===//
12513 // (select cc -1, y) [AllOnes=1]
12514 // (select cc y, -1) [AllOnes=1]
12522 switch (N->getOpcode()) { in isConditionalZeroOrAllOnes()
12525 CC = N->getOperand(0); in isConditionalZeroOrAllOnes()
12526 SDValue N1 = N->getOperand(1); in isConditionalZeroOrAllOnes()
12527 SDValue N2 = N->getOperand(2); in isConditionalZeroOrAllOnes()
12547 EVT VT = N->getValueType(0); in isConditionalZeroOrAllOnes()
12548 CC = N->getOperand(0); in isConditionalZeroOrAllOnes()
12556 else if (N->getOpcode() == ISD::ZERO_EXTEND) in isConditionalZeroOrAllOnes()
12568 // (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12569 // (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
12570 // (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
12571 // (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
12572 // (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
12579 // (add (zext cc), x) -> (select cc (add x, 1), x)
12580 // (add (sext cc), x) -> (select cc (add x, -1), x)
12595 EVT VT = N->getValueType(0); in combineSelectAndUse()
12605 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, in combineSelectAndUse()
12619 SDValue N0 = N->getOperand(0); in combineSelectAndUseCommutative()
12620 SDValue N1 = N->getOperand(1); in combineSelectAndUseCommutative()
12621 if (N0.getNode()->hasOneUse()) in combineSelectAndUseCommutative()
12624 if (N1.getNode()->hasOneUse()) in combineSelectAndUseCommutative()
12632 if (N->getOpcode() == ARMISD::VUZP) in IsVUZPShuffleNode()
12636 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32) in IsVUZPShuffleNode()
12650 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD. in AddCombineToVPADD()
12651 if (!N->getValueType(0).is64BitVector()) in AddCombineToVPADD()
12659 EVT VT = N->getValueType(0); in AddCombineToVPADD()
12664 Ops.push_back(Unzip->getOperand(0)); in AddCombineToVPADD()
12665 Ops.push_back(Unzip->getOperand(1)); in AddCombineToVPADD()
12698 EVT VT = N->getValueType(0); in AddCombineVUZPToVPADDL()
12728 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON() in AddCombineBUILD_VECTORToVPADDL()
12734 EVT VT = N->getValueType(0); in AddCombineBUILD_VECTORToVPADDL()
12745 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in AddCombineBUILD_VECTORToVPADDL()
12747 SDValue Vec = N0->getOperand(0)->getOperand(0); in AddCombineBUILD_VECTORToVPADDL()
12754 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) { in AddCombineBUILD_VECTORToVPADDL()
12755 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT in AddCombineBUILD_VECTORToVPADDL()
12756 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in AddCombineBUILD_VECTORToVPADDL()
12758 SDValue ExtVec0 = N0->getOperand(i); in AddCombineBUILD_VECTORToVPADDL()
12759 SDValue ExtVec1 = N1->getOperand(i); in AddCombineBUILD_VECTORToVPADDL()
12762 if (V != ExtVec0->getOperand(0).getNode() || in AddCombineBUILD_VECTORToVPADDL()
12763 V != ExtVec1->getOperand(0).getNode()) in AddCombineBUILD_VECTORToVPADDL()
12767 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1)); in AddCombineBUILD_VECTORToVPADDL()
12768 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1)); in AddCombineBUILD_VECTORToVPADDL()
12771 if (!C0 || !C1 || C0->getZExtValue() != nextIndex in AddCombineBUILD_VECTORToVPADDL()
12772 || C1->getZExtValue() != nextIndex+1) in AddCombineBUILD_VECTORToVPADDL()
12821 if (V->getOpcode() == ISD::UMUL_LOHI || in findMUL_LOHI()
12822 V->getOpcode() == ISD::SMUL_LOHI) in findMUL_LOHI()
12830 if (!Subtarget->hasBaseDSP()) in AddCombineTo64BitSMLAL16()
12833 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and in AddCombineTo64BitSMLAL16()
12834 // accumulates the product into a 64-bit value. The 16-bit values will in AddCombineTo64BitSMLAL16()
12835 // be sign extended somehow or SRA'd into 32-bit values in AddCombineTo64BitSMLAL16()
12837 SDValue Mul = AddcNode->getOperand(0); in AddCombineTo64BitSMLAL16()
12838 SDValue Lo = AddcNode->getOperand(1); in AddCombineTo64BitSMLAL16()
12840 Lo = AddcNode->getOperand(0); in AddCombineTo64BitSMLAL16()
12841 Mul = AddcNode->getOperand(1); in AddCombineTo64BitSMLAL16()
12846 SDValue SRA = AddeNode->getOperand(0); in AddCombineTo64BitSMLAL16()
12847 SDValue Hi = AddeNode->getOperand(1); in AddCombineTo64BitSMLAL16()
12849 SRA = AddeNode->getOperand(1); in AddCombineTo64BitSMLAL16()
12850 Hi = AddeNode->getOperand(0); in AddCombineTo64BitSMLAL16()
12855 if (Const->getZExtValue() != 31) in AddCombineTo64BitSMLAL16()
12883 Op0 = Mul->getOperand(0).getOperand(0); in AddCombineTo64BitSMLAL16()
12884 Op1 = Mul->getOperand(1).getOperand(0); in AddCombineTo64BitSMLAL16()
12916 // loAdd -> ADDC | in AddCombineTo64bitMLAL()
12919 // ADDE <- hiAdd in AddCombineTo64bitMLAL()
12927 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE || in AddCombineTo64bitMLAL()
12928 AddeSubeNode->getOpcode() == ARMISD::SUBE) && in AddCombineTo64bitMLAL()
12931 assert(AddeSubeNode->getNumOperands() == 3 && in AddCombineTo64bitMLAL()
12932 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 && in AddCombineTo64bitMLAL()
12936 SDNode *AddcSubcNode = AddeSubeNode->getOperand(2).getNode(); in AddCombineTo64bitMLAL()
12937 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE && in AddCombineTo64bitMLAL()
12938 AddcSubcNode->getOpcode() != ARMISD::ADDC) || in AddCombineTo64bitMLAL()
12939 (AddeSubeNode->getOpcode() == ARMISD::SUBE && in AddCombineTo64bitMLAL()
12940 AddcSubcNode->getOpcode() != ARMISD::SUBC)) in AddCombineTo64bitMLAL()
12943 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0); in AddCombineTo64bitMLAL()
12944 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1); in AddCombineTo64bitMLAL()
12950 assert(AddcSubcNode->getNumValues() == 2 && in AddCombineTo64bitMLAL()
12951 AddcSubcNode->getValueType(0) == MVT::i32 && in AddCombineTo64bitMLAL()
12955 // maybe a SMLAL which multiplies two 16-bit values. in AddCombineTo64bitMLAL()
12956 if (AddeSubeNode->getOpcode() == ARMISD::ADDE && in AddCombineTo64bitMLAL()
12957 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI && in AddCombineTo64bitMLAL()
12958 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI && in AddCombineTo64bitMLAL()
12959 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI && in AddCombineTo64bitMLAL()
12960 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI) in AddCombineTo64bitMLAL()
12964 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0); in AddCombineTo64bitMLAL()
12965 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1); in AddCombineTo64bitMLAL()
12982 unsigned Opc = MULOp->getOpcode(); in AddCombineTo64bitMLAL()
13016 if (AddcSubcNode == HiAddSub->getNode() || in AddCombineTo64bitMLAL()
13017 AddcSubcNode->isPredecessorOf(HiAddSub->getNode())) in AddCombineTo64bitMLAL()
13025 Ops.push_back(LoMul->getOperand(0)); in AddCombineTo64bitMLAL()
13026 Ops.push_back(LoMul->getOperand(1)); in AddCombineTo64bitMLAL()
13032 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() && in AddCombineTo64bitMLAL()
13033 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(1) && in AddCombineTo64bitMLAL()
13034 LowAddSub->getNode()->getOpcode() == ISD::Constant && in AddCombineTo64bitMLAL()
13035 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() == in AddCombineTo64bitMLAL()
13038 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) { in AddCombineTo64bitMLAL()
13047 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC) in AddCombineTo64bitMLAL()
13079 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) in AddCombineTo64bitUMAAL()
13083 SDNode* AddcNode = AddeNode->getOperand(2).getNode(); in AddCombineTo64bitUMAAL()
13084 if (AddcNode->getOpcode() != ARMISD::ADDC) in AddCombineTo64bitUMAAL()
13090 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) { in AddCombineTo64bitUMAAL()
13091 UmlalNode = AddcNode->getOperand(0).getNode(); in AddCombineTo64bitUMAAL()
13092 AddHi = AddcNode->getOperand(1); in AddCombineTo64bitUMAAL()
13093 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) { in AddCombineTo64bitUMAAL()
13094 UmlalNode = AddcNode->getOperand(1).getNode(); in AddCombineTo64bitUMAAL()
13095 AddHi = AddcNode->getOperand(0); in AddCombineTo64bitUMAAL()
13102 if (!isNullConstant(UmlalNode->getOperand(3))) in AddCombineTo64bitUMAAL()
13105 if ((isNullConstant(AddeNode->getOperand(0)) && in AddCombineTo64bitUMAAL()
13106 AddeNode->getOperand(1).getNode() == UmlalNode) || in AddCombineTo64bitUMAAL()
13107 (AddeNode->getOperand(0).getNode() == UmlalNode && in AddCombineTo64bitUMAAL()
13108 isNullConstant(AddeNode->getOperand(1)))) { in AddCombineTo64bitUMAAL()
13110 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1), in AddCombineTo64bitUMAAL()
13111 UmlalNode->getOperand(2), AddHi }; in AddCombineTo64bitUMAAL()
13127 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP()) in PerformUMLALCombine()
13132 SDNode* AddcNode = N->getOperand(2).getNode(); in PerformUMLALCombine()
13133 SDNode* AddeNode = N->getOperand(3).getNode(); in PerformUMLALCombine()
13134 if ((AddcNode->getOpcode() == ARMISD::ADDC) && in PerformUMLALCombine()
13135 (AddeNode->getOpcode() == ARMISD::ADDE) && in PerformUMLALCombine()
13136 isNullConstant(AddeNode->getOperand(0)) && in PerformUMLALCombine()
13137 isNullConstant(AddeNode->getOperand(1)) && in PerformUMLALCombine()
13138 (AddeNode->getOperand(2).getNode() == AddcNode)) in PerformUMLALCombine()
13141 {N->getOperand(0), N->getOperand(1), in PerformUMLALCombine()
13142 AddcNode->getOperand(0), AddcNode->getOperand(1)}); in PerformUMLALCombine()
13152 if (N->getOpcode() == ARMISD::SUBC && N->hasAnyUseOfValue(1)) { in PerformAddcSubcCombine()
13153 // (SUBC (ADDE 0, 0, C), 1) -> C in PerformAddcSubcCombine()
13154 SDValue LHS = N->getOperand(0); in PerformAddcSubcCombine()
13155 SDValue RHS = N->getOperand(1); in PerformAddcSubcCombine()
13156 if (LHS->getOpcode() == ARMISD::ADDE && in PerformAddcSubcCombine()
13157 isNullConstant(LHS->getOperand(0)) && in PerformAddcSubcCombine()
13158 isNullConstant(LHS->getOperand(1)) && isOneConstant(RHS)) { in PerformAddcSubcCombine()
13159 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2)); in PerformAddcSubcCombine()
13163 if (Subtarget->isThumb1Only()) { in PerformAddcSubcCombine()
13164 SDValue RHS = N->getOperand(1); in PerformAddcSubcCombine()
13166 int32_t imm = C->getSExtValue(); in PerformAddcSubcCombine()
13169 RHS = DAG.getConstant(-imm, DL, MVT::i32); in PerformAddcSubcCombine()
13170 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC in PerformAddcSubcCombine()
13172 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS); in PerformAddcSubcCombine()
13183 if (Subtarget->isThumb1Only()) { in PerformAddeSubeCombine()
13185 SDValue RHS = N->getOperand(1); in PerformAddeSubeCombine()
13187 int64_t imm = C->getSExtValue(); in PerformAddeSubeCombine()
13191 // The with-carry-in form matches bitwise not instead of the negation. in PerformAddeSubeCombine()
13196 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE in PerformAddeSubeCombine()
13198 return DAG.getNode(Opcode, DL, N->getVTList(), in PerformAddeSubeCombine()
13199 N->getOperand(0), RHS, N->getOperand(2)); in PerformAddeSubeCombine()
13202 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) { in PerformAddeSubeCombine()
13211 if (!Subtarget->hasMVEIntegerOps()) in PerformSELECTCombine()
13222 if (N->getOpcode() == ISD::SELECT && in PerformSELECTCombine()
13223 N->getOperand(0)->getOpcode() == ISD::SETCC) { in PerformSELECTCombine()
13224 SetCC = N->getOperand(0); in PerformSELECTCombine()
13225 LHS = SetCC->getOperand(0); in PerformSELECTCombine()
13226 RHS = SetCC->getOperand(1); in PerformSELECTCombine()
13227 CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get(); in PerformSELECTCombine()
13228 TrueVal = N->getOperand(1); in PerformSELECTCombine()
13229 FalseVal = N->getOperand(2); in PerformSELECTCombine()
13230 } else if (N->getOpcode() == ISD::SELECT_CC) { in PerformSELECTCombine()
13231 LHS = N->getOperand(0); in PerformSELECTCombine()
13232 RHS = N->getOperand(1); in PerformSELECTCombine()
13233 CC = cast<CondCodeSDNode>(N->getOperand(4))->get(); in PerformSELECTCombine()
13234 TrueVal = N->getOperand(2); in PerformSELECTCombine()
13235 FalseVal = N->getOperand(3); in PerformSELECTCombine()
13241 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN || in PerformSELECTCombine()
13242 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) && in PerformSELECTCombine()
13247 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN || in PerformSELECTCombine()
13248 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) && in PerformSELECTCombine()
13253 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX || in PerformSELECTCombine()
13254 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) && in PerformSELECTCombine()
13259 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX || in PerformSELECTCombine()
13260 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) && in PerformSELECTCombine()
13269 switch (TrueVal->getOpcode()) { in PerformSELECTCombine()
13279 EVT VectorType = FalseVal->getOperand(0).getValueType(); in PerformSELECTCombine()
13291 EVT LeftType = LHS->getValueType(0); in PerformSELECTCombine()
13292 EVT RightType = RHS->getValueType(0); in PerformSELECTCombine()
13304 DCI.DAG.getNode(Opcode, dl, MVT::i32, LHS, RHS->getOperand(0)); in PerformSELECTCombine()
13320 EVT VT = N->getValueType(0); in PerformVQDMULHCombine()
13327 if (N->getOpcode() == ISD::SMIN) { in PerformVQDMULHCombine()
13328 Shft = N->getOperand(0); in PerformVQDMULHCombine()
13329 Clamp = isConstOrConstSplat(N->getOperand(1)); in PerformVQDMULHCombine()
13330 } else if (N->getOpcode() == ISD::VSELECT) { in PerformVQDMULHCombine()
13332 SDValue Cmp = N->getOperand(0); in PerformVQDMULHCombine()
13334 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() != ISD::SETLT || in PerformVQDMULHCombine()
13335 Cmp.getOperand(0) != N->getOperand(1) || in PerformVQDMULHCombine()
13336 Cmp.getOperand(1) != N->getOperand(2)) in PerformVQDMULHCombine()
13338 Shft = N->getOperand(1); in PerformVQDMULHCombine()
13339 Clamp = isConstOrConstSplat(N->getOperand(2)); in PerformVQDMULHCombine()
13348 switch (Clamp->getSExtValue()) { in PerformVQDMULHCombine()
13349 case (1 << 7) - 1: in PerformVQDMULHCombine()
13353 case (1 << 15) - 1: in PerformVQDMULHCombine()
13357 case (1ULL << 31) - 1: in PerformVQDMULHCombine()
13368 if (!N1 || N1->getSExtValue() != ShftAmt) in PerformVQDMULHCombine()
13430 if (!Subtarget->hasMVEIntegerOps()) in PerformVSELECTCombine()
13438 // We need to re-implement this optimization here as the implementation in the in PerformVSELECTCombine()
13439 // Target-Independent DAGCombiner does not handle the kind of constant we make in PerformVSELECTCombine()
13440 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for in PerformVSELECTCombine()
13445 if (N->getOperand(0).getOpcode() != ISD::XOR) in PerformVSELECTCombine()
13447 SDValue XOR = N->getOperand(0); in PerformVSELECTCombine()
13453 isConstOrConstSplat(XOR->getOperand(1), /*AllowUndefs*/ false, in PerformVSELECTCombine()
13455 if (!Const || !Const->isOne()) in PerformVSELECTCombine()
13459 SDValue Cond = XOR->getOperand(0); in PerformVSELECTCombine()
13460 SDValue LHS = N->getOperand(1); in PerformVSELECTCombine()
13461 SDValue RHS = N->getOperand(2); in PerformVSELECTCombine()
13462 EVT Type = N->getValueType(0); in PerformVSELECTCombine()
13466 // Convert vsetcc([0,1,2,..], splat(n), ult) -> vctp n
13470 SDValue Op0 = N->getOperand(0); in PerformVSetCCToVCTPCombine()
13471 SDValue Op1 = N->getOperand(1); in PerformVSetCCToVCTPCombine()
13472 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); in PerformVSetCCToVCTPCombine()
13473 EVT VT = N->getValueType(0); in PerformVSetCCToVCTPCombine()
13475 if (!Subtarget->hasMVEIntegerOps() || in PerformVSetCCToVCTPCombine()
13525 /// PerformADDECombine - Target-specific dag combine transform from
13532 if (Subtarget->isThumb1Only()) in PerformADDECombine()
13541 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13559 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) in PerformADDCombineWithOperands()
13560 if (N0.getNode()->hasOneUse()) in PerformADDCombineWithOperands()
13567 EVT VT = N->getValueType(0); in TryDistrubutionADDVecReduce()
13568 SDValue N0 = N->getOperand(0); in TryDistrubutionADDVecReduce()
13569 SDValue N1 = N->getOperand(1); in TryDistrubutionADDVecReduce()
13585 // Distribute add(X, add(vecreduce(Y), vecreduce(Z))) -> in TryDistrubutionADDVecReduce()
13590 !isa<ConstantSDNode>(N0) && N1->hasOneUse()) { in TryDistrubutionADDVecReduce()
13594 // And turn add(add(A, reduce(B)), add(C, reduce(D))) -> in TryDistrubutionADDVecReduce()
13597 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) { in TryDistrubutionADDVecReduce()
13611 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp), in TryDistrubutionADDVecReduce()
13612 N1.getOperand(1 - N1RedOp)); in TryDistrubutionADDVecReduce()
13644 if (!Load0 || !Load1 || Load0->getChain() != Load1->getChain() || in TryDistrubutionADDVecReduce()
13645 !Load0->isSimple() || !Load1->isSimple() || Load0->isIndexed() || in TryDistrubutionADDVecReduce()
13646 Load1->isIndexed()) in TryDistrubutionADDVecReduce()
13657 return -1; in TryDistrubutionADDVecReduce()
13664 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) { in TryDistrubutionADDVecReduce()
13713 if (!Subtarget->hasMVEIntegerOps()) in PerformADDVecReduce()
13719 EVT VT = N->getValueType(0); in PerformADDVecReduce()
13720 SDValue N0 = N->getOperand(0); in PerformADDVecReduce()
13721 SDValue N1 = N->getOperand(1); in PerformADDVecReduce()
13737 if (NB->getOpcode() != ISD::BUILD_PAIR) in PerformADDVecReduce()
13739 SDValue VecRed = NB->getOperand(0); in PerformADDVecReduce()
13740 if ((VecRed->getOpcode() != Opcode && VecRed->getOpcode() != OpcodeA) || in PerformADDVecReduce()
13742 NB->getOperand(1) != SDValue(VecRed.getNode(), 1)) in PerformADDVecReduce()
13745 if (VecRed->getOpcode() == OpcodeA) { in PerformADDVecReduce()
13746 // add(NA, VADDLVA(Inp), Y) -> VADDLVA(add(NA, Inp), Y) in PerformADDVecReduce()
13755 unsigned S = VecRed->getOpcode() == OpcodeA ? 2 : 0; in PerformADDVecReduce()
13757 Ops.push_back(VecRed->getOperand(I)); in PerformADDVecReduce()
13802 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
13803 N->getOpcode() == ISD::SRL) && in isDesirableToCommuteWithShift()
13809 if (N->getOpcode() != ISD::SHL) in isDesirableToCommuteWithShift()
13812 if (Subtarget->isThumb1Only()) { in isDesirableToCommuteWithShift()
13816 if (N->getOpcode() != ISD::SHL) in isDesirableToCommuteWithShift()
13818 SDValue N1 = N->getOperand(0); in isDesirableToCommuteWithShift()
13819 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND && in isDesirableToCommuteWithShift()
13820 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR) in isDesirableToCommuteWithShift()
13822 if (auto *Const = dyn_cast<ConstantSDNode>(N1->getOperand(1))) { in isDesirableToCommuteWithShift()
13823 if (Const->getAPIntValue().ult(256)) in isDesirableToCommuteWithShift()
13825 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(0) && in isDesirableToCommuteWithShift()
13826 Const->getAPIntValue().sgt(-256)) in isDesirableToCommuteWithShift()
13832 // Turn off commute-with-shift transform after legalization, so it doesn't in isDesirableToCommuteWithShift()
13841 assert(N->getOpcode() == ISD::XOR && in isDesirableToCommuteXorWithShift()
13842 (N->getOperand(0).getOpcode() == ISD::SHL || in isDesirableToCommuteXorWithShift()
13843 N->getOperand(0).getOpcode() == ISD::SRL) && in isDesirableToCommuteXorWithShift()
13847 auto *XorC = dyn_cast<ConstantSDNode>(N->getOperand(1)); in isDesirableToCommuteXorWithShift()
13848 auto *ShiftC = dyn_cast<ConstantSDNode>(N->getOperand(0).getOperand(1)); in isDesirableToCommuteXorWithShift()
13851 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) { in isDesirableToCommuteXorWithShift()
13852 unsigned ShiftAmt = ShiftC->getZExtValue(); in isDesirableToCommuteXorWithShift()
13853 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits(); in isDesirableToCommuteXorWithShift()
13854 if (N->getOperand(0).getOpcode() == ISD::SHL) in isDesirableToCommuteXorWithShift()
13855 return MaskIdx == ShiftAmt && MaskLen == (BitWidth - ShiftAmt); in isDesirableToCommuteXorWithShift()
13856 return MaskIdx == 0 && MaskLen == (BitWidth - ShiftAmt); in isDesirableToCommuteXorWithShift()
13865 assert(((N->getOpcode() == ISD::SHL && in shouldFoldConstantShiftPairToMask()
13866 N->getOperand(0).getOpcode() == ISD::SRL) || in shouldFoldConstantShiftPairToMask()
13867 (N->getOpcode() == ISD::SRL && in shouldFoldConstantShiftPairToMask()
13868 N->getOperand(0).getOpcode() == ISD::SHL)) && in shouldFoldConstantShiftPairToMask()
13869 "Expected shift-shift mask"); in shouldFoldConstantShiftPairToMask()
13871 if (!Subtarget->isThumb1Only()) in shouldFoldConstantShiftPairToMask()
13882 return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT); in shouldFoldSelectWithIdentityConstant()
13886 if (!Subtarget->hasNEON()) { in preferIncOfAddToSubOfNot()
13887 if (Subtarget->isThumb1Only()) in preferIncOfAddToSubOfNot()
13901 return Subtarget->hasVFP2Base(); in shouldConvertFpToSat()
13903 return Subtarget->hasVFP2Base(); in shouldConvertFpToSat()
13905 return Subtarget->hasFP64(); in shouldConvertFpToSat()
13908 return Subtarget->hasMVEFloatOps(); in shouldConvertFpToSat()
13922 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2) in PerformSHLSimplify()
13923 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2 in PerformSHLSimplify()
13933 // - if c1 and c2 are small enough that they don't require mov imms. in PerformSHLSimplify()
13934 // - the user(s) of the node can perform an shl in PerformSHLSimplify()
13936 // No shifted operands for 16-bit instructions. in PerformSHLSimplify()
13937 if (ST->isThumb() && ST->isThumb1Only()) in PerformSHLSimplify()
13941 for (auto *U : N->uses()) { in PerformSHLSimplify()
13942 switch(U->getOpcode()) { in PerformSHLSimplify()
13955 if (isa<ConstantSDNode>(U->getOperand(0)) || in PerformSHLSimplify()
13956 isa<ConstantSDNode>(U->getOperand(1))) in PerformSHLSimplify()
13960 if (U->getOperand(0).getOpcode() == ISD::SHL || in PerformSHLSimplify()
13961 U->getOperand(1).getOpcode() == ISD::SHL) in PerformSHLSimplify()
13967 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR && in PerformSHLSimplify()
13968 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND) in PerformSHLSimplify()
13971 if (N->getOperand(0).getOpcode() != ISD::SHL) in PerformSHLSimplify()
13974 SDValue SHL = N->getOperand(0); in PerformSHLSimplify()
13976 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(N->getOperand(1)); in PerformSHLSimplify()
13981 APInt C2Int = C2->getAPIntValue(); in PerformSHLSimplify()
13982 APInt C1Int = C1ShlC2->getAPIntValue(); in PerformSHLSimplify()
13989 APInt Mask = APInt::getHighBitsSet(C2Width, C2Width - C2Value); in PerformSHLSimplify()
13996 // The immediates are encoded as an 8-bit value that can be rotated. in PerformSHLSimplify()
13999 return Imm.getBitWidth() - Zeros > 8; in PerformSHLSimplify()
14008 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X, in PerformSHLSimplify()
14014 SHL.dump(); N->dump()); in PerformSHLSimplify()
14020 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
14025 SDValue N0 = N->getOperand(0); in PerformADDCombine()
14026 SDValue N1 = N->getOperand(1); in PerformADDCombine()
14043 // Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
14044 // providing -X is as cheap as X (currently, just a constant).
14046 if (N->getValueType(0) != MVT::i32 || !isNullConstant(N->getOperand(0))) in PerformSubCSINCCombine()
14048 SDValue CSINC = N->getOperand(1); in PerformSubCSINCCombine()
14057 DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32, N->getOperand(0), in PerformSubCSINCCombine()
14063 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
14068 SDValue N0 = N->getOperand(0); in PerformSUBCombine()
14069 SDValue N1 = N->getOperand(1); in PerformSUBCombine()
14071 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) in PerformSUBCombine()
14072 if (N1.getNode()->hasOneUse()) in PerformSUBCombine()
14079 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(0).isVector()) in PerformSUBCombine()
14082 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x)) in PerformSUBCombine()
14085 SDValue VDup = N->getOperand(1); in PerformSUBCombine()
14086 if (VDup->getOpcode() != ARMISD::VDUP) in PerformSUBCombine()
14089 SDValue VMov = N->getOperand(0); in PerformSUBCombine()
14090 if (VMov->getOpcode() == ISD::BITCAST) in PerformSUBCombine()
14091 VMov = VMov->getOperand(0); in PerformSUBCombine()
14093 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(VMov)) in PerformSUBCombine()
14099 VDup->getOperand(0)); in PerformSUBCombine()
14100 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), Negate); in PerformSUBCombine()
14121 if (!Subtarget->hasVMLxForwarding()) in PerformVMULCombine()
14125 SDValue N0 = N->getOperand(0); in PerformVMULCombine()
14126 SDValue N1 = N->getOperand(1); in PerformVMULCombine()
14140 EVT VT = N->getValueType(0); in PerformVMULCombine()
14142 SDValue N00 = N0->getOperand(0); in PerformVMULCombine()
14143 SDValue N01 = N0->getOperand(1); in PerformVMULCombine()
14151 EVT VT = N->getValueType(0); in PerformMVEVMULLCombine()
14155 SDValue N0 = N->getOperand(0); in PerformMVEVMULLCombine()
14156 SDValue N1 = N->getOperand(1); in PerformMVEVMULLCombine()
14159 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG) in PerformMVEVMULLCombine()
14161 EVT VT = cast<VTSDNode>(Op->getOperand(1))->getVT(); in PerformMVEVMULLCombine()
14163 return Op->getOperand(0); in PerformMVEVMULLCombine()
14168 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask. in PerformMVEVMULLCombine()
14172 if (!Subtarget->isLittle()) in PerformMVEVMULLCombine()
14176 if (And->getOpcode() == ISD::BITCAST) in PerformMVEVMULLCombine()
14177 And = And->getOperand(0); in PerformMVEVMULLCombine()
14178 if (And->getOpcode() != ISD::AND) in PerformMVEVMULLCombine()
14180 SDValue Mask = And->getOperand(1); in PerformMVEVMULLCombine()
14181 if (Mask->getOpcode() == ISD::BITCAST) in PerformMVEVMULLCombine()
14182 Mask = Mask->getOperand(0); in PerformMVEVMULLCombine()
14184 if (Mask->getOpcode() != ISD::BUILD_VECTOR || in PerformMVEVMULLCombine()
14187 if (isAllOnesConstant(Mask->getOperand(0)) && in PerformMVEVMULLCombine()
14188 isNullConstant(Mask->getOperand(1)) && in PerformMVEVMULLCombine()
14189 isAllOnesConstant(Mask->getOperand(2)) && in PerformMVEVMULLCombine()
14190 isNullConstant(Mask->getOperand(3))) in PerformMVEVMULLCombine()
14191 return And->getOperand(0); in PerformMVEVMULLCombine()
14219 EVT VT = N->getValueType(0); in PerformMULCombine()
14220 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64) in PerformMULCombine()
14223 if (Subtarget->isThumb1Only()) in PerformMULCombine()
14234 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); in PerformMULCombine()
14238 int64_t MulAmt = C->getSExtValue(); in PerformMULCombine()
14241 ShiftAmt = ShiftAmt & (32 - 1); in PerformMULCombine()
14242 SDValue V = N->getOperand(0); in PerformMULCombine()
14249 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) { in PerformMULCombine()
14255 DAG.getConstant(Log2_32(MulAmt - 1), DL, in PerformMULCombine()
14258 // (mul x, 2^N - 1) => (sub (shl x, N), x) in PerformMULCombine()
14268 uint64_t MulAmtAbs = -MulAmt; in PerformMULCombine()
14270 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) in PerformMULCombine()
14277 } else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) { in PerformMULCombine()
14278 // (mul x, -(2^N + 1)) => - (add (shl x, N), x) in PerformMULCombine()
14283 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL, in PerformMULCombine()
14303 // Allow DAGCombine to pattern-match before we touch the canonical form. in CombineANDShift()
14307 if (N->getValueType(0) != MVT::i32) in CombineANDShift()
14310 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)); in CombineANDShift()
14314 uint32_t C1 = (uint32_t)N1C->getZExtValue(); in CombineANDShift()
14319 SDNode *N0 = N->getOperand(0).getNode(); in CombineANDShift()
14320 if (!N0->hasOneUse()) in CombineANDShift()
14323 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL) in CombineANDShift()
14326 bool LeftShift = N0->getOpcode() == ISD::SHL; in CombineANDShift()
14328 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1)); in CombineANDShift()
14332 uint32_t C2 = (uint32_t)N01C->getZExtValue(); in CombineANDShift()
14338 C1 &= (-1U << C2); in CombineANDShift()
14340 C1 &= (-1U >> C2); in CombineANDShift()
14354 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14355 DAG.getConstant(C3 - C2, DL, MVT::i32)); in CombineANDShift()
14365 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14366 DAG.getConstant(C3 - C2, DL, MVT::i32)); in CombineANDShift()
14378 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14391 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14403 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14415 // Attempt to use immediate-form VBIC in PerformANDCombine()
14416 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); in PerformANDCombine()
14418 EVT VT = N->getValueType(0); in PerformANDCombine()
14428 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) && in PerformANDCombine()
14429 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { in PerformANDCombine()
14438 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0)); in PerformANDCombine()
14445 if (!Subtarget->isThumb1Only()) { in PerformANDCombine()
14446 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) in PerformANDCombine()
14454 if (Subtarget->isThumb1Only()) in PerformANDCombine()
14465 if (!Subtarget->hasV6Ops() || in PerformORCombineToSMULWBT()
14466 (Subtarget->isThumb() && in PerformORCombineToSMULWBT()
14467 (!Subtarget->hasThumb2() || !Subtarget->hasDSP()))) in PerformORCombineToSMULWBT()
14470 SDValue SRL = OR->getOperand(0); in PerformORCombineToSMULWBT()
14471 SDValue SHL = OR->getOperand(1); in PerformORCombineToSMULWBT()
14474 SRL = OR->getOperand(1); in PerformORCombineToSMULWBT()
14475 SHL = OR->getOperand(0); in PerformORCombineToSMULWBT()
14493 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments. in PerformORCombineToSMULWBT()
14494 // For SMUWB the 16-bit value will signed extended somehow. in PerformORCombineToSMULWBT()
14497 SDValue OpS16 = SMULLOHI->getOperand(0); in PerformORCombineToSMULWBT()
14498 SDValue OpS32 = SMULLOHI->getOperand(1); in PerformORCombineToSMULWBT()
14503 OpS32 = SMULLOHI->getOperand(0); in PerformORCombineToSMULWBT()
14512 OpS16 = OpS16->getOperand(0); in PerformORCombineToSMULWBT()
14526 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops()) in PerformORCombineToBFI()
14529 EVT VT = N->getValueType(0); in PerformORCombineToBFI()
14530 SDValue N0 = N->getOperand(0); in PerformORCombineToBFI()
14531 SDValue N1 = N->getOperand(1); in PerformORCombineToBFI()
14556 unsigned Mask = MaskC->getZExtValue(); in PerformORCombineToBFI()
14563 unsigned Val = N1C->getZExtValue(); in PerformORCombineToBFI()
14584 unsigned Mask2 = N11C->getZExtValue(); in PerformORCombineToBFI()
14592 if (Subtarget->hasDSP() && in PerformORCombineToBFI()
14609 if (Subtarget->hasDSP() && in PerformORCombineToBFI()
14625 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) && in PerformORCombineToBFI()
14631 unsigned ShAmtC = ShAmt->getAsZExtVal(); in PerformORCombineToBFI()
14666 if (N->getOpcode() == ARMISD::VCMP) in getVCMPCondCode()
14667 return (ARMCC::CondCodes)N->getConstantOperandVal(2); in getVCMPCondCode()
14668 else if (N->getOpcode() == ARMISD::VCMPZ) in getVCMPCondCode()
14669 return (ARMCC::CondCodes)N->getConstantOperandVal(1); in getVCMPCondCode()
14676 return isValidMVECond(CC, N->getOperand(0).getValueType().isFloatingPoint()); in CanInvertMVEVCMP()
14681 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain in PerformORCombine_i1()
14683 EVT VT = N->getValueType(0); in PerformORCombine_i1()
14685 SDValue N0 = N->getOperand(0); in PerformORCombine_i1()
14686 SDValue N1 = N->getOperand(1); in PerformORCombine_i1()
14689 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ) in PerformORCombine_i1()
14704 /// PerformORCombine - Target-specific dag combine xforms for ISD::OR
14708 // Attempt to use immediate-form VORR in PerformORCombine()
14709 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); in PerformORCombine()
14711 EVT VT = N->getValueType(0); in PerformORCombine()
14717 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 || in PerformORCombine()
14724 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) && in PerformORCombine()
14725 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { in PerformORCombine()
14734 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0)); in PerformORCombine()
14741 if (!Subtarget->isThumb1Only()) { in PerformORCombine()
14742 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c)) in PerformORCombine()
14749 SDValue N0 = N->getOperand(0); in PerformORCombine()
14750 SDValue N1 = N->getOperand(1); in PerformORCombine()
14753 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() && in PerformORCombine()
14767 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1)); in PerformORCombine()
14768 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1)); in PerformORCombine()
14770 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize, in PerformORCombine()
14772 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize, in PerformORCombine()
14783 N0->getOperand(1), in PerformORCombine()
14784 N0->getOperand(0), in PerformORCombine()
14785 N1->getOperand(0)); in PerformORCombine()
14808 EVT VT = N->getValueType(0); in PerformXORCombine()
14814 if (!Subtarget->isThumb1Only()) { in PerformXORCombine()
14815 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c)) in PerformXORCombine()
14823 if (Subtarget->hasMVEIntegerOps()) { in PerformXORCombine()
14825 SDValue N0 = N->getOperand(0); in PerformXORCombine()
14826 SDValue N1 = N->getOperand(1); in PerformXORCombine()
14827 const TargetLowering *TLI = Subtarget->getTargetLowering(); in PerformXORCombine()
14828 if (TLI->isConstTrueVal(N1) && in PerformXORCombine()
14829 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) { in PerformXORCombine()
14835 Ops.push_back(N0->getOperand(0)); in PerformXORCombine()
14836 if (N0->getOpcode() == ARMISD::VCMP) in PerformXORCombine()
14837 Ops.push_back(N0->getOperand(1)); in PerformXORCombine()
14839 return DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops); in PerformXORCombine()
14847 // ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
14851 assert(N->getOpcode() == ARMISD::BFI); in ParseBFI()
14853 SDValue From = N->getOperand(1); in ParseBFI()
14854 ToMask = ~N->getConstantOperandAPInt(2); in ParseBFI()
14859 if (From->getOpcode() == ISD::SRL && in ParseBFI()
14860 isa<ConstantSDNode>(From->getOperand(1))) { in ParseBFI()
14861 APInt Shift = From->getConstantOperandAPInt(1); in ParseBFI()
14864 From = From->getOperand(0); in ParseBFI()
14875 unsigned FirstActiveBitInB = B.getBitWidth() - B.countl_zero() - 1; in BitsProperlyConcatenate()
14876 return LastActiveBitInA - 1 == FirstActiveBitInB; in BitsProperlyConcatenate()
14883 SDValue To = N->getOperand(0); in FindBFIToCombineWith()
14911 SDValue N0 = N->getOperand(0); in PerformBFICombine()
14912 SDValue N1 = N->getOperand(1); in PerformBFICombine()
14915 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff in PerformBFICombine()
14920 unsigned InvMask = N->getConstantOperandVal(2); in PerformBFICombine()
14922 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB; in PerformBFICombine()
14926 unsigned Mask = (1u << Width) - 1; in PerformBFICombine()
14927 unsigned Mask2 = N11C->getZExtValue(); in PerformBFICombine()
14929 return DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0), in PerformBFICombine()
14930 N->getOperand(0), N1.getOperand(0), N->getOperand(2)); in PerformBFICombine()
14949 EVT VT = N->getValueType(0); in PerformBFICombine()
14963 if (N->getOperand(0).getOpcode() == ARMISD::BFI) { in PerformBFICombine()
14964 APInt ToMask1 = ~N->getConstantOperandAPInt(2); in PerformBFICombine()
14971 EVT VT = N->getValueType(0); in PerformBFICombine()
14974 N->getOperand(1), N->getOperand(2)); in PerformBFICombine()
14986 if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1))) in IsCMPZCSINC()
14988 SDValue CSInc = Cmp->getOperand(0); in IsCMPZCSINC()
14995 CSInc.getConstantOperandVal(1) == 1 && CSInc->hasOneUse()) in IsCMPZCSINC()
15000 isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) { in IsCMPZCSINC()
15005 isNullConstant(CSInc.getOperand(1)) && CSInc->hasOneUse()) { in IsCMPZCSINC()
15010 isNullConstant(CSInc.getOperand(0)) && CSInc->hasOneUse()) { in IsCMPZCSINC()
15033 // CSXYZ A, B, C1 (CMPZ (CSINC 0, 0, C2, D), 0) -> in PerformCSETCombine()
15034 // if C1==EQ -> CSXYZ A, B, C2, D in PerformCSETCombine()
15035 // if C1==NE -> CSXYZ A, B, NOT(C2), D in PerformCSETCombine()
15037 if (SDValue C = IsCMPZCSINC(N->getOperand(3).getNode(), Cond)) { in PerformCSETCombine()
15038 if (N->getConstantOperandVal(2) == ARMCC::EQ) in PerformCSETCombine()
15039 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0), in PerformCSETCombine()
15040 N->getOperand(1), in PerformCSETCombine()
15042 if (N->getConstantOperandVal(2) == ARMCC::NE) in PerformCSETCombine()
15044 N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0), in PerformCSETCombine()
15045 N->getOperand(1), in PerformCSETCombine()
15051 /// PerformVMOVRRDCombine - Target-specific dag combine xforms for
15056 // vmovrrd(vmovdrr x, y) -> x,y in PerformVMOVRRDCombine()
15057 SDValue InDouble = N->getOperand(0); in PerformVMOVRRDCombine()
15058 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64()) in PerformVMOVRRDCombine()
15061 // vmovrrd(load f64) -> (load i32), (load i32) in PerformVMOVRRDCombine()
15063 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() && in PerformVMOVRRDCombine()
15064 InNode->getValueType(0) == MVT::f64 && in PerformVMOVRRDCombine()
15065 InNode->getOperand(1).getOpcode() == ISD::FrameIndex && in PerformVMOVRRDCombine()
15066 !cast<LoadSDNode>(InNode)->isVolatile()) { in PerformVMOVRRDCombine()
15067 // TODO: Should this be done for non-FrameIndex operands? in PerformVMOVRRDCombine()
15072 SDValue BasePtr = LD->getBasePtr(); in PerformVMOVRRDCombine()
15074 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(), in PerformVMOVRRDCombine()
15075 LD->getAlign(), LD->getMemOperand()->getFlags()); in PerformVMOVRRDCombine()
15080 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr, in PerformVMOVRRDCombine()
15081 LD->getPointerInfo().getWithOffset(4), in PerformVMOVRRDCombine()
15082 commonAlignment(LD->getAlign(), 4), in PerformVMOVRRDCombine()
15083 LD->getMemOperand()->getFlags()); in PerformVMOVRRDCombine()
15092 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d in PerformVMOVRRDCombine()
15093 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b in PerformVMOVRRDCombine()
15116 if (!Subtarget->isLittle() && BVSwap) in PerformVMOVRRDCombine()
15134 if (!Subtarget->isLittle() && BVSwap) in PerformVMOVRRDCombine()
15143 /// PerformVMOVDRRCombine - Target-specific dag combine xforms for
15146 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X) in PerformVMOVDRRCombine()
15147 SDValue Op0 = N->getOperand(0); in PerformVMOVDRRCombine()
15148 SDValue Op1 = N->getOperand(1); in PerformVMOVDRRCombine()
15157 N->getValueType(0), Op0.getOperand(0)); in PerformVMOVDRRCombine()
15163 SDValue Op0 = N->getOperand(0); in PerformVMOVhrCombine()
15165 // VMOVhr (VMOVrh (X)) -> X in PerformVMOVhrCombine()
15166 if (Op0->getOpcode() == ARMISD::VMOVrh) in PerformVMOVhrCombine()
15167 return Op0->getOperand(0); in PerformVMOVhrCombine()
15169 // FullFP16: half values are passed in S-registers, and we don't in PerformVMOVhrCombine()
15177 if (Op0->getOpcode() == ISD::BITCAST) { in PerformVMOVhrCombine()
15178 SDValue Copy = Op0->getOperand(0); in PerformVMOVhrCombine()
15180 Copy->getOpcode() == ISD::CopyFromReg) { in PerformVMOVhrCombine()
15181 bool HasGlue = Copy->getNumOperands() == 3; in PerformVMOVhrCombine()
15182 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1), in PerformVMOVhrCombine()
15183 HasGlue ? Copy->getOperand(2) : SDValue()}; in PerformVMOVhrCombine()
15184 EVT OutTys[] = {N->getValueType(0), MVT::Other, MVT::Glue}; in PerformVMOVhrCombine()
15201 // fold (VMOVhr (load x)) -> (load (f16*)x) in PerformVMOVhrCombine()
15203 if (LN0->hasOneUse() && LN0->isUnindexed() && in PerformVMOVhrCombine()
15204 LN0->getMemoryVT() == MVT::i16) { in PerformVMOVhrCombine()
15206 DCI.DAG.getLoad(N->getValueType(0), SDLoc(N), LN0->getChain(), in PerformVMOVhrCombine()
15207 LN0->getBasePtr(), LN0->getMemOperand()); in PerformVMOVhrCombine()
15224 SDValue N0 = N->getOperand(0); in PerformVMOVrhCombine()
15225 EVT VT = N->getValueType(0); in PerformVMOVrhCombine()
15227 // fold (VMOVrh (fpconst x)) -> const x in PerformVMOVrhCombine()
15229 APFloat V = C->getValueAPF(); in PerformVMOVrhCombine()
15233 // fold (VMOVrh (load x)) -> (zextload (i16*)x) in PerformVMOVrhCombine()
15238 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(), in PerformVMOVrhCombine()
15239 LN0->getBasePtr(), MVT::i16, LN0->getMemOperand()); in PerformVMOVrhCombine()
15245 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n) in PerformVMOVrhCombine()
15246 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformVMOVrhCombine()
15247 isa<ConstantSDNode>(N0->getOperand(1))) in PerformVMOVrhCombine()
15248 return DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0), in PerformVMOVrhCombine()
15249 N0->getOperand(1)); in PerformVMOVrhCombine()
15254 /// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
15255 /// are normal, non-volatile loads. If so, it is profitable to bitcast an
15259 unsigned NumElts = N->getValueType(0).getVectorNumElements(); in hasNormalLoadOperand()
15261 SDNode *Elt = N->getOperand(i).getNode(); in hasNormalLoadOperand()
15262 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile()) in hasNormalLoadOperand()
15268 /// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
15273 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X): in PerformBUILD_VECTORCombine()
15278 if (N->getNumOperands() == 2) in PerformBUILD_VECTORCombine()
15284 EVT VT = N->getValueType(0); in PerformBUILD_VECTORCombine()
15291 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i)); in PerformBUILD_VECTORCombine()
15301 /// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
15315 // 2. The size of its operands are 32-bits (64-bits are not legal). in PerformARMBUILD_VECTORCombine()
15316 EVT VT = N->getValueType(0); in PerformARMBUILD_VECTORCombine()
15320 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse()) in PerformARMBUILD_VECTORCombine()
15327 SDNode *Use = *N->use_begin(); in PerformARMBUILD_VECTORCombine()
15328 if (Use->getOpcode() != ISD::BITCAST || in PerformARMBUILD_VECTORCombine()
15329 Use->getValueType(0).isFloatingPoint()) in PerformARMBUILD_VECTORCombine()
15341 SDValue Elt = N->getOperand(Idx); in PerformARMBUILD_VECTORCombine()
15342 if (Elt->getOpcode() == ISD::BITCAST) { in PerformARMBUILD_VECTORCombine()
15344 if (Elt->getOperand(0).getValueType() == MVT::i32) in PerformARMBUILD_VECTORCombine()
15349 --NumOfRelevantElts; in PerformARMBUILD_VECTORCombine()
15352 // Check if more than half of the elements require a non-free bitcast. in PerformARMBUILD_VECTORCombine()
15367 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1), in PerformARMBUILD_VECTORCombine()
15372 SDValue V = N->getOperand(Idx); in PerformARMBUILD_VECTORCombine()
15376 V->getOperand(0).getValueType() == MVT::i32) in PerformARMBUILD_VECTORCombine()
15395 EVT VT = N->getValueType(0); in PerformPREDICATE_CASTCombine()
15396 SDValue Op = N->getOperand(0); in PerformPREDICATE_CASTCombine()
15400 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) { in PerformPREDICATE_CASTCombine()
15402 if (Op->getOperand(0).getValueType() == VT) in PerformPREDICATE_CASTCombine()
15403 return Op->getOperand(0); in PerformPREDICATE_CASTCombine()
15404 return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0)); in PerformPREDICATE_CASTCombine()
15407 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce in PerformPREDICATE_CASTCombine()
15411 DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0)); in PerformPREDICATE_CASTCombine()
15429 EVT VT = N->getValueType(0); in PerformVECTOR_REG_CASTCombine()
15430 SDValue Op = N->getOperand(0); in PerformVECTOR_REG_CASTCombine()
15434 if (ST->isLittle()) in PerformVECTOR_REG_CASTCombine()
15437 // VECTOR_REG_CAST undef -> undef in PerformVECTOR_REG_CASTCombine()
15442 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) { in PerformVECTOR_REG_CASTCombine()
15444 if (Op->getOperand(0).getValueType() == VT) in PerformVECTOR_REG_CASTCombine()
15445 return Op->getOperand(0); in PerformVECTOR_REG_CASTCombine()
15446 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0)); in PerformVECTOR_REG_CASTCombine()
15454 if (!Subtarget->hasMVEIntegerOps()) in PerformVCMPCombine()
15457 EVT VT = N->getValueType(0); in PerformVCMPCombine()
15458 SDValue Op0 = N->getOperand(0); in PerformVCMPCombine()
15459 SDValue Op1 = N->getOperand(1); in PerformVCMPCombine()
15460 ARMCC::CondCodes Cond = (ARMCC::CondCodes)N->getConstantOperandVal(2); in PerformVCMPCombine()
15463 // vcmp X, 0, cc -> vcmpz X, cc in PerformVCMPCombine()
15465 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, N->getOperand(2)); in PerformVCMPCombine()
15469 // vcmp 0, X, cc -> vcmpz X, reversed(cc) in PerformVCMPCombine()
15473 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc) in PerformVCMPCombine()
15474 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP) in PerformVCMPCombine()
15482 /// PerformInsertEltCombine - Target-specific dag combine xforms for
15488 EVT VT = N->getValueType(0); in PerformInsertEltCombine()
15489 SDNode *Elt = N->getOperand(1).getNode(); in PerformInsertEltCombine()
15491 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile()) in PerformInsertEltCombine()
15498 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0)); in PerformInsertEltCombine()
15499 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1)); in PerformInsertEltCombine()
15504 Vec, V, N->getOperand(2)); in PerformInsertEltCombine()
15510 // extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
15511 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
15514 EVT VT = N->getValueType(0); in PerformExtractEltToVMOVRRD()
15529 if (Ext->use_size() == 1 && in PerformExtractEltToVMOVRRD()
15530 (Ext->use_begin()->getOpcode() == ISD::SINT_TO_FP || in PerformExtractEltToVMOVRRD()
15531 Ext->use_begin()->getOpcode() == ISD::UINT_TO_FP)) in PerformExtractEltToVMOVRRD()
15542 auto OtherIt = find_if(Op0->uses(), [&](SDNode *V) { in PerformExtractEltToVMOVRRD()
15543 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformExtractEltToVMOVRRD()
15544 isa<ConstantSDNode>(V->getOperand(1)) && in PerformExtractEltToVMOVRRD()
15545 V->getConstantOperandVal(1) == Lane + 1 && in PerformExtractEltToVMOVRRD()
15546 V->getOperand(0).getResNo() == ResNo; in PerformExtractEltToVMOVRRD()
15548 if (OtherIt == Op0->uses().end()) in PerformExtractEltToVMOVRRD()
15555 if (OtherExt->use_size() != 1 || in PerformExtractEltToVMOVRRD()
15556 OtherExt->use_begin()->getOpcode() != ISD::BITCAST || in PerformExtractEltToVMOVRRD()
15557 OtherExt->use_begin()->getValueType(0) != MVT::i32) in PerformExtractEltToVMOVRRD()
15559 OtherExt = SDValue(*OtherExt->use_begin(), 0); in PerformExtractEltToVMOVRRD()
15577 SDValue Op0 = N->getOperand(0); in PerformExtractEltCombine()
15578 EVT VT = N->getValueType(0); in PerformExtractEltCombine()
15581 // extract (vdup x) -> x in PerformExtractEltCombine()
15582 if (Op0->getOpcode() == ARMISD::VDUP) { in PerformExtractEltCombine()
15583 SDValue X = Op0->getOperand(0); in PerformExtractEltCombine()
15591 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST) in PerformExtractEltCombine()
15592 X = X->getOperand(0); in PerformExtractEltCombine()
15597 // extract ARM_BUILD_VECTOR -> x in PerformExtractEltCombine()
15598 if (Op0->getOpcode() == ARMISD::BUILD_VECTOR && in PerformExtractEltCombine()
15599 isa<ConstantSDNode>(N->getOperand(1)) && in PerformExtractEltCombine()
15600 N->getConstantOperandVal(1) < Op0.getNumOperands()) { in PerformExtractEltCombine()
15601 return Op0.getOperand(N->getConstantOperandVal(1)); in PerformExtractEltCombine()
15604 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b in PerformExtractEltCombine()
15606 isa<ConstantSDNode>(N->getOperand(1)) && in PerformExtractEltCombine()
15611 unsigned Offset = N->getConstantOperandVal(1); in PerformExtractEltCombine()
15614 return MOV.getOperand(ST->isLittle() ? Offset % 2 : 1 - Offset % 2); in PerformExtractEltCombine()
15617 // extract x, n; extract x, n+1 -> VMOVRRD x in PerformExtractEltCombine()
15621 // extract (MVETrunc(x)) -> extract x in PerformExtractEltCombine()
15622 if (Op0->getOpcode() == ARMISD::MVETRUNC) { in PerformExtractEltCombine()
15623 unsigned Idx = N->getConstantOperandVal(1); in PerformExtractEltCombine()
15625 Idx / Op0->getOperand(0).getValueType().getVectorNumElements(); in PerformExtractEltCombine()
15627 Idx % Op0->getOperand(0).getValueType().getVectorNumElements(); in PerformExtractEltCombine()
15636 SDValue Op = N->getOperand(0); in PerformSignExtendInregCombine()
15637 EVT VT = N->getValueType(0); in PerformSignExtendInregCombine()
15639 // sext_inreg(VGETLANEu) -> VGETLANEs in PerformSignExtendInregCombine()
15641 cast<VTSDNode>(N->getOperand(1))->getVT() == in PerformSignExtendInregCombine()
15651 SDValue Vec = N->getOperand(0); in PerformInsertSubvectorCombine()
15652 SDValue SubVec = N->getOperand(1); in PerformInsertSubvectorCombine()
15653 uint64_t IdxVal = N->getConstantOperandVal(2); in PerformInsertSubvectorCombine()
15673 // Fold insert_subvector -> concat_vectors in PerformInsertSubvectorCombine()
15674 // insert_subvector(Vec,Sub,lo) -> concat_vectors(Sub,extract(Vec,hi)) in PerformInsertSubvectorCombine()
15675 // insert_subvector(Vec,Sub,hi) -> concat_vectors(extract(Vec,lo),Sub) in PerformInsertSubvectorCombine()
15690 // shuffle(MVETrunc(x, y)) -> VMOVN(x, y)
15693 SDValue Trunc = N->getOperand(0); in PerformShuffleVMOVNCombine()
15695 if (Trunc.getOpcode() != ARMISD::MVETRUNC || !N->getOperand(1).isUndef()) in PerformShuffleVMOVNCombine()
15699 if (isVMOVNTruncMask(N->getMask(), VT, false)) in PerformShuffleVMOVNCombine()
15705 else if (isVMOVNTruncMask(N->getMask(), VT, true)) in PerformShuffleVMOVNCombine()
15714 /// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
15725 // targets, but for NEON it is better to concatenate two double-register in PerformVECTOR_SHUFFLECombine()
15726 // size vector operands into a single quad-register size vector. Do that in PerformVECTOR_SHUFFLECombine()
15728 // shuffle(concat(v1, undef), concat(v2, undef)) -> in PerformVECTOR_SHUFFLECombine()
15730 SDValue Op0 = N->getOperand(0); in PerformVECTOR_SHUFFLECombine()
15731 SDValue Op1 = N->getOperand(1); in PerformVECTOR_SHUFFLECombine()
15743 EVT VT = N->getValueType(0); in PerformVECTOR_SHUFFLECombine()
15757 int MaskElt = SVN->getMaskElt(n); in PerformVECTOR_SHUFFLECombine()
15758 int NewElt = -1; in PerformVECTOR_SHUFFLECombine()
15762 NewElt = HalfElts + MaskElt - NumElts; in PerformVECTOR_SHUFFLECombine()
15805 unsigned IntNo = N->getConstantOperandVal(1); in TryCombineBaseUpdate()
15926 switch (N->getOpcode()) { in TryCombineBaseUpdate()
15962 VecTy = N->getValueType(0); in TryCombineBaseUpdate()
15964 VecTy = N->getOperand(Target.AddrOpIdx + 1).getValueType(); in TryCombineBaseUpdate()
15968 VecTy = N->getOperand(1).getValueType(); in TryCombineBaseUpdate()
15980 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two in TryCombineBaseUpdate()
15981 // separate instructions that make it harder to use a non-constant update. in TryCombineBaseUpdate()
15992 Align Alignment = MemN->getAlign(); in TryCombineBaseUpdate()
15994 // If this is a less-than-standard-aligned load/store, change the type to in TryCombineBaseUpdate()
15998 // There are 3 ways to get to this base-update combine: in TryCombineBaseUpdate()
15999 // - intrinsics: they are assumed to be properly aligned (to the standard in TryCombineBaseUpdate()
16001 // - ARMISD::VLDx nodes: they are only generated from the aforementioned in TryCombineBaseUpdate()
16003 // - generic load/store instructions: the alignment is specified as an in TryCombineBaseUpdate()
16007 // generate non-standard-aligned ARMISD::VLDx nodes. in TryCombineBaseUpdate()
16011 assert(NumVecs == 1 && "Unexpected multi-element generic load/store."); in TryCombineBaseUpdate()
16039 Ops.push_back(N->getOperand(0)); // incoming chain in TryCombineBaseUpdate()
16040 Ops.push_back(N->getOperand(Target.AddrOpIdx)); in TryCombineBaseUpdate()
16045 Ops.push_back(StN->getValue()); in TryCombineBaseUpdate()
16050 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands(); in TryCombineBaseUpdate()
16052 Ops.push_back(N->getOperand(i)); in TryCombineBaseUpdate()
16058 // If this is a non-standard-aligned STORE, the penultimate operand is the in TryCombineBaseUpdate()
16060 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) { in TryCombineBaseUpdate()
16061 SDValue &StVal = Ops[Ops.size() - 2]; in TryCombineBaseUpdate()
16067 MemN->getMemOperand()); in TryCombineBaseUpdate()
16074 // If this is an non-standard-aligned LOAD, the first result is the loaded in TryCombineBaseUpdate()
16076 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) { in TryCombineBaseUpdate()
16088 // If (opcode ptr inc) is and ADD-like instruction, return the
16099 return CInc->getZExtValue(); in getPointerConstIncrement()
16103 return CInc->getZExtValue(); in getPointerConstIncrement()
16113 switch (N->getOpcode()) { in findPointerConstIncrement()
16116 if (isa<ConstantSDNode>(N->getOperand(1))) { in findPointerConstIncrement()
16117 *Ptr = N->getOperand(0); in findPointerConstIncrement()
16118 *CInc = N->getOperand(1); in findPointerConstIncrement()
16124 if (isa<ConstantSDNode>(N->getOperand(2))) { in findPointerConstIncrement()
16125 *Ptr = N->getOperand(1); in findPointerConstIncrement()
16126 *CInc = N->getOperand(2); in findPointerConstIncrement()
16151 /// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16158 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID || in CombineBaseUpdate()
16159 N->getOpcode() == ISD::INTRINSIC_W_CHAIN); in CombineBaseUpdate()
16160 const bool isStore = N->getOpcode() == ISD::STORE; in CombineBaseUpdate()
16164 SDValue Addr = N->getOperand(AddrOpIdx); in CombineBaseUpdate()
16169 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), in CombineBaseUpdate()
16170 UE = Addr.getNode()->use_end(); UI != UE; ++UI) { in CombineBaseUpdate()
16173 User->getNumOperands() != 2) in CombineBaseUpdate()
16176 SDValue Inc = User->getOperand(UI.getOperandNo() == 1 ? 0 : 1); in CombineBaseUpdate()
16178 getPointerConstIncrement(User->getOpcode(), Addr, Inc, DCI.DAG); in CombineBaseUpdate()
16180 if (ConstInc || User->getOpcode() == ISD::ADD) in CombineBaseUpdate()
16190 getPointerConstIncrement(Addr->getOpcode(), Base, CInc, DCI.DAG); in CombineBaseUpdate()
16191 for (SDNode::use_iterator UI = Base->use_begin(), UE = Base->use_end(); in CombineBaseUpdate()
16196 User->getNumOperands() != 2) in CombineBaseUpdate()
16199 SDValue UserInc = User->getOperand(UI.getOperandNo() == 0 ? 1 : 0); in CombineBaseUpdate()
16201 getPointerConstIncrement(User->getOpcode(), Base, UserInc, DCI.DAG); in CombineBaseUpdate()
16206 unsigned NewConstInc = UserOffset - Offset; in CombineBaseUpdate()
16220 --NumValidUpd; in CombineBaseUpdate()
16231 // Try to fold with other users. Non-constant updates are considered in CombineBaseUpdate()
16259 SDValue Addr = N->getOperand(2); in PerformMVEVLDCombine()
16264 // to post-inc the last of the them. in PerformMVEVLDCombine()
16265 unsigned IntNo = N->getConstantOperandVal(1); in PerformMVEVLDCombine()
16266 if (IntNo == Intrinsic::arm_mve_vst2q && N->getConstantOperandVal(5) != 1) in PerformMVEVLDCombine()
16268 if (IntNo == Intrinsic::arm_mve_vst4q && N->getConstantOperandVal(7) != 3) in PerformMVEVLDCombine()
16272 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), in PerformMVEVLDCombine()
16273 UE = Addr.getNode()->use_end(); in PerformMVEVLDCombine()
16276 if (User->getOpcode() != ISD::ADD || in PerformMVEVLDCombine()
16322 VecTy = N->getValueType(0); in PerformMVEVLDCombine()
16324 VecTy = N->getOperand(3).getValueType(); in PerformMVEVLDCombine()
16330 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0); in PerformMVEVLDCombine()
16332 if (!CInc || CInc->getZExtValue() != NumBytes) in PerformMVEVLDCombine()
16348 Ops.push_back(N->getOperand(0)); // incoming chain in PerformMVEVLDCombine()
16349 Ops.push_back(N->getOperand(2)); // ptr in PerformMVEVLDCombine()
16352 for (unsigned i = 3; i < N->getNumOperands(); ++i) in PerformMVEVLDCombine()
16353 Ops.push_back(N->getOperand(i)); in PerformMVEVLDCombine()
16356 MemN->getMemOperand()); in PerformMVEVLDCombine()
16373 /// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
16374 /// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
16375 /// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
16379 EVT VT = N->getValueType(0); in CombineVLDDUP()
16380 // vldN-dup instructions only support 64-bit vectors for N > 1. in CombineVLDDUP()
16384 // Check if the VDUPLANE operand is a vldN-dup intrinsic. in CombineVLDDUP()
16385 SDNode *VLD = N->getOperand(0).getNode(); in CombineVLDDUP()
16386 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN) in CombineVLDDUP()
16390 unsigned IntNo = VLD->getConstantOperandVal(1); in CombineVLDDUP()
16404 // First check that all the vldN-lane uses are VDUPLANEs and that the lane in CombineVLDDUP()
16406 unsigned VLDLaneNo = VLD->getConstantOperandVal(NumVecs + 3); in CombineVLDDUP()
16407 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end(); in CombineVLDDUP()
16413 if (User->getOpcode() != ARMISD::VDUPLANE || in CombineVLDDUP()
16414 VLDLaneNo != User->getConstantOperandVal(1)) in CombineVLDDUP()
16418 // Create the vldN-dup node. in CombineVLDDUP()
16425 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) }; in CombineVLDDUP()
16428 Ops, VLDMemInt->getMemoryVT(), in CombineVLDDUP()
16429 VLDMemInt->getMemOperand()); in CombineVLDDUP()
16432 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end(); in CombineVLDDUP()
16442 // Now the vldN-lane intrinsic is dead except for its chain result. in CombineVLDDUP()
16453 /// PerformVDUPLANECombine - Target-specific dag combine xforms for
16458 SDValue Op = N->getOperand(0); in PerformVDUPLANECombine()
16459 EVT VT = N->getValueType(0); in PerformVDUPLANECombine()
16462 if (Subtarget->hasMVEIntegerOps()) { in PerformVDUPLANECombine()
16468 N->getOperand(0), N->getOperand(1)); in PerformVDUPLANECombine()
16472 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses in PerformVDUPLANECombine()
16473 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation. in PerformVDUPLANECombine()
16486 // The canonical VMOV for a zero vector uses a 32-bit element size. in PerformVDUPLANECombine()
16497 /// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
16500 SDValue Op = N->getOperand(0); in PerformVDUPCombine()
16503 if (Subtarget->hasMVEIntegerOps()) { in PerformVDUPCombine()
16504 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will in PerformVDUPCombine()
16507 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), in PerformVDUPCombine()
16510 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), in PerformVDUPCombine()
16514 if (!Subtarget->hasNEON()) in PerformVDUPCombine()
16517 // Match VDUP(LOAD) -> VLD1DUP. in PerformVDUPCombine()
16521 if (LD && Op.hasOneUse() && LD->isUnindexed() && in PerformVDUPCombine()
16522 LD->getMemoryVT() == N->getValueType(0).getVectorElementType()) { in PerformVDUPCombine()
16523 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1), in PerformVDUPCombine()
16524 DAG.getConstant(LD->getAlign().value(), SDLoc(N), MVT::i32)}; in PerformVDUPCombine()
16525 SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other); in PerformVDUPCombine()
16528 LD->getMemoryVT(), LD->getMemOperand()); in PerformVDUPCombine()
16539 EVT VT = N->getValueType(0); in PerformLOADCombine()
16542 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() && in PerformLOADCombine()
16554 SDValue StVal = St->getValue(); in PerformTruncatingStoreCombine()
16556 if (!St->isTruncatingStore() || !VT.isVector()) in PerformTruncatingStoreCombine()
16559 EVT StVT = St->getMemoryVT(); in PerformTruncatingStoreCombine()
16584 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1); in PerformTruncatingStoreCombine()
16586 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1 in PerformTruncatingStoreCombine()
16608 // Bitcast the original vector into a vector of store-size units in PerformTruncatingStoreCombine()
16617 SDValue BasePtr = St->getBasePtr(); in PerformTruncatingStoreCombine()
16625 DAG.getStore(St->getChain(), DL, SubVec, BasePtr, St->getPointerInfo(), in PerformTruncatingStoreCombine()
16626 St->getAlign(), St->getMemOperand()->getFlags()); in PerformTruncatingStoreCombine()
16639 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed()) in PerformSplittingToNarrowingStores()
16641 SDValue Trunc = St->getValue(); in PerformSplittingToNarrowingStores()
16642 if (Trunc->getOpcode() != ISD::FP_ROUND) in PerformSplittingToNarrowingStores()
16644 EVT FromVT = Trunc->getOperand(0).getValueType(); in PerformSplittingToNarrowingStores()
16667 ArrayRef<int> M = SVN->getMask(); in PerformSplittingToNarrowingStores()
16669 if (SVN->getOperand(1).isUndef()) in PerformSplittingToNarrowingStores()
16692 SDValue Ch = St->getChain(); in PerformSplittingToNarrowingStores()
16693 SDValue BasePtr = St->getBasePtr(); in PerformSplittingToNarrowingStores()
16694 Align Alignment = St->getOriginalAlign(); in PerformSplittingToNarrowingStores()
16695 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags(); in PerformSplittingToNarrowingStores()
16696 AAMDNodes AAInfo = St->getAAInfo(); in PerformSplittingToNarrowingStores()
16720 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset), in PerformSplittingToNarrowingStores()
16732 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed()) in PerformSplittingMVETruncToNarrowingStores()
16734 SDValue Trunc = St->getValue(); in PerformSplittingMVETruncToNarrowingStores()
16735 if (Trunc->getOpcode() != ARMISD::MVETRUNC) in PerformSplittingMVETruncToNarrowingStores()
16737 EVT FromVT = Trunc->getOperand(0).getValueType(); in PerformSplittingMVETruncToNarrowingStores()
16743 SDValue Ch = St->getChain(); in PerformSplittingMVETruncToNarrowingStores()
16744 SDValue BasePtr = St->getBasePtr(); in PerformSplittingMVETruncToNarrowingStores()
16745 Align Alignment = St->getOriginalAlign(); in PerformSplittingMVETruncToNarrowingStores()
16746 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags(); in PerformSplittingMVETruncToNarrowingStores()
16747 AAMDNodes AAInfo = St->getAAInfo(); in PerformSplittingMVETruncToNarrowingStores()
16761 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset), in PerformSplittingMVETruncToNarrowingStores()
16771 // use of more integer post-inc stores not available with vstr.
16773 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed()) in PerformExtractFpToIntStores()
16775 SDValue Extract = St->getValue(); in PerformExtractFpToIntStores()
16779 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in PerformExtractFpToIntStores()
16791 SDValue Ch = St->getChain(); in PerformExtractFpToIntStores()
16792 SDValue BasePtr = St->getBasePtr(); in PerformExtractFpToIntStores()
16793 Align Alignment = St->getOriginalAlign(); in PerformExtractFpToIntStores()
16794 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags(); in PerformExtractFpToIntStores()
16795 AAMDNodes AAInfo = St->getAAInfo(); in PerformExtractFpToIntStores()
16798 St->getPointerInfo(), NewToVT, Alignment, in PerformExtractFpToIntStores()
16804 /// PerformSTORECombine - Target-specific dag combine xforms for
16810 if (St->isVolatile()) in PerformSTORECombine()
16812 SDValue StVal = St->getValue(); in PerformSTORECombine()
16815 if (Subtarget->hasNEON()) in PerformSTORECombine()
16819 if (Subtarget->hasMVEFloatOps()) in PerformSTORECombine()
16823 if (Subtarget->hasMVEIntegerOps()) { in PerformSTORECombine()
16836 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR && in PerformSTORECombine()
16837 StVal.getNode()->hasOneUse()) { in PerformSTORECombine()
16841 SDValue BasePtr = St->getBasePtr(); in PerformSTORECombine()
16843 St->getChain(), DL, StVal.getNode()->getOperand(isBigEndian ? 1 : 0), in PerformSTORECombine()
16844 BasePtr, St->getPointerInfo(), St->getOriginalAlign(), in PerformSTORECombine()
16845 St->getMemOperand()->getFlags()); in PerformSTORECombine()
16850 StVal.getNode()->getOperand(isBigEndian ? 0 : 1), in PerformSTORECombine()
16851 OffsetPtr, St->getPointerInfo().getWithOffset(4), in PerformSTORECombine()
16852 St->getOriginalAlign(), in PerformSTORECombine()
16853 St->getMemOperand()->getFlags()); in PerformSTORECombine()
16857 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in PerformSTORECombine()
16875 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(), in PerformSTORECombine()
16876 St->getPointerInfo(), St->getAlign(), in PerformSTORECombine()
16877 St->getMemOperand()->getFlags(), St->getAAInfo()); in PerformSTORECombine()
16881 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() && in PerformSTORECombine()
16888 /// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
16889 /// can replace combinations of VMUL and VCVT (floating-point to integer)
16899 if (!Subtarget->hasNEON()) in PerformVCVTCombine()
16902 SDValue Op = N->getOperand(0); in PerformVCVTCombine()
16907 SDValue ConstVec = Op->getOperand(1); in PerformVCVTCombine()
16913 MVT IntTy = N->getSimpleValueType(0).getVectorElementType(); in PerformVCVTCombine()
16926 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33); in PerformVCVTCombine()
16927 if (C == -1 || C == 0 || C > 32) in PerformVCVTCombine()
16931 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT; in PerformVCVTCombine()
16936 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0), in PerformVCVTCombine()
16940 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv); in PerformVCVTCombine()
16947 if (!Subtarget->hasMVEFloatOps()) in PerformFAddVSelectCombine()
16950 // Turn (fadd x, (vselect c, y, -0.0)) into (vselect c, (fadd x, y), x) in PerformFAddVSelectCombine()
16953 SDValue Op0 = N->getOperand(0); in PerformFAddVSelectCombine()
16954 SDValue Op1 = N->getOperand(1); in PerformFAddVSelectCombine()
16955 EVT VT = N->getValueType(0); in PerformFAddVSelectCombine()
16958 // The identity element for a fadd is -0.0 or +0.0 when the nsz flag is set, in PerformFAddVSelectCombine()
16978 SDNodeFlags FaddFlags = N->getFlags(); in PerformFAddVSelectCombine()
16989 SDValue LHS = N->getOperand(0); in PerformFADDVCMLACombine()
16990 SDValue RHS = N->getOperand(1); in PerformFADDVCMLACombine()
16991 EVT VT = N->getValueType(0); in PerformFADDVCMLACombine()
16994 if (!N->getFlags().hasAllowReassociation()) in PerformFADDVCMLACombine()
16997 // Combine fadd(a, vcmla(b, c, d)) -> vcmla(fadd(a, b), b, c) in PerformFADDVCMLACombine()
17006 DAG.getNode(ISD::FADD, DL, VT, A.getOperand(2), B, N->getFlags()), in PerformFADDVCMLACombine()
17008 VCMLA->setFlags(A->getFlags()); in PerformFADDVCMLACombine()
17028 /// PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
17029 /// can replace combinations of VCVT (integer to floating-point) and VMUL
17039 if (!Subtarget->hasNEON()) in PerformVMulVCTPCombine()
17042 SDValue Op = N->getOperand(0); in PerformVMulVCTPCombine()
17043 unsigned OpOpcode = Op.getNode()->getOpcode(); in PerformVMulVCTPCombine()
17044 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() || in PerformVMulVCTPCombine()
17048 SDValue ConstVec = N->getOperand(1); in PerformVMulVCTPCombine()
17052 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType(); in PerformVMulVCTPCombine()
17067 if (!CN || !CN->getValueAPF().getExactInverse(&Recip)) in PerformVMulVCTPCombine()
17078 if (C == -1 || C == 0 || C > 32) in PerformVMulVCTPCombine()
17097 if (!ST->hasMVEIntegerOps()) in PerformVECREDUCE_ADDCombine()
17100 assert(N->getOpcode() == ISD::VECREDUCE_ADD); in PerformVECREDUCE_ADDCombine()
17101 EVT ResVT = N->getValueType(0); in PerformVECREDUCE_ADDCombine()
17102 SDValue N0 = N->getOperand(0); in PerformVECREDUCE_ADDCombine()
17145 if (ResVT != RetTy || N0->getOpcode() != ExtendCode) in PerformVECREDUCE_ADDCombine()
17147 SDValue A = N0->getOperand(0); in PerformVECREDUCE_ADDCombine()
17154 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT || in PerformVECREDUCE_ADDCombine()
17155 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode())) in PerformVECREDUCE_ADDCombine()
17157 Mask = N0->getOperand(0); in PerformVECREDUCE_ADDCombine()
17158 SDValue Ext = N0->getOperand(1); in PerformVECREDUCE_ADDCombine()
17159 if (Ext->getOpcode() != ExtendCode) in PerformVECREDUCE_ADDCombine()
17161 SDValue A = Ext->getOperand(0); in PerformVECREDUCE_ADDCombine()
17179 if (Mul->getOpcode() == ExtendCode && in PerformVECREDUCE_ADDCombine()
17180 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >= in PerformVECREDUCE_ADDCombine()
17182 Mul = Mul->getOperand(0); in PerformVECREDUCE_ADDCombine()
17183 if (Mul->getOpcode() != ISD::MUL) in PerformVECREDUCE_ADDCombine()
17185 SDValue ExtA = Mul->getOperand(0); in PerformVECREDUCE_ADDCombine()
17186 SDValue ExtB = Mul->getOperand(1); in PerformVECREDUCE_ADDCombine()
17187 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode) in PerformVECREDUCE_ADDCombine()
17189 A = ExtA->getOperand(0); in PerformVECREDUCE_ADDCombine()
17190 B = ExtB->getOperand(0); in PerformVECREDUCE_ADDCombine()
17206 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT || in PerformVECREDUCE_ADDCombine()
17207 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode())) in PerformVECREDUCE_ADDCombine()
17209 Mask = N0->getOperand(0); in PerformVECREDUCE_ADDCombine()
17210 SDValue Mul = N0->getOperand(1); in PerformVECREDUCE_ADDCombine()
17211 if (Mul->getOpcode() == ExtendCode && in PerformVECREDUCE_ADDCombine()
17212 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >= in PerformVECREDUCE_ADDCombine()
17214 Mul = Mul->getOperand(0); in PerformVECREDUCE_ADDCombine()
17215 if (Mul->getOpcode() != ISD::MUL) in PerformVECREDUCE_ADDCombine()
17217 SDValue ExtA = Mul->getOperand(0); in PerformVECREDUCE_ADDCombine()
17218 SDValue ExtB = Mul->getOperand(1); in PerformVECREDUCE_ADDCombine()
17219 if (ExtA->getOpcode() != ExtendCode || ExtB->getOpcode() != ExtendCode) in PerformVECREDUCE_ADDCombine()
17221 A = ExtA->getOperand(0); in PerformVECREDUCE_ADDCombine()
17222 B = ExtB->getOperand(0); in PerformVECREDUCE_ADDCombine()
17231 // Split illegal MVT::v16i8->i64 vector reductions into two legal v8i16->i64 in PerformVECREDUCE_ADDCombine()
17334 if (Op->getOpcode() == ISD::VSELECT) in PerformVECREDUCE_ADDCombine()
17335 Op = Op->getOperand(1); in PerformVECREDUCE_ADDCombine()
17336 if (Op->getOpcode() == ISD::ZERO_EXTEND && in PerformVECREDUCE_ADDCombine()
17337 Op->getOperand(0)->getOpcode() == ISD::MUL) { in PerformVECREDUCE_ADDCombine()
17338 SDValue Mul = Op->getOperand(0); in PerformVECREDUCE_ADDCombine()
17339 if (Mul->getOperand(0) == Mul->getOperand(1) && in PerformVECREDUCE_ADDCombine()
17340 Mul->getOperand(0)->getOpcode() == ISD::SIGN_EXTEND) { in PerformVECREDUCE_ADDCombine()
17341 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, N0->getValueType(0), Mul); in PerformVECREDUCE_ADDCombine()
17343 Ext = DAG.getNode(ISD::VSELECT, dl, N0->getValueType(0), in PerformVECREDUCE_ADDCombine()
17344 N0->getOperand(0), Ext, N0->getOperand(2)); in PerformVECREDUCE_ADDCombine()
17356 unsigned VecOp = N->getOperand(0).getValueType().isVector() ? 0 : 2; in PerformReduceShuffleCombine()
17357 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N->getOperand(VecOp)); in PerformReduceShuffleCombine()
17358 if (!Shuf || !Shuf->getOperand(1).isUndef()) in PerformReduceShuffleCombine()
17362 ArrayRef<int> Mask = Shuf->getMask(); in PerformReduceShuffleCombine()
17372 if (N->getNumOperands() != VecOp + 1) { in PerformReduceShuffleCombine()
17373 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(N->getOperand(VecOp + 1)); in PerformReduceShuffleCombine()
17374 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask) in PerformReduceShuffleCombine()
17379 for (SDValue Op : N->ops()) { in PerformReduceShuffleCombine()
17385 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops); in PerformReduceShuffleCombine()
17390 SDValue Op0 = N->getOperand(0); in PerformVMOVNCombine()
17391 SDValue Op1 = N->getOperand(1); in PerformVMOVNCombine()
17392 unsigned IsTop = N->getConstantOperandVal(2); in PerformVMOVNCombine()
17394 // VMOVNT a undef -> a in PerformVMOVNCombine()
17395 // VMOVNB a undef -> a in PerformVMOVNCombine()
17396 // VMOVNB undef a -> a in PerformVMOVNCombine()
17397 if (Op1->isUndef()) in PerformVMOVNCombine()
17399 if (Op0->isUndef() && !IsTop) in PerformVMOVNCombine()
17404 if ((Op1->getOpcode() == ARMISD::VQMOVNs || in PerformVMOVNCombine()
17405 Op1->getOpcode() == ARMISD::VQMOVNu) && in PerformVMOVNCombine()
17406 Op1->getConstantOperandVal(2) == 0) in PerformVMOVNCombine()
17407 return DCI.DAG.getNode(Op1->getOpcode(), SDLoc(Op1), N->getValueType(0), in PerformVMOVNCombine()
17408 Op0, Op1->getOperand(1), N->getOperand(2)); in PerformVMOVNCombine()
17413 unsigned NumElts = N->getValueType(0).getVectorNumElements(); in PerformVMOVNCombine()
17430 SDValue Op0 = N->getOperand(0); in PerformVQMOVNCombine()
17431 unsigned IsTop = N->getConstantOperandVal(2); in PerformVQMOVNCombine()
17433 unsigned NumElts = N->getValueType(0).getVectorNumElements(); in PerformVQMOVNCombine()
17446 EVT VT = N->getValueType(0); in PerformVQDMULHCombine()
17447 SDValue LHS = N->getOperand(0); in PerformVQDMULHCombine()
17448 SDValue RHS = N->getOperand(1); in PerformVQDMULHCombine()
17452 // Turn VQDMULH(shuffle, shuffle) -> shuffle(VQDMULH) in PerformVQDMULHCombine()
17453 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) && in PerformVQDMULHCombine()
17457 SDValue NewBinOp = DCI.DAG.getNode(N->getOpcode(), DL, VT, in PerformVQDMULHCombine()
17460 return DCI.DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); in PerformVQDMULHCombine()
17467 SDValue Op0 = N->getOperand(0); in PerformLongShiftCombine()
17468 SDValue Op1 = N->getOperand(1); in PerformLongShiftCombine()
17470 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from in PerformLongShiftCombine()
17472 if (auto C = dyn_cast<ConstantSDNode>(N->getOperand(2))) { in PerformLongShiftCombine()
17473 int ShiftAmt = C->getSExtValue(); in PerformLongShiftCombine()
17480 if (ShiftAmt >= -32 && ShiftAmt < 0) { in PerformLongShiftCombine()
17482 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL; in PerformLongShiftCombine()
17483 SDValue NewShift = DAG.getNode(NewOpcode, DL, N->getVTList(), Op0, Op1, in PerformLongShiftCombine()
17484 DAG.getConstant(-ShiftAmt, DL, MVT::i32)); in PerformLongShiftCombine()
17493 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
17497 unsigned IntNo = N->getConstantOperandVal(0); in PerformIntrinsicCombine()
17505 // the build_vectors for 64-bit vector element shift counts are generally in PerformIntrinsicCombine()
17522 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine()
17529 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) { in PerformIntrinsicCombine()
17533 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) { in PerformIntrinsicCombine()
17542 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) in PerformIntrinsicCombine()
17548 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
17553 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
17565 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt)) in PerformIntrinsicCombine()
17618 return DAG.getNode(VShiftOpc, dl, N->getValueType(0), in PerformIntrinsicCombine()
17619 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32)); in PerformIntrinsicCombine()
17623 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine()
17627 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt)) in PerformIntrinsicCombine()
17629 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt)) in PerformIntrinsicCombine()
17636 return DAG.getNode(VShiftOpc, dl, N->getValueType(0), in PerformIntrinsicCombine()
17637 N->getOperand(1), N->getOperand(2), in PerformIntrinsicCombine()
17660 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits(); in PerformIntrinsicCombine()
17662 if (SimplifyDemandedBits(N->getOperand(3), DemandedMask, DCI)) in PerformIntrinsicCombine()
17677 unsigned BitWidth = N->getOperand(2)->getValueType(0).getScalarSizeInBits(); in PerformIntrinsicCombine()
17679 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)) in PerformIntrinsicCombine()
17687 bool Unsigned = N->getConstantOperandVal(2); in PerformIntrinsicCombine()
17689 return DAG.getNode(Opc, SDLoc(N), N->getVTList(), N->getOperand(1)); in PerformIntrinsicCombine()
17696 bool Unsigned = N->getConstantOperandVal(2); in PerformIntrinsicCombine()
17702 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++) in PerformIntrinsicCombine()
17704 Ops.push_back(N->getOperand(i)); in PerformIntrinsicCombine()
17716 /// PerformShiftCombine - Checks for immediate versions of vector shifts and
17718 /// combining instead of DAG legalizing because the build_vectors for 64-bit
17725 EVT VT = N->getValueType(0); in PerformShiftCombine()
17727 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 && in PerformShiftCombine()
17728 N->getOperand(0)->getOpcode() == ISD::AND && in PerformShiftCombine()
17729 N->getOperand(0)->hasOneUse()) { in PerformShiftCombine()
17736 SDValue N0 = N->getOperand(0); in PerformShiftCombine()
17737 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(N->getOperand(1)); in PerformShiftCombine()
17740 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue()); in PerformShiftCombine()
17741 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(N0->getOperand(1)); in PerformShiftCombine()
17744 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue()); in PerformShiftCombine()
17752 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), in PerformShiftCombine()
17756 DAG.getConstant(MaskedBits - ShiftAmt, DL, MVT::i32)); in PerformShiftCombine()
17765 if (ST->hasMVEIntegerOps()) in PerformShiftCombine()
17770 switch (N->getOpcode()) { in PerformShiftCombine()
17774 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) { in PerformShiftCombine()
17776 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0), in PerformShiftCombine()
17783 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { in PerformShiftCombine()
17785 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM); in PerformShiftCombine()
17787 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), in PerformShiftCombine()
17799 SDValue N0 = N->getOperand(0); in PerformSplittingToWideningLoad()
17803 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() || in PerformSplittingToWideningLoad()
17804 LD->getExtensionType() != ISD::NON_EXTLOAD) in PerformSplittingToWideningLoad()
17806 EVT FromVT = LD->getValueType(0); in PerformSplittingToWideningLoad()
17807 EVT ToVT = N->getValueType(0); in PerformSplittingToWideningLoad()
17828 SDValue Ch = LD->getChain(); in PerformSplittingToWideningLoad()
17829 SDValue BasePtr = LD->getBasePtr(); in PerformSplittingToWideningLoad()
17830 Align Alignment = LD->getOriginalAlign(); in PerformSplittingToWideningLoad()
17831 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); in PerformSplittingToWideningLoad()
17832 AAMDNodes AAInfo = LD->getAAInfo(); in PerformSplittingToWideningLoad()
17835 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in PerformSplittingToWideningLoad()
17851 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT, in PerformSplittingToWideningLoad()
17877 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
17881 SDValue N0 = N->getOperand(0); in PerformExtendCombine()
17883 // Check for sign- and zero-extensions of vector extract operations of 8- and in PerformExtendCombine()
17884 // 16-bit vector elements. NEON and MVE support these directly. They are in PerformExtendCombine()
17886 // to 32-bit types and it is messy to recognize the operations after that. in PerformExtendCombine()
17887 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) && in PerformExtendCombine()
17891 EVT VT = N->getValueType(0); in PerformExtendCombine()
17901 switch (N->getOpcode()) { in PerformExtendCombine()
17915 if (ST->hasMVEIntegerOps()) in PerformExtendCombine()
17924 if (ST->hasMVEFloatOps()) in PerformFPExtendCombine()
17935 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) && in PerformMinMaxToSatCombine()
17936 !Subtarget->isThumb2()) in PerformMinMaxToSatCombine()
17972 /// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
17976 EVT VT = N->getValueType(0); in PerformMinMaxCombine()
17977 SDValue N0 = N->getOperand(0); in PerformMinMaxCombine()
17982 if (!ST->hasMVEIntegerOps()) in PerformMinMaxCombine()
17993 if (Min->getOpcode() != ISD::SMIN) in PerformMinMaxCombine()
17995 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX) in PerformMinMaxCombine()
18000 SaturateC = APInt(32, (1 << 15) - 1, true); in PerformMinMaxCombine()
18002 SaturateC = APInt(16, (1 << 7) - 1, true); in PerformMinMaxCombine()
18005 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) || in PerformMinMaxCombine()
18008 if (!ISD::isConstantSplatVector(Max->getOperand(1).getNode(), MaxC) || in PerformMinMaxCombine()
18030 N0->getOperand(0), DAG.getConstant(0, DL, MVT::i32)); in PerformMinMaxCombine()
18038 if (Min->getOpcode() != ISD::UMIN) in PerformMinMaxCombine()
18043 SaturateC = APInt(32, (1 << 16) - 1, true); in PerformMinMaxCombine()
18045 SaturateC = APInt(16, (1 << 8) - 1, true); in PerformMinMaxCombine()
18048 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) || in PerformMinMaxCombine()
18084 const APInt *CV = &C->getAPIntValue(); in isPowerOf2Constant()
18085 return CV->isPowerOf2() ? CV : nullptr; in isPowerOf2Constant()
18102 SDValue Op0 = CMOV->getOperand(0); in PerformCMOVToBFICombine()
18103 SDValue Op1 = CMOV->getOperand(1); in PerformCMOVToBFICombine()
18104 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue(); in PerformCMOVToBFICombine()
18105 SDValue CmpZ = CMOV->getOperand(4); in PerformCMOVToBFICombine()
18108 if (!isNullConstant(CmpZ->getOperand(1))) in PerformCMOVToBFICombine()
18111 assert(CmpZ->getOpcode() == ARMISD::CMPZ); in PerformCMOVToBFICombine()
18112 SDValue And = CmpZ->getOperand(0); in PerformCMOVToBFICombine()
18113 if (And->getOpcode() != ISD::AND) in PerformCMOVToBFICombine()
18115 const APInt *AndC = isPowerOf2Constant(And->getOperand(1)); in PerformCMOVToBFICombine()
18118 SDValue X = And->getOperand(0); in PerformCMOVToBFICombine()
18128 if (Op1->getOpcode() != ISD::OR) in PerformCMOVToBFICombine()
18131 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1)); in PerformCMOVToBFICombine()
18134 SDValue Y = Op1->getOperand(0); in PerformCMOVToBFICombine()
18140 APInt OrCI = OrC->getAPIntValue(); in PerformCMOVToBFICombine()
18141 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2; in PerformCMOVToBFICombine()
18155 unsigned BitInX = AndC->logBase2(); in PerformCMOVToBFICombine()
18185 switch (N->getOpcode()) { in SearchLoopIntrinsic()
18191 if (!cast<ConstantSDNode>(N.getOperand(1))->isOne()) in SearchLoopIntrinsic()
18200 if (Const->isZero()) in SearchLoopIntrinsic()
18202 else if (Const->isOne()) in SearchLoopIntrinsic()
18206 CC = cast<CondCodeSDNode>(N.getOperand(2))->get(); in SearchLoopIntrinsic()
18207 return SearchLoopIntrinsic(N->getOperand(0), CC, Imm, Negate); in SearchLoopIntrinsic()
18224 // The hwloop intrinsics that we're interested are used for control-flow, in PerformHWLoopCombine()
18226 // - test.start.loop.iterations will test whether its operand is zero. If it in PerformHWLoopCombine()
18228 // - loop.decrement.reg also tests whether its operand is zero. If it is in PerformHWLoopCombine()
18239 SDValue Chain = N->getOperand(0); in PerformHWLoopCombine()
18242 if (N->getOpcode() == ISD::BRCOND) { in PerformHWLoopCombine()
18244 Cond = N->getOperand(1); in PerformHWLoopCombine()
18245 Dest = N->getOperand(2); in PerformHWLoopCombine()
18247 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!"); in PerformHWLoopCombine()
18248 CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); in PerformHWLoopCombine()
18249 Cond = N->getOperand(2); in PerformHWLoopCombine()
18250 Dest = N->getOperand(4); in PerformHWLoopCombine()
18251 if (auto *Const = dyn_cast<ConstantSDNode>(N->getOperand(3))) { in PerformHWLoopCombine()
18252 if (!Const->isOne() && !Const->isZero()) in PerformHWLoopCombine()
18254 Imm = Const->getZExtValue(); in PerformHWLoopCombine()
18288 unsigned IntOp = Int->getConstantOperandVal(1); in PerformHWLoopCombine()
18289 assert((N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BR) in PerformHWLoopCombine()
18291 SDNode *Br = *N->use_begin(); in PerformHWLoopCombine()
18292 SDValue OtherTarget = Br->getOperand(1); in PerformHWLoopCombine()
18296 SDValue NewBrOps[] = { Br->getOperand(0), Dest }; in PerformHWLoopCombine()
18346 /// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18349 SDValue Cmp = N->getOperand(4); in PerformBRCONDCombine()
18354 EVT VT = N->getValueType(0); in PerformBRCONDCombine()
18358 SDValue Chain = N->getOperand(0); in PerformBRCONDCombine()
18359 SDValue BB = N->getOperand(1); in PerformBRCONDCombine()
18360 SDValue ARMcc = N->getOperand(2); in PerformBRCONDCombine()
18361 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal(); in PerformBRCONDCombine()
18364 // -> (brcond Chain BB CC CPSR Cmp) in PerformBRCONDCombine()
18365 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() && in PerformBRCONDCombine()
18366 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV && in PerformBRCONDCombine()
18367 LHS->getOperand(0)->hasOneUse() && in PerformBRCONDCombine()
18368 isNullConstant(LHS->getOperand(0)->getOperand(0)) && in PerformBRCONDCombine()
18369 isOneConstant(LHS->getOperand(0)->getOperand(1)) && in PerformBRCONDCombine()
18370 isOneConstant(LHS->getOperand(1)) && isNullConstant(RHS)) { in PerformBRCONDCombine()
18372 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2), in PerformBRCONDCombine()
18373 LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4)); in PerformBRCONDCombine()
18379 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18382 SDValue Cmp = N->getOperand(4); in PerformCMOVCombine()
18387 EVT VT = N->getValueType(0); in PerformCMOVCombine()
18391 SDValue FalseVal = N->getOperand(0); in PerformCMOVCombine()
18392 SDValue TrueVal = N->getOperand(1); in PerformCMOVCombine()
18393 SDValue ARMcc = N->getOperand(2); in PerformCMOVCombine()
18394 ARMCC::CondCodes CC = (ARMCC::CondCodes)ARMcc->getAsZExtVal(); in PerformCMOVCombine()
18397 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) { in PerformCMOVCombine()
18423 N->getOperand(3), Cmp); in PerformCMOVCombine()
18428 N->getOperand(3), NewCmp); in PerformCMOVCombine()
18432 // -> (cmov F T CC CPSR Cmp) in PerformCMOVCombine()
18433 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse() && in PerformCMOVCombine()
18434 isNullConstant(LHS->getOperand(0)) && isOneConstant(LHS->getOperand(1)) && in PerformCMOVCombine()
18437 LHS->getOperand(2), LHS->getOperand(3), in PerformCMOVCombine()
18438 LHS->getOperand(4)); in PerformCMOVCombine()
18445 // CMOV A, B, C1, $cpsr, (CMPZ (CMOV 1, 0, C2, D), 0) -> in PerformCMOVCombine()
18446 // if C1==EQ -> CMOV A, B, C2, $cpsr, D in PerformCMOVCombine()
18447 // if C1==NE -> CMOV A, B, NOT(C2), $cpsr, D in PerformCMOVCombine()
18448 if (N->getConstantOperandVal(2) == ARMCC::EQ || in PerformCMOVCombine()
18449 N->getConstantOperandVal(2) == ARMCC::NE) { in PerformCMOVCombine()
18451 if (SDValue C = IsCMPZCSINC(N->getOperand(4).getNode(), Cond)) { in PerformCMOVCombine()
18452 if (N->getConstantOperandVal(2) == ARMCC::NE) in PerformCMOVCombine()
18454 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0), in PerformCMOVCombine()
18455 N->getOperand(1), in PerformCMOVCombine()
18457 N->getOperand(3), C); in PerformCMOVCombine()
18464 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) { in PerformCMOVCombine()
18465 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it in PerformCMOVCombine()
18467 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5 in PerformCMOVCombine()
18472 // CMOV 0, 1, ==, (CMPZ x, y) -> in PerformCMOVCombine()
18476 // The USUBO_CARRY computes 0 - (x - y) and this will give a borrow when in PerformCMOVCombine()
18480 // x - y + (0 - (x - y)) + C == C in PerformCMOVCombine()
18492 (!Subtarget->isThumb1Only() || isPowerOf2Constant(TrueVal))) { in PerformCMOVCombine()
18494 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1 in PerformCMOVCombine()
18500 N->getOperand(3), CPSRGlue.getValue(1)); in PerformCMOVCombine()
18505 (!Subtarget->isThumb1Only() || isPowerOf2Constant(FalseVal))) { in PerformCMOVCombine()
18508 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBC x, y), z, !=, (SUBC x, y):1 in PerformCMOVCombine()
18515 N->getOperand(3), CPSRGlue.getValue(1)); in PerformCMOVCombine()
18522 // CMOV (SUBC x, y), z, !=, (SUBC x, y):1 -> in PerformCMOVCombine()
18529 // CMOV x, z, !=, (CMPZ x, 0) -> in PerformCMOVCombine()
18534 if (Subtarget->isThumb1Only() && CC == ARMCC::NE && in PerformCMOVCombine()
18540 unsigned ShiftAmount = TrueConst->logBase2(); in PerformCMOVCombine()
18573 SDValue Src = N->getOperand(0); in PerformBITCASTCombine()
18574 EVT DstVT = N->getValueType(0); in PerformBITCASTCombine()
18576 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE. in PerformBITCASTCombine()
18577 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) { in PerformBITCASTCombine()
18588 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that in PerformBITCASTCombine()
18598 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x in PerformBITCASTCombine()
18610 EVT VT = N->getValueType(0); in PerformMVETruncCombine()
18613 // MVETrunc(Undef, Undef) -> Undef in PerformMVETruncCombine()
18614 if (all_of(N->ops(), [](SDValue Op) { return Op.isUndef(); })) in PerformMVETruncCombine()
18617 // MVETrunc(MVETrunc a b, MVETrunc c, d) -> MVETrunc in PerformMVETruncCombine()
18618 if (N->getNumOperands() == 2 && in PerformMVETruncCombine()
18619 N->getOperand(0).getOpcode() == ARMISD::MVETRUNC && in PerformMVETruncCombine()
18620 N->getOperand(1).getOpcode() == ARMISD::MVETRUNC) in PerformMVETruncCombine()
18621 return DAG.getNode(ARMISD::MVETRUNC, DL, VT, N->getOperand(0).getOperand(0), in PerformMVETruncCombine()
18622 N->getOperand(0).getOperand(1), in PerformMVETruncCombine()
18623 N->getOperand(1).getOperand(0), in PerformMVETruncCombine()
18624 N->getOperand(1).getOperand(1)); in PerformMVETruncCombine()
18626 // MVETrunc(shuffle, shuffle) -> VMOVN in PerformMVETruncCombine()
18627 if (N->getNumOperands() == 2 && in PerformMVETruncCombine()
18628 N->getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE && in PerformMVETruncCombine()
18629 N->getOperand(1).getOpcode() == ISD::VECTOR_SHUFFLE) { in PerformMVETruncCombine()
18630 auto *S0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode()); in PerformMVETruncCombine()
18631 auto *S1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode()); in PerformMVETruncCombine()
18633 if (S0->getOperand(0) == S1->getOperand(0) && in PerformMVETruncCombine()
18634 S0->getOperand(1) == S1->getOperand(1)) { in PerformMVETruncCombine()
18636 SmallVector<int, 8> Mask(S0->getMask()); in PerformMVETruncCombine()
18637 Mask.append(S1->getMask().begin(), S1->getMask().end()); in PerformMVETruncCombine()
18642 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)), in PerformMVETruncCombine()
18643 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)), in PerformMVETruncCombine()
18648 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)), in PerformMVETruncCombine()
18649 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)), in PerformMVETruncCombine()
18656 if (all_of(N->ops(), [](SDValue Op) { in PerformMVETruncCombine()
18663 for (unsigned Op = 0; Op < N->getNumOperands(); Op++) { in PerformMVETruncCombine()
18664 SDValue O = N->getOperand(Op); in PerformMVETruncCombine()
18682 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in PerformMVETruncCombine()
18683 int NumIns = N->getNumOperands(); in PerformMVETruncCombine()
18687 if (N->getNumOperands() == 4) in PerformMVETruncCombine()
18697 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), DL, N->getOperand(I), in PerformMVETruncCombine()
18711 SDValue N0 = N->getOperand(0); in PerformSplittingMVEEXTToWideningLoad()
18713 if (!LD || !LD->isSimple() || !N0.hasOneUse() || LD->isIndexed()) in PerformSplittingMVEEXTToWideningLoad()
18716 EVT FromVT = LD->getMemoryVT(); in PerformSplittingMVEEXTToWideningLoad()
18717 EVT ToVT = N->getValueType(0); in PerformSplittingMVEEXTToWideningLoad()
18732 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in PerformSplittingMVEEXTToWideningLoad()
18733 if (LD->getExtensionType() != ISD::NON_EXTLOAD && in PerformSplittingMVEEXTToWideningLoad()
18734 LD->getExtensionType() != ISD::EXTLOAD && in PerformSplittingMVEEXTToWideningLoad()
18735 LD->getExtensionType() != NewExtType) in PerformSplittingMVEEXTToWideningLoad()
18741 SDValue Ch = LD->getChain(); in PerformSplittingMVEEXTToWideningLoad()
18742 SDValue BasePtr = LD->getBasePtr(); in PerformSplittingMVEEXTToWideningLoad()
18743 Align Alignment = LD->getOriginalAlign(); in PerformSplittingMVEEXTToWideningLoad()
18744 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); in PerformSplittingMVEEXTToWideningLoad()
18745 AAMDNodes AAInfo = LD->getAAInfo(); in PerformSplittingMVEEXTToWideningLoad()
18762 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT, in PerformSplittingMVEEXTToWideningLoad()
18779 EVT VT = N->getValueType(0); in PerformMVEExtCombine()
18781 assert(N->getNumValues() == 2 && "Expected MVEEXT with 2 elements"); in PerformMVEExtCombine()
18784 EVT ExtVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT( in PerformMVEExtCombine()
18788 return N->getOpcode() == ARMISD::MVESEXT in PerformMVEExtCombine()
18794 // MVEEXT(VDUP) -> SIGN_EXTEND_INREG(VDUP) in PerformMVEExtCombine()
18795 if (N->getOperand(0).getOpcode() == ARMISD::VDUP) { in PerformMVEExtCombine()
18796 SDValue Ext = Extend(N->getOperand(0)); in PerformMVEExtCombine()
18800 // MVEEXT(shuffle) -> SIGN_EXTEND_INREG/ZERO_EXTEND_INREG in PerformMVEExtCombine()
18801 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0))) { in PerformMVEExtCombine()
18802 ArrayRef<int> Mask = SVN->getMask(); in PerformMVEExtCombine()
18804 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements()); in PerformMVEExtCombine()
18806 SDValue Op0 = SVN->getOperand(0); in PerformMVEExtCombine()
18807 SDValue Op1 = SVN->getOperand(1); in PerformMVEExtCombine()
18820 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0)); in PerformMVEExtCombine()
18824 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1)); in PerformMVEExtCombine()
18829 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1)); in PerformMVEExtCombine()
18833 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0)); in PerformMVEExtCombine()
18839 // MVEEXT(load) -> extload, extload in PerformMVEExtCombine()
18840 if (N->getOperand(0)->getOpcode() == ISD::LOAD) in PerformMVEExtCombine()
18850 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in PerformMVEExtCombine()
18851 int NumOuts = N->getNumValues(); in PerformMVEExtCombine()
18854 EVT LoadVT = N->getOperand(0).getValueType().getHalfNumVectorElementsVT( in PerformMVEExtCombine()
18856 if (N->getNumOperands() == 4) in PerformMVEExtCombine()
18861 SDValue Chain = DAG.getStore(DAG.getEntryNode(), DL, N->getOperand(0), in PerformMVEExtCombine()
18872 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, DL, in PerformMVEExtCombine()
18882 switch (N->getOpcode()) { in PerformDAGCombine()
18999 unsigned BitWidth = N->getValueType(0).getSizeInBits(); in PerformDAGCombine()
19001 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)) in PerformDAGCombine()
19006 unsigned BitWidth = N->getValueType(0).getSizeInBits(); in PerformDAGCombine()
19008 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)) in PerformDAGCombine()
19017 unsigned BitWidth = N->getValueType(0).getSizeInBits(); in PerformDAGCombine()
19019 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || in PerformDAGCombine()
19020 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))) in PerformDAGCombine()
19025 unsigned LowWidth = N->getOperand(0).getValueType().getSizeInBits(); in PerformDAGCombine()
19027 unsigned HighWidth = N->getOperand(1).getValueType().getSizeInBits(); in PerformDAGCombine()
19029 if ((SimplifyDemandedBits(N->getOperand(0), LowMask, DCI)) || in PerformDAGCombine()
19030 (SimplifyDemandedBits(N->getOperand(1), HighMask, DCI))) in PerformDAGCombine()
19035 unsigned HighWidth = N->getOperand(0).getValueType().getSizeInBits(); in PerformDAGCombine()
19037 unsigned LowWidth = N->getOperand(1).getValueType().getSizeInBits(); in PerformDAGCombine()
19039 if ((SimplifyDemandedBits(N->getOperand(0), HighMask, DCI)) || in PerformDAGCombine()
19040 (SimplifyDemandedBits(N->getOperand(1), LowMask, DCI))) in PerformDAGCombine()
19045 unsigned BitWidth = N->getValueType(0).getSizeInBits(); in PerformDAGCombine()
19047 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || in PerformDAGCombine()
19048 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))) in PerformDAGCombine()
19056 unsigned BitWidth = N->getValueType(0).getSizeInBits(); in PerformDAGCombine()
19058 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) || in PerformDAGCombine()
19059 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))) in PerformDAGCombine()
19065 switch (N->getConstantOperandVal(1)) { in PerformDAGCombine()
19116 bool AllowsUnaligned = Subtarget->allowsUnalignedMem(); in allowsMisalignedMemoryAccesses()
19123 *Fast = Subtarget->hasV7Ops(); in allowsMisalignedMemoryAccesses()
19129 // For any little-endian targets with neon, we can support unaligned ld/st in allowsMisalignedMemoryAccesses()
19131 // A big-endian target may also explicitly support unaligned accesses in allowsMisalignedMemoryAccesses()
19132 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) { in allowsMisalignedMemoryAccesses()
19139 if (!Subtarget->hasMVEIntegerOps()) in allowsMisalignedMemoryAccesses()
19159 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and in allowsMisalignedMemoryAccesses()
19183 if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() && in getOptimalMemOpType()
19201 // Let the target-independent logic figure it out. in getOptimalMemOpType()
19205 // 64-bit integers are split into their high and low parts and held in two
19209 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy()) in isTruncateFree()
19211 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits(); in isTruncateFree()
19212 unsigned DestBits = DstTy->getPrimitiveSizeInBits(); in isTruncateFree()
19239 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits. in isZExtFree()
19258 return Subtarget->hasFullFP16(); in isFNegFree()
19268 return Ext->getType()->getScalarSizeInBits() == in areExtractExts()
19269 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits(); in areExtractExts()
19286 if (!I->getType()->isVectorTy()) in shouldSinkOperands()
19289 if (Subtarget->hasNEON()) { in shouldSinkOperands()
19290 switch (I->getOpcode()) { in shouldSinkOperands()
19293 if (!areExtractExts(I->getOperand(0), I->getOperand(1))) in shouldSinkOperands()
19295 Ops.push_back(&I->getOperandUse(0)); in shouldSinkOperands()
19296 Ops.push_back(&I->getOperandUse(1)); in shouldSinkOperands()
19304 if (!Subtarget->hasMVEIntegerOps()) in shouldSinkOperands()
19308 if (!I->hasOneUse()) in shouldSinkOperands()
19310 auto *Sub = cast<Instruction>(*I->users().begin()); in shouldSinkOperands()
19311 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) == I; in shouldSinkOperands()
19314 if (match(I->getOperand(0), m_FNeg(m_Value())) || in shouldSinkOperands()
19315 match(I->getOperand(1), m_FNeg(m_Value()))) in shouldSinkOperands()
19321 switch (I->getOpcode()) { in shouldSinkOperands()
19338 switch (II->getIntrinsicID()) { in shouldSinkOperands()
19373 for (auto OpIdx : enumerate(I->operands())) { in shouldSinkOperands()
19376 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; })) in shouldSinkOperands()
19380 if (Shuffle->getOpcode() == Instruction::BitCast) in shouldSinkOperands()
19381 Shuffle = dyn_cast<Instruction>(Shuffle->getOperand(0)); in shouldSinkOperands()
19393 for (Use &U : Op->uses()) { in shouldSinkOperands()
19399 Ops.push_back(&Shuffle->getOperandUse(0)); in shouldSinkOperands()
19401 Ops.push_back(&Op->getOperandUse(0)); in shouldSinkOperands()
19408 if (!Subtarget->hasMVEIntegerOps()) in shouldConvertSplatType()
19410 Type *SVIType = SVI->getType(); in shouldConvertSplatType()
19411 Type *ScalarType = SVIType->getScalarType(); in shouldConvertSplatType()
19413 if (ScalarType->isFloatTy()) in shouldConvertSplatType()
19414 return Type::getInt32Ty(SVIType->getContext()); in shouldConvertSplatType()
19415 if (ScalarType->isHalfTy()) in shouldConvertSplatType()
19416 return Type::getInt16Ty(SVIType->getContext()); in shouldConvertSplatType()
19427 if (Ld->isExpandingLoad()) in isVectorLoadExtDesirable()
19431 if (Subtarget->hasMVEIntegerOps()) in isVectorLoadExtDesirable()
19438 if (ExtVal->use_empty() || in isVectorLoadExtDesirable()
19439 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode())) in isVectorLoadExtDesirable()
19442 SDNode *U = *ExtVal->use_begin(); in isVectorLoadExtDesirable()
19443 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB || in isVectorLoadExtDesirable()
19444 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM)) in isVectorLoadExtDesirable()
19451 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) in allowTruncateForTailCall()
19457 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop"); in allowTruncateForTailCall()
19464 /// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
19469 /// ARM supports both fused and unfused multiply-add operations; we already
19475 /// patterns (and we don't have the non-fused floating point instruction).
19484 return Subtarget->hasMVEFloatOps(); in isFMAFasterThanFMulAndFAdd()
19486 return Subtarget->useFPVFMx16(); in isFMAFasterThanFMulAndFAdd()
19488 return Subtarget->useFPVFMx(); in isFMAFasterThanFMulAndFAdd()
19490 return Subtarget->useFPVFMx64(); in isFMAFasterThanFMulAndFAdd()
19519 if ((V & (Scale - 1)) != 0) in isLegalT1AddressImmediate()
19528 if (VT.isVector() && Subtarget->hasNEON()) in isLegalT2AddressImmediate()
19530 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() && in isLegalT2AddressImmediate()
19531 !Subtarget->hasMVEFloatOps()) in isLegalT2AddressImmediate()
19537 V = -V; in isLegalT2AddressImmediate()
19543 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) { in isLegalT2AddressImmediate()
19559 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16()) in isLegalT2AddressImmediate()
19562 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8) in isLegalT2AddressImmediate()
19566 // + imm12 or - imm8 in isLegalT2AddressImmediate()
19575 /// isLegalAddressImmediate - Return true if the integer value can be used
19586 if (Subtarget->isThumb1Only()) in isLegalAddressImmediate()
19588 else if (Subtarget->isThumb2()) in isLegalAddressImmediate()
19593 V = - V; in isLegalAddressImmediate()
19599 // +- imm12 in isLegalAddressImmediate()
19602 // +- imm8 in isLegalAddressImmediate()
19606 if (!Subtarget->hasVFP2Base()) // FIXME: NEON? in isLegalAddressImmediate()
19665 /// isLegalAddressingMode - Return true if the addressing mode represented
19689 if (Subtarget->isThumb1Only()) in isLegalAddressingMode()
19692 if (Subtarget->isThumb2()) in isLegalAddressingMode()
19701 if (Scale < 0) Scale = -Scale; in isLegalAddressingMode()
19708 // r +/- r in isLegalAddressingMode()
19709 if (Scale == 1 || (AM.HasBaseReg && Scale == -1)) in isLegalAddressingMode()
19729 /// isLegalICmpImmediate - Return true if the specified immediate is legal
19735 if (!Subtarget->isThumb()) in isLegalICmpImmediate()
19736 return ARM_AM::getSOImmVal((uint32_t)Imm) != -1 || in isLegalICmpImmediate()
19737 ARM_AM::getSOImmVal(-(uint32_t)Imm) != -1; in isLegalICmpImmediate()
19738 if (Subtarget->isThumb2()) in isLegalICmpImmediate()
19739 return ARM_AM::getT2SOImmVal((uint32_t)Imm) != -1 || in isLegalICmpImmediate()
19740 ARM_AM::getT2SOImmVal(-(uint32_t)Imm) != -1; in isLegalICmpImmediate()
19741 // Thumb1 doesn't have cmn, and only 8-bit immediates. in isLegalICmpImmediate()
19745 /// isLegalAddImmediate - Return true if the specified immediate is a legal add
19752 if (!Subtarget->isThumb()) in isLegalAddImmediate()
19753 return ARM_AM::getSOImmVal(AbsImm) != -1; in isLegalAddImmediate()
19754 if (Subtarget->isThumb2()) in isLegalAddImmediate()
19755 return ARM_AM::getT2SOImmVal(AbsImm) != -1; in isLegalAddImmediate()
19756 // Thumb1 only has 8-bit unsigned immediate. in isLegalAddImmediate()
19761 // (mul (add r, c0), c1) -> (add (mul r, c1), c0*c1) in DAGCombine,
19774 const int64_t C0 = C0Node->getSExtValue(); in isMulAddWithConstProfitable()
19775 APInt CA = C0Node->getAPIntValue() * C1Node->getAPIntValue(); in isMulAddWithConstProfitable()
19789 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getARMIndexedAddressParts()
19794 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
19795 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { in getARMIndexedAddressParts()
19796 int RHSC = (int)RHS->getZExtValue(); in getARMIndexedAddressParts()
19797 if (RHSC < 0 && RHSC > -256) { in getARMIndexedAddressParts()
19798 assert(Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19800 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getARMIndexedAddressParts()
19804 isInc = (Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19805 Offset = Ptr->getOperand(1); in getARMIndexedAddressParts()
19809 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { in getARMIndexedAddressParts()
19810 int RHSC = (int)RHS->getZExtValue(); in getARMIndexedAddressParts()
19811 if (RHSC < 0 && RHSC > -0x1000) { in getARMIndexedAddressParts()
19812 assert(Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19814 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getARMIndexedAddressParts()
19815 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
19820 if (Ptr->getOpcode() == ISD::ADD) { in getARMIndexedAddressParts()
19823 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode()); in getARMIndexedAddressParts()
19825 Base = Ptr->getOperand(1); in getARMIndexedAddressParts()
19826 Offset = Ptr->getOperand(0); in getARMIndexedAddressParts()
19828 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
19829 Offset = Ptr->getOperand(1); in getARMIndexedAddressParts()
19834 isInc = (Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19835 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
19836 Offset = Ptr->getOperand(1); in getARMIndexedAddressParts()
19848 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getT2IndexedAddressParts()
19851 Base = Ptr->getOperand(0); in getT2IndexedAddressParts()
19852 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { in getT2IndexedAddressParts()
19853 int RHSC = (int)RHS->getZExtValue(); in getT2IndexedAddressParts()
19854 if (RHSC < 0 && RHSC > -0x100) { // 8 bits. in getT2IndexedAddressParts()
19855 assert(Ptr->getOpcode() == ISD::ADD); in getT2IndexedAddressParts()
19857 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getT2IndexedAddressParts()
19860 isInc = Ptr->getOpcode() == ISD::ADD; in getT2IndexedAddressParts()
19861 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getT2IndexedAddressParts()
19873 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getMVEIndexedAddressParts()
19875 if (!isa<ConstantSDNode>(Ptr->getOperand(1))) in getMVEIndexedAddressParts()
19878 // We allow LE non-masked loads to change the type (for example use a vldrb.8 in getMVEIndexedAddressParts()
19883 ConstantSDNode *RHS = cast<ConstantSDNode>(Ptr->getOperand(1)); in getMVEIndexedAddressParts()
19884 int RHSC = (int)RHS->getZExtValue(); in getMVEIndexedAddressParts()
19887 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) { in getMVEIndexedAddressParts()
19888 assert(Ptr->getOpcode() == ISD::ADD); in getMVEIndexedAddressParts()
19890 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getMVEIndexedAddressParts()
19893 isInc = Ptr->getOpcode() == ISD::ADD; in getMVEIndexedAddressParts()
19894 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getMVEIndexedAddressParts()
19900 // Try to find a matching instruction based on s/zext, Alignment, Offset and in getMVEIndexedAddressParts()
19902 Base = Ptr->getOperand(0); in getMVEIndexedAddressParts()
19922 /// getPreIndexedAddressParts - returns true by value, base pointer and
19924 /// can be legally represented as pre-indexed load / store address.
19930 if (Subtarget->isThumb1Only()) in getPreIndexedAddressParts()
19939 Ptr = LD->getBasePtr(); in getPreIndexedAddressParts()
19940 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
19941 Alignment = LD->getAlign(); in getPreIndexedAddressParts()
19942 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPreIndexedAddressParts()
19944 Ptr = ST->getBasePtr(); in getPreIndexedAddressParts()
19945 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
19946 Alignment = ST->getAlign(); in getPreIndexedAddressParts()
19948 Ptr = LD->getBasePtr(); in getPreIndexedAddressParts()
19949 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
19950 Alignment = LD->getAlign(); in getPreIndexedAddressParts()
19951 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPreIndexedAddressParts()
19954 Ptr = ST->getBasePtr(); in getPreIndexedAddressParts()
19955 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
19956 Alignment = ST->getAlign(); in getPreIndexedAddressParts()
19964 isLegal = Subtarget->hasMVEIntegerOps() && in getPreIndexedAddressParts()
19967 Subtarget->isLittle(), Base, Offset, isInc, DAG); in getPreIndexedAddressParts()
19969 if (Subtarget->isThumb2()) in getPreIndexedAddressParts()
19983 /// getPostIndexedAddressParts - returns true by value, base pointer and
19985 /// combined with a load / store to form a post-indexed load / store.
19997 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
19998 Ptr = LD->getBasePtr(); in getPostIndexedAddressParts()
19999 Alignment = LD->getAlign(); in getPostIndexedAddressParts()
20000 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPostIndexedAddressParts()
20001 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD; in getPostIndexedAddressParts()
20003 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
20004 Ptr = ST->getBasePtr(); in getPostIndexedAddressParts()
20005 Alignment = ST->getAlign(); in getPostIndexedAddressParts()
20006 isNonExt = !ST->isTruncatingStore(); in getPostIndexedAddressParts()
20008 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
20009 Ptr = LD->getBasePtr(); in getPostIndexedAddressParts()
20010 Alignment = LD->getAlign(); in getPostIndexedAddressParts()
20011 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPostIndexedAddressParts()
20012 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD; in getPostIndexedAddressParts()
20015 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
20016 Ptr = ST->getBasePtr(); in getPostIndexedAddressParts()
20017 Alignment = ST->getAlign(); in getPostIndexedAddressParts()
20018 isNonExt = !ST->isTruncatingStore(); in getPostIndexedAddressParts()
20023 if (Subtarget->isThumb1Only()) { in getPostIndexedAddressParts()
20024 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It in getPostIndexedAddressParts()
20025 // must be non-extending/truncating, i32, with an offset of 4. in getPostIndexedAddressParts()
20026 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!"); in getPostIndexedAddressParts()
20027 if (Op->getOpcode() != ISD::ADD || !isNonExt) in getPostIndexedAddressParts()
20029 auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1)); in getPostIndexedAddressParts()
20030 if (!RHS || RHS->getZExtValue() != 4) in getPostIndexedAddressParts()
20035 Offset = Op->getOperand(1); in getPostIndexedAddressParts()
20036 Base = Op->getOperand(0); in getPostIndexedAddressParts()
20044 isLegal = Subtarget->hasMVEIntegerOps() && in getPostIndexedAddressParts()
20046 Subtarget->isLittle(), Base, Offset, in getPostIndexedAddressParts()
20049 if (Subtarget->isThumb2()) in getPostIndexedAddressParts()
20060 // Swap base ptr and offset to catch more post-index load / store when in getPostIndexedAddressParts()
20062 if (Ptr == Offset && Op->getOpcode() == ISD::ADD && in getPostIndexedAddressParts()
20063 !Subtarget->isThumb2()) in getPostIndexedAddressParts()
20066 // Post-indexed load / store update the base pointer. in getPostIndexedAddressParts()
20093 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(LHS) && in computeKnownBitsForTargetNode()
20095 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); in computeKnownBitsForTargetNode()
20112 static_cast<Intrinsic::ID>(Op->getConstantOperandVal(1)); in computeKnownBitsForTargetNode()
20117 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode()
20119 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); in computeKnownBitsForTargetNode()
20143 assert(Pos->getAPIntValue().ult(NumSrcElts) && in computeKnownBitsForTargetNode()
20145 unsigned Idx = Pos->getZExtValue(); in computeKnownBitsForTargetNode()
20164 KnownBits KnownOp = DAG.computeKnownBits(Op->getOperand(0), Depth + 1); in computeKnownBitsForTargetNode()
20172 KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1); in computeKnownBitsForTargetNode()
20173 KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1); in computeKnownBitsForTargetNode()
20178 // CSNEG: KnownOp0 or KnownOp1 * -1 in computeKnownBitsForTargetNode()
20187 KnownOp1, KnownBits::makeConstant(APInt(32, -1))); in computeKnownBitsForTargetNode()
20220 unsigned Mask = C->getZExtValue(); in targetShrinkDemandedConstant()
20226 // If the mask is all zeros, let the target-independent code replace the in targetShrinkDemandedConstant()
20231 // If the mask is all ones, erase the AND. (Currently, the target-independent in targetShrinkDemandedConstant()
20237 auto IsLegalMask = [ShrunkMask, ExpandedMask](unsigned Mask) -> bool { in targetShrinkDemandedConstant()
20240 auto UseMask = [Mask, Op, VT, &TLO](unsigned NewMask) -> bool { in targetShrinkDemandedConstant()
20262 // [-256, -2] is Thumb1 movs+bics, legal immediate for ARM/Thumb2. in targetShrinkDemandedConstant()
20264 if ((int)ExpandedMask <= -2 && (int)ExpandedMask >= -256) in targetShrinkDemandedConstant()
20271 // two-instruction sequence. in targetShrinkDemandedConstant()
20289 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(1) && in SimplifyDemandedBitsForTargetNode()
20290 isa<ConstantSDNode>(Op->getOperand(2))) { in SimplifyDemandedBitsForTargetNode()
20291 unsigned ShAmt = Op->getConstantOperandVal(2); in SimplifyDemandedBitsForTargetNode()
20293 << (32 - ShAmt))) in SimplifyDemandedBitsForTargetNode()
20297 TLO.DAG.getConstant(32 - ShAmt, SDLoc(Op), MVT::i32))); in SimplifyDemandedBitsForTargetNode()
20315 //===----------------------------------------------------------------------===//
20317 //===----------------------------------------------------------------------===//
20321 if (!Subtarget->hasV6Ops()) in ExpandInlineAsm()
20324 InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand()); in ExpandInlineAsm()
20325 StringRef AsmStr = IA->getAsmString(); in ExpandInlineAsm()
20339 IA->getConstraintString().compare(0, 4, "=l,l") == 0) { in ExpandInlineAsm()
20340 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); in ExpandInlineAsm()
20341 if (Ty && Ty->getBitWidth() == 32) in ExpandInlineAsm()
20358 if (!Subtarget->hasVFP2Base()) in LowerXConstraint()
20362 if (ConstraintVT.isVector() && Subtarget->hasNEON() && in LowerXConstraint()
20370 /// getConstraintType - Given a constraint letter, return the type of
20411 Type *type = CallOperandVal->getType(); in getSingleConstraintMatchWeight()
20418 if (type->isIntegerTy()) { in getSingleConstraintMatchWeight()
20419 if (Subtarget->isThumb()) in getSingleConstraintMatchWeight()
20426 if (type->isFloatingPointTy()) in getSingleConstraintMatchWeight()
20442 if (Subtarget->isThumb()) in getRegForInlineAsmConstraint()
20446 if (Subtarget->isThumb()) in getRegForInlineAsmConstraint()
20450 if (Subtarget->isThumb1Only()) in getRegForInlineAsmConstraint()
20509 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
20531 int64_t CVal64 = C->getSExtValue(); in LowerAsmOperandForConstraint()
20542 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps())) in LowerAsmOperandForConstraint()
20547 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20552 } else if (Subtarget->isThumb2()) { in LowerAsmOperandForConstraint()
20554 // data-processing instruction. in LowerAsmOperandForConstraint()
20555 if (ARM_AM::getT2SOImmVal(CVal) != -1) in LowerAsmOperandForConstraint()
20559 // data-processing instruction. in LowerAsmOperandForConstraint()
20560 if (ARM_AM::getSOImmVal(CVal) != -1) in LowerAsmOperandForConstraint()
20566 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20567 // This must be a constant between -255 and -1, for negated ADD in LowerAsmOperandForConstraint()
20571 if (CVal >= -255 && CVal <= -1) in LowerAsmOperandForConstraint()
20574 // This must be a constant between -4095 and 4095. It is not clear in LowerAsmOperandForConstraint()
20577 if (CVal >= -4095 && CVal <= 4095) in LowerAsmOperandForConstraint()
20583 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20584 // A 32-bit value where only one byte has a nonzero value. Exclude in LowerAsmOperandForConstraint()
20590 } else if (Subtarget->isThumb2()) { in LowerAsmOperandForConstraint()
20592 // value in a data-processing instruction. This can be used in GCC in LowerAsmOperandForConstraint()
20596 if (ARM_AM::getT2SOImmVal(~CVal) != -1) in LowerAsmOperandForConstraint()
20600 // value in a data-processing instruction. This can be used in GCC in LowerAsmOperandForConstraint()
20604 if (ARM_AM::getSOImmVal(~CVal) != -1) in LowerAsmOperandForConstraint()
20610 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20611 // This must be a constant between -7 and 7, in LowerAsmOperandForConstraint()
20612 // for 3-operand ADD/SUB immediate instructions. in LowerAsmOperandForConstraint()
20613 if (CVal >= -7 && CVal < 7) in LowerAsmOperandForConstraint()
20615 } else if (Subtarget->isThumb2()) { in LowerAsmOperandForConstraint()
20617 // data-processing instruction. This can be used in GCC with an "n" in LowerAsmOperandForConstraint()
20621 if (ARM_AM::getT2SOImmVal(-CVal) != -1) in LowerAsmOperandForConstraint()
20625 // data-processing instruction. This can be used in GCC with an "n" in LowerAsmOperandForConstraint()
20629 if (ARM_AM::getSOImmVal(-CVal) != -1) in LowerAsmOperandForConstraint()
20635 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20644 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0)) in LowerAsmOperandForConstraint()
20650 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20658 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
20659 // This must be a multiple of 4 between -508 and 508, for in LowerAsmOperandForConstraint()
20661 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0)) in LowerAsmOperandForConstraint()
20679 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || in getDivRemLibcall()
20680 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && in getDivRemLibcall()
20682 bool isSigned = N->getOpcode() == ISD::SDIVREM || in getDivRemLibcall()
20683 N->getOpcode() == ISD::SREM; in getDivRemLibcall()
20697 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || in getDivRemArgList()
20698 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && in getDivRemArgList()
20700 bool isSigned = N->getOpcode() == ISD::SDIVREM || in getDivRemArgList()
20701 N->getOpcode() == ISD::SREM; in getDivRemArgList()
20704 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { in getDivRemArgList()
20705 EVT ArgVT = N->getOperand(i).getValueType(); in getDivRemArgList()
20707 Entry.Node = N->getOperand(i); in getDivRemArgList()
20713 if (Subtarget->isTargetWindows() && Args.size() >= 2) in getDivRemArgList()
20719 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || in LowerDivRem()
20720 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || in LowerDivRem()
20721 Subtarget->isTargetWindows()) && in LowerDivRem()
20722 "Register-based DivRem lowering only"); in LowerDivRem()
20723 unsigned Opcode = Op->getOpcode(); in LowerDivRem()
20727 EVT VT = Op->getValueType(0); in LowerDivRem()
20737 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), in LowerDivRem()
20746 // rem = a - b * div in LowerDivRem()
20749 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode() in LowerDivRem()
20750 : Subtarget->hasDivideInARMMode(); in LowerDivRem()
20751 if (hasDivide && Op->getValueType(0).isSimple() && in LowerDivRem()
20752 Op->getSimpleValueType(0) == MVT::i32) { in LowerDivRem()
20754 const SDValue Dividend = Op->getOperand(0); in LowerDivRem()
20755 const SDValue Divisor = Op->getOperand(1); in LowerDivRem()
20777 if (Subtarget->isTargetWindows()) in LowerDivRem()
20792 EVT VT = N->getValueType(0); in LowerREM()
20794 if (VT == MVT::i64 && isa<ConstantSDNode>(N->getOperand(1))) { in LowerREM()
20797 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), N->getValueType(0), in LowerREM()
20818 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT(). in LowerREM()
20823 bool isSigned = N->getOpcode() == ISD::SREM; in LowerREM()
20827 if (Subtarget->isTargetWindows()) in LowerREM()
20839 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands"); in LowerREM()
20840 return ResNode->getOperand(1); in LowerREM()
20845 assert(Subtarget->isTargetWindows() && "unsupported target platform"); in LowerDYNAMIC_STACKALLOC()
20853 "no-stack-arg-probe")) { in LowerDYNAMIC_STACKALLOC()
20855 cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue(); in LowerDYNAMIC_STACKALLOC()
20862 DAG.getConstant(-(uint64_t)Align->value(), DL, MVT::i32)); in LowerDYNAMIC_STACKALLOC()
20886 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_EXTEND()
20891 "Unexpected type for custom-lowering FP_EXTEND"); in LowerFP_EXTEND()
20893 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) && in LowerFP_EXTEND()
20896 assert(!(DstSz == 32 && Subtarget->hasFP16()) && in LowerFP_EXTEND()
20899 // Converting from 32 -> 64 is valid if we have FP64. in LowerFP_EXTEND()
20900 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) { in LowerFP_EXTEND()
20911 // Either we are converting from 16 -> 64, without FP16 and/or in LowerFP_EXTEND()
20912 // FP.double-precision or without Armv8-fp. So we must do it in two in LowerFP_EXTEND()
20914 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32 in LowerFP_EXTEND()
20921 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64()); in LowerFP_EXTEND()
20935 "Unexpected type for custom-lowering FP_EXTEND"); in LowerFP_EXTEND()
20945 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_ROUND()
20954 "Unexpected type for custom-lowering FP_ROUND"); in LowerFP_ROUND()
20956 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) && in LowerFP_ROUND()
20961 // Instruction from 32 -> 16 if hasFP16 is valid in LowerFP_ROUND()
20962 if (SrcSz == 32 && Subtarget->hasFP16()) in LowerFP_ROUND()
20965 // Lib call from 32 -> 16 / 64 -> [32, 16] in LowerFP_ROUND()
20968 "Unexpected type for custom-lowering FP_ROUND"); in LowerFP_ROUND()
20992 /// isFPImmLegal - Returns true if the target can instruction select the
20997 if (!Subtarget->hasVFP3Base()) in isFPImmLegal()
20999 if (VT == MVT::f16 && Subtarget->hasFullFP16()) in isFPImmLegal()
21000 return ARM_AM::getFP16Imm(Imm) != -1; in isFPImmLegal()
21001 if (VT == MVT::f32 && Subtarget->hasFullFP16() && in isFPImmLegal()
21002 ARM_AM::getFP32FP16Imm(Imm) != -1) in isFPImmLegal()
21005 return ARM_AM::getFP32Imm(Imm) != -1; in isFPImmLegal()
21006 if (VT == MVT::f64 && Subtarget->hasFP64()) in isFPImmLegal()
21007 return ARM_AM::getFP64Imm(Imm) != -1; in isFPImmLegal()
21011 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
21033 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
21036 Value *AlignArg = I.getArgOperand(I.arg_size() - 1); in getTgtMemIntrinsic()
21037 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue(); in getTgtMemIntrinsic()
21049 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
21050 Info.ptrVal = I.getArgOperand(I.arg_size() - 1); in getTgtMemIntrinsic()
21069 Type *ArgTy = I.getArgOperand(ArgI)->getType(); in getTgtMemIntrinsic()
21070 if (!ArgTy->isVectorTy()) in getTgtMemIntrinsic()
21074 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
21077 Value *AlignArg = I.getArgOperand(I.arg_size() - 1); in getTgtMemIntrinsic()
21078 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue(); in getTgtMemIntrinsic()
21091 Type *ArgTy = I.getArgOperand(ArgI)->getType(); in getTgtMemIntrinsic()
21092 if (!ArgTy->isVectorTy()) in getTgtMemIntrinsic()
21096 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
21108 Type *VecTy = cast<StructType>(I.getType())->getElementType(1); in getTgtMemIntrinsic()
21110 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2); in getTgtMemIntrinsic()
21113 Info.align = Align(VecTy->getScalarSizeInBits() / 8); in getTgtMemIntrinsic()
21122 Type *VecTy = I.getArgOperand(1)->getType(); in getTgtMemIntrinsic()
21124 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2); in getTgtMemIntrinsic()
21127 Info.align = Align(VecTy->getScalarSizeInBits() / 8); in getTgtMemIntrinsic()
21145 Info.memVT = MVT::getVT(I.getType()->getContainedType(0)); in getTgtMemIntrinsic()
21155 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue(); in getTgtMemIntrinsic()
21166 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType()); in getTgtMemIntrinsic()
21175 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType()); in getTgtMemIntrinsic()
21184 MVT DataVT = MVT::getVT(I.getArgOperand(2)->getType()); in getTgtMemIntrinsic()
21185 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue(); in getTgtMemIntrinsic()
21247 assert(Ty->isIntegerTy()); in shouldConvertConstantLoadToIntImm()
21249 unsigned Bits = Ty->getPrimitiveSizeInBits(); in shouldConvertConstantLoadToIntImm()
21265 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in makeDMB()
21268 if (!Subtarget->hasDataBarrier()) { in makeDMB()
21270 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get in makeDMB()
21272 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) { in makeDMB()
21285 // Only a full system barrier exists in the M-class architectures. in makeDMB()
21286 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain; in makeDMB()
21292 // Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
21299 llvm_unreachable("Invalid fence: unordered/non-atomic"); in emitLeadingFence()
21304 if (!Inst->hasAtomicStore()) in emitLeadingFence()
21309 if (Subtarget->preferISHSTBarriers()) in emitLeadingFence()
21324 llvm_unreachable("Invalid fence: unordered/not-atomic"); in emitTrailingFence()
21336 // Loads and stores less than 64-bits are already atomic; ones above that
21343 if (Subtarget->isMClass()) in shouldExpandAtomicStoreInIR()
21345 else if (Subtarget->isThumb()) in shouldExpandAtomicStoreInIR()
21346 has64BitAtomicStore = Subtarget->hasV7Ops(); in shouldExpandAtomicStoreInIR()
21348 has64BitAtomicStore = Subtarget->hasV6Ops(); in shouldExpandAtomicStoreInIR()
21350 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicStoreInIR()
21355 // Loads and stores less than 64-bits are already atomic; ones above that
21361 // sections A8.8.72-74 LDRD)
21365 if (Subtarget->isMClass()) in shouldExpandAtomicLoadInIR()
21367 else if (Subtarget->isThumb()) in shouldExpandAtomicLoadInIR()
21368 has64BitAtomicLoad = Subtarget->hasV7Ops(); in shouldExpandAtomicLoadInIR()
21370 has64BitAtomicLoad = Subtarget->hasV6Ops(); in shouldExpandAtomicLoadInIR()
21372 unsigned Size = LI->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicLoadInIR()
21378 // and up to 64 bits on the non-M profiles
21381 if (AI->isFloatingPointOperation()) in shouldExpandAtomicRMWInIR()
21384 unsigned Size = AI->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicRMWInIR()
21386 if (Subtarget->isMClass()) in shouldExpandAtomicRMWInIR()
21387 hasAtomicRMW = Subtarget->hasV8MBaselineOps(); in shouldExpandAtomicRMWInIR()
21388 else if (Subtarget->isThumb()) in shouldExpandAtomicRMWInIR()
21389 hasAtomicRMW = Subtarget->hasV7Ops(); in shouldExpandAtomicRMWInIR()
21391 hasAtomicRMW = Subtarget->hasV6Ops(); in shouldExpandAtomicRMWInIR()
21392 if (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW) { in shouldExpandAtomicRMWInIR()
21393 // At -O0, fast-regalloc cannot cope with the live vregs necessary to in shouldExpandAtomicRMWInIR()
21397 // can never succeed. So at -O0 lower this operation to a CAS loop. in shouldExpandAtomicRMWInIR()
21406 // bits, and up to 64 bits on the non-M profiles.
21409 // At -O0, fast-regalloc cannot cope with the live vregs necessary to in shouldExpandAtomicCmpXchgInIR()
21413 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. in shouldExpandAtomicCmpXchgInIR()
21414 unsigned Size = AI->getOperand(1)->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicCmpXchgInIR()
21416 if (Subtarget->isMClass()) in shouldExpandAtomicCmpXchgInIR()
21417 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps(); in shouldExpandAtomicCmpXchgInIR()
21418 else if (Subtarget->isThumb()) in shouldExpandAtomicCmpXchgInIR()
21419 HasAtomicCmpXchg = Subtarget->hasV7Ops(); in shouldExpandAtomicCmpXchgInIR()
21421 HasAtomicCmpXchg = Subtarget->hasV6Ops(); in shouldExpandAtomicCmpXchgInIR()
21423 HasAtomicCmpXchg && Size <= (Subtarget->isMClass() ? 32U : 64U)) in shouldExpandAtomicCmpXchgInIR()
21435 return !Subtarget->isROPI() && !Subtarget->isRWPI(); in useLoadStackGuardNode()
21439 if (!Subtarget->getTargetTriple().isWindowsMSVCEnvironment()) in insertSSPDeclarations()
21451 F->addParamAttr(0, Attribute::AttrKind::InReg); in insertSSPDeclarations()
21456 if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment()) in getSDagStackGuard()
21463 if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment()) in getSSPStackGuardCheck()
21471 if (!Subtarget->hasNEON()) in canCombineStoreAndExtract()
21478 if (VectorTy->isFPOrFPVectorTy()) in canCombineStoreAndExtract()
21486 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type"); in canCombineStoreAndExtract()
21487 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedValue(); in canCombineStoreAndExtract()
21498 return Subtarget->hasV6T2Ops(); in isCheapToSpeculateCttz()
21502 return Subtarget->hasV6T2Ops(); in isCheapToSpeculateCtlz()
21507 if (!Subtarget->hasV7Ops()) in isMaskAndCmp0FoldingBeneficial()
21513 if (!Mask || Mask->getValue().getBitWidth() > 32u) in isMaskAndCmp0FoldingBeneficial()
21515 auto MaskVal = unsigned(Mask->getValue().getZExtValue()); in isMaskAndCmp0FoldingBeneficial()
21516 return (Subtarget->isThumb2() ? ARM_AM::getT2SOImmVal(MaskVal) in isMaskAndCmp0FoldingBeneficial()
21517 : ARM_AM::getSOImmVal(MaskVal)) != -1; in isMaskAndCmp0FoldingBeneficial()
21523 if (Subtarget->hasMinSize() && !Subtarget->isTargetWindows()) in preferredShiftLegalizationStrategy()
21532 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in emitLoadLinked()
21535 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd in emitLoadLinked()
21538 if (ValueTy->getPrimitiveSizeInBits() == 64) { in emitLoadLinked()
21547 if (!Subtarget->isLittle()) in emitLoadLinked()
21555 Type *Tys[] = { Addr->getType() }; in emitLoadLinked()
21560 CI->addParamAttr( in emitLoadLinked()
21561 0, Attribute::get(M->getContext(), Attribute::ElementType, ValueTy)); in emitLoadLinked()
21567 if (!Subtarget->hasV7Ops()) in emitAtomicCmpXchgNoStoreLLBalance()
21569 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in emitAtomicCmpXchgNoStoreLLBalance()
21576 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in emitStoreConditional()
21582 if (Val->getType()->getPrimitiveSizeInBits() == 64) { in emitStoreConditional()
21586 Type *Int32Ty = Type::getInt32Ty(M->getContext()); in emitStoreConditional()
21590 if (!Subtarget->isLittle()) in emitStoreConditional()
21596 Type *Tys[] = { Addr->getType() }; in emitStoreConditional()
21601 Val, Strex->getFunctionType()->getParamType(0)), in emitStoreConditional()
21603 CI->addParamAttr(1, Attribute::get(M->getContext(), Attribute::ElementType, in emitStoreConditional()
21604 Val->getType())); in emitStoreConditional()
21610 return Subtarget->isMClass(); in alignLoopsWithOptSize()
21626 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType()); in isLegalInterleavedAccessType()
21628 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps()) in isLegalInterleavedAccessType()
21634 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy()) in isLegalInterleavedAccessType()
21636 if (Subtarget->hasMVEIntegerOps() && Factor == 3) in isLegalInterleavedAccessType()
21640 if (VecTy->getNumElements() < 2) in isLegalInterleavedAccessType()
21647 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8) in isLegalInterleavedAccessType()
21652 if (Subtarget->hasNEON() && VecSize == 64) in isLegalInterleavedAccessType()
21658 if (Subtarget->hasNEON()) in getMaxSupportedInterleaveFactor()
21660 if (Subtarget->hasMVEIntegerOps()) in getMaxSupportedInterleaveFactor()
21685 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->getType()); in lowerInterleavedLoad()
21686 Type *EltTy = VecTy->getElementType(); in lowerInterleavedLoad()
21688 const DataLayout &DL = LI->getDataLayout(); in lowerInterleavedLoad()
21689 Align Alignment = LI->getAlign(); in lowerInterleavedLoad()
21701 if (EltTy->isPointerTy()) in lowerInterleavedLoad()
21707 Value *BaseAddr = LI->getPointerOperand(); in lowerInterleavedLoad()
21710 // If we're going to generate more than one load, reset the sub-vector type in lowerInterleavedLoad()
21712 VecTy = FixedVectorType::get(VecTy->getElementType(), in lowerInterleavedLoad()
21713 VecTy->getNumElements() / NumLoads); in lowerInterleavedLoad()
21719 if (Subtarget->hasNEON()) { in lowerInterleavedLoad()
21720 Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace()); in lowerInterleavedLoad()
21726 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys); in lowerInterleavedLoad()
21730 Ops.push_back(Builder.getInt32(LI->getAlign().value())); in lowerInterleavedLoad()
21738 Type *PtrTy = Builder.getPtrTy(LI->getPointerAddressSpace()); in lowerInterleavedLoad()
21741 Intrinsic::getDeclaration(LI->getModule(), LoadInts, Tys); in lowerInterleavedLoad()
21749 // Holds sub-vectors extracted from the load intrinsic return values. The in lowerInterleavedLoad()
21750 // sub-vectors are associated with the shufflevector instructions they will in lowerInterleavedLoad()
21758 BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr, in lowerInterleavedLoad()
21759 VecTy->getNumElements() * Factor); in lowerInterleavedLoad()
21772 if (EltTy->isPointerTy()) in lowerInterleavedLoad()
21775 FixedVectorType::get(SV->getType()->getElementType(), VecTy)); in lowerInterleavedLoad()
21781 // Replace uses of the shufflevector instructions with the sub-vectors in lowerInterleavedLoad()
21783 // associated with more than one sub-vector, those sub-vectors will be in lowerInterleavedLoad()
21789 SVI->replaceAllUsesWith(WideVec); in lowerInterleavedLoad()
21827 auto *VecTy = cast<FixedVectorType>(SVI->getType()); in lowerInterleavedStore()
21828 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store"); in lowerInterleavedStore()
21830 unsigned LaneLen = VecTy->getNumElements() / Factor; in lowerInterleavedStore()
21831 Type *EltTy = VecTy->getElementType(); in lowerInterleavedStore()
21834 const DataLayout &DL = SI->getDataLayout(); in lowerInterleavedStore()
21835 Align Alignment = SI->getAlign(); in lowerInterleavedStore()
21845 Value *Op0 = SVI->getOperand(0); in lowerInterleavedStore()
21846 Value *Op1 = SVI->getOperand(1); in lowerInterleavedStore()
21851 if (EltTy->isPointerTy()) { in lowerInterleavedStore()
21856 FixedVectorType::get(IntTy, cast<FixedVectorType>(Op0->getType())); in lowerInterleavedStore()
21864 Value *BaseAddr = SI->getPointerOperand(); in lowerInterleavedStore()
21868 // and sub-vector type to something legal. in lowerInterleavedStore()
21870 SubVecTy = FixedVectorType::get(SubVecTy->getElementType(), LaneLen); in lowerInterleavedStore()
21875 auto Mask = SVI->getShuffleMask(); in lowerInterleavedStore()
21879 if (Subtarget->hasNEON()) { in lowerInterleavedStore()
21883 Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace()); in lowerInterleavedStore()
21887 SI->getModule(), StoreInts[Factor - 2], Tys); in lowerInterleavedStore()
21892 Ops.push_back(Builder.getInt32(SI->getAlign().value())); in lowerInterleavedStore()
21899 Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace()); in lowerInterleavedStore()
21902 Intrinsic::getDeclaration(SI->getModule(), StoreInts, Tys); in lowerInterleavedStore()
21919 BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(), in lowerInterleavedStore()
21935 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ; in lowerInterleavedStore()
21966 for (unsigned i = 0; i < ST->getNumElements(); ++i) { in isHomogeneousAggregate()
21968 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers)) in isHomogeneousAggregate()
21974 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers)) in isHomogeneousAggregate()
21976 Members += SubMembers * AT->getNumElements(); in isHomogeneousAggregate()
21977 } else if (Ty->isFloatTy()) { in isHomogeneousAggregate()
21982 } else if (Ty->isDoubleTy()) { in isHomogeneousAggregate()
21994 return VT->getPrimitiveSizeInBits().getFixedValue() == 64; in isHomogeneousAggregate()
21996 return VT->getPrimitiveSizeInBits().getFixedValue() == 128; in isHomogeneousAggregate()
21998 switch (VT->getPrimitiveSizeInBits().getFixedValue()) { in isHomogeneousAggregate()
22018 if (!ArgTy->isVectorTy()) in getABIAlignmentForCallingConv()
22021 // Avoid over-aligning vector parameters. It would require realigning the in getABIAlignmentForCallingConv()
22026 /// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
22027 /// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
22039 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump()); in functionArgumentNeedsConsecutiveRegisters()
22041 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy(); in functionArgumentNeedsConsecutiveRegisters()
22049 return Subtarget->useSjLjEH() ? Register() : ARM::R0; in getExceptionPointerRegister()
22056 return Subtarget->useSjLjEH() ? Register() : ARM::R1; in getExceptionSelectorRegister()
22061 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>(); in initializeSplitCSR()
22062 AFI->setIsSplitCSR(true); in initializeSplitCSR()
22068 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); in insertCopiesSplitCSR()
22069 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); in insertCopiesSplitCSR()
22073 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in insertCopiesSplitCSR()
22074 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); in insertCopiesSplitCSR()
22075 MachineBasicBlock::iterator MBBI = Entry->begin(); in insertCopiesSplitCSR()
22085 Register NewVR = MRI->createVirtualRegister(RC); in insertCopiesSplitCSR()
22087 // FIXME: this currently does not emit CFI pseudo-instructions, it works in insertCopiesSplitCSR()
22088 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be in insertCopiesSplitCSR()
22090 // CFI pseudo-instructions. in insertCopiesSplitCSR()
22091 assert(Entry->getParent()->getFunction().hasFnAttribute( in insertCopiesSplitCSR()
22094 Entry->addLiveIn(*I); in insertCopiesSplitCSR()
22095 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) in insertCopiesSplitCSR()
22098 // Insert the copy-back instructions right before the terminator. in insertCopiesSplitCSR()
22100 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), in insertCopiesSplitCSR()
22101 TII->get(TargetOpcode::COPY), *I) in insertCopiesSplitCSR()
22112 return Subtarget->hasMVEIntegerOps(); in isComplexDeinterleavingSupported()
22121 auto *ScalarTy = VTy->getScalarType(); in isComplexDeinterleavingOperationSupported()
22122 unsigned NumElements = VTy->getNumElements(); in isComplexDeinterleavingOperationSupported()
22124 unsigned VTyWidth = VTy->getScalarSizeInBits() * NumElements; in isComplexDeinterleavingOperationSupported()
22129 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy()) in isComplexDeinterleavingOperationSupported()
22130 return Subtarget->hasMVEFloatOps(); in isComplexDeinterleavingOperationSupported()
22135 return Subtarget->hasMVEIntegerOps() && in isComplexDeinterleavingOperationSupported()
22136 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) || in isComplexDeinterleavingOperationSupported()
22137 ScalarTy->isIntegerTy(32)); in isComplexDeinterleavingOperationSupported()
22145 FixedVectorType *Ty = cast<FixedVectorType>(InputA->getType()); in createComplexDeinterleavingIR()
22147 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements(); in createComplexDeinterleavingIR()
22152 int Stride = Ty->getNumElements() / 2; in createComplexDeinterleavingIR()
22153 auto SplitSeq = llvm::seq<int>(0, Ty->getNumElements()); in createComplexDeinterleavingIR()
22175 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements()); in createComplexDeinterleavingIR()