Lines Matching +full:mid +full:- +full:scale
1 //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
16 //===----------------------------------------------------------------------===//
88 STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
89 STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
99 CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
103 UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
108 CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
109 cl::desc("Only use DAG-combiner alias analysis in this"
116 StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
121 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
125 EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true),
130 "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048),
134 "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10),
139 "combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
144 "combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true),
149 "combiner-vector-fcopysign-extend-round", cl::Hidden, cl::init(false),
169 /// This must behave as a stack -- new nodes to process are pushed onto the
191 // AA - Used for DAG load/store alias analysis.
202 for (SDNode *Node : N->uses()) in AddUsersToWorklist()
219 if (N->use_empty()) in clearAddedDanglingWorklistEntries()
235 assert(N->getCombinerWorklistIndex() >= 0 && in getNextWorklistEntry()
237 // Set to -2 to indicate that we combined the node. in getNextWorklistEntry()
238 N->setCombinerWorklistIndex(-2); in getNextWorklistEntry()
243 /// Call the node-specific routine that folds each particular type of node.
251 DisableGenericCombines = STI && STI->disableGenericCombines(OptLevel); in DAGCombiner()
272 assert(N->getOpcode() != ISD::DELETED_NODE && in AddToWorklist()
276 // zero-use deletion strategy. in AddToWorklist()
277 if (N->getOpcode() == ISD::HANDLENODE) in AddToWorklist()
280 if (SkipIfCombinedBefore && N->getCombinerWorklistIndex() == -2) in AddToWorklist()
286 if (N->getCombinerWorklistIndex() < 0) { in AddToWorklist()
287 N->setCombinerWorklistIndex(Worklist.size()); in AddToWorklist()
297 int WorklistIndex = N->getCombinerWorklistIndex(); in removeFromWorklist()
298 // If not in the worklist, the index might be -1 or -2 (was combined in removeFromWorklist()
306 N->setCombinerWorklistIndex(-1); in removeFromWorklist()
408 /// Call the node-specific routine that knows how to fold each
410 /// target-specific DAG combines.
413 // Visitation implementation - Implement dag node combining for different
416 // SDValue.getNode() == 0 - No change was made
417 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
418 // otherwise - N should be replaced by the returned Operand.
679 /// Walk up chain skipping non-aliasing memory nodes,
687 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
739 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
830 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
834 /// single-use) and if missed an empty SDValue is returned.
901 //===----------------------------------------------------------------------===//
903 //===----------------------------------------------------------------------===//
906 ((DAGCombiner*)DC)->AddToWorklist(N); in AddToWorklist()
911 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo); in CombineTo()
916 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo); in CombineTo()
921 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo); in CombineTo()
926 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N); in recursivelyDeleteUnusedNodes()
931 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO); in CommitTargetLoweringOpt()
934 //===----------------------------------------------------------------------===//
936 //===----------------------------------------------------------------------===//
942 // dead. Make sure to re-visit them and recursively delete dead nodes. in deleteAndRecombine()
943 for (const SDValue &Op : N->ops()) in deleteAndRecombine()
947 if (Op->hasOneUse() || Op->getNumValues() > 1) in deleteAndRecombine()
999 /// Return true if this is a SetCC-equivalent operation with only one use.
1004 if (isSetCCEquivalent(N, N0, N1, N2) && N->hasOneUse()) in isOneUseSetCC()
1041 return !(Const->isOpaque() && NoOpaques); in isConstantOrConstantVector()
1045 for (const SDValue &Op : N->op_values()) { in isConstantOrConstantVector()
1049 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth || in isConstantOrConstantVector()
1050 (Const->isOpaque() && NoOpaques)) in isConstantOrConstantVector()
1056 // Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
1068 (LD->getOperand(2).getOpcode() != ISD::TargetConstant || in canSplitIdx()
1069 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque()); in canSplitIdx()
1080 // (load/store (add, (add, x, offset1), offset2)) -> in reassociationCanBreakAddressingModePattern()
1083 // (load/store (add, (add, x, y), offset2)) -> in reassociationCanBreakAddressingModePattern()
1105 ScalableOffset = -ScalableOffset; in reassociationCanBreakAddressingModePattern()
1106 if (all_of(N->uses(), [&](SDNode *Node) { in reassociationCanBreakAddressingModePattern()
1108 LoadStore && LoadStore->getBasePtr().getNode() == N) { in reassociationCanBreakAddressingModePattern()
1112 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern()
1113 unsigned AS = LoadStore->getAddressSpace(); in reassociationCanBreakAddressingModePattern()
1130 const APInt &C2APIntVal = C2->getAPIntValue(); in reassociationCanBreakAddressingModePattern()
1138 const APInt &C1APIntVal = C1->getAPIntValue(); in reassociationCanBreakAddressingModePattern()
1144 for (SDNode *Node : N->uses()) { in reassociationCanBreakAddressingModePattern()
1152 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern()
1153 unsigned AS = LoadStore->getAddressSpace(); in reassociationCanBreakAddressingModePattern()
1166 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA)) in reassociationCanBreakAddressingModePattern()
1169 for (SDNode *Node : N->uses()) { in reassociationCanBreakAddressingModePattern()
1179 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern()
1180 unsigned AS = LoadStore->getAddressSpace(); in reassociationCanBreakAddressingModePattern()
1206 if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() && in reassociateOpsCommutative()
1211 // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2)) in reassociateOpsCommutative()
1217 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1) in reassociateOpsCommutative()
1226 // (N00 & N01) & N00 --> N00 & N01 in reassociateOpsCommutative()
1227 // (N00 & N01) & N01 --> N00 & N01 in reassociateOpsCommutative()
1228 // (N00 | N01) | N00 --> N00 | N01 in reassociateOpsCommutative()
1229 // (N00 | N01) | N01 --> N00 | N01 in reassociateOpsCommutative()
1234 // (N00 ^ N01) ^ N00 --> N01 in reassociateOpsCommutative()
1237 // (N00 ^ N01) ^ N01 --> N00 in reassociateOpsCommutative()
1271 if (N1->getOpcode() == ISD::SETCC && N00->getOpcode() == ISD::SETCC && in reassociateOpsCommutative()
1272 N01->getOpcode() == ISD::SETCC) { in reassociateOpsCommutative()
1273 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get(); in reassociateOpsCommutative()
1274 ISD::CondCode CC00 = cast<CondCodeSDNode>(N00.getOperand(2))->get(); in reassociateOpsCommutative()
1275 ISD::CondCode CC01 = cast<CondCodeSDNode>(N01.getOperand(2))->get(); in reassociateOpsCommutative()
1297 // Floating-point reassociation is not allowed without loose FP math. in reassociateOps()
1310 // Try to fold Opc(vecreduce(x), vecreduce(y)) -> vecreduce(Opc(x, y))
1318 N0->hasOneUse() && N1->hasOneUse() && in reassociateReduction()
1331 assert(N->getNumValues() == NumTo && "Broken CombineTo call!"); in CombineTo()
1333 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: "; in CombineTo()
1335 dbgs() << " and " << NumTo - 1 << " other values\n"); in CombineTo()
1338 N->getValueType(i) == To[i].getValueType()) && in CombineTo()
1354 if (N->use_empty()) in CombineTo()
1415 EVT VT = Load->getValueType(0); in ReplaceLoadWithPromotedLoad()
1418 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: "; in ReplaceLoadWithPromotedLoad()
1433 EVT MemVT = LD->getMemoryVT(); in PromoteOperand()
1435 : LD->getExtensionType(); in PromoteOperand()
1438 LD->getChain(), LD->getBasePtr(), in PromoteOperand()
1439 MemVT, LD->getMemOperand()); in PromoteOperand()
1538 Replace0 &= !N0->hasOneUse(); in PromoteIntBinOp()
1539 Replace1 &= (N0 != N1) && !N1->hasOneUse(); in PromoteIntBinOp()
1546 if (Replace0 && Replace1 && N0->isPredecessorOf(N1.getNode())) { in PromoteIntBinOp()
1635 // fold (aext (aext x)) -> (aext x) in PromoteExtend()
1636 // fold (aext (zext x)) -> (zext x) in PromoteExtend()
1637 // fold (aext (sext x)) -> (sext x) in PromoteExtend()
1670 EVT MemVT = LD->getMemoryVT(); in PromoteLoad()
1672 : LD->getExtensionType(); in PromoteLoad()
1674 LD->getChain(), LD->getBasePtr(), in PromoteLoad()
1675 MemVT, LD->getMemOperand()); in PromoteLoad()
1678 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: "; in PromoteLoad()
1699 if (!N->use_empty()) in recursivelyDeleteUnusedNodes()
1709 if (N->use_empty()) { in recursivelyDeleteUnusedNodes()
1710 for (const SDValue &ChildN : N->op_values()) in recursivelyDeleteUnusedNodes()
1722 //===----------------------------------------------------------------------===//
1724 //===----------------------------------------------------------------------===//
1759 // If this combine is running after legalizing the DAG, re-legalize any in Run()
1772 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG)); in Run()
1778 for (const SDValue &ChildN : N->op_values()) in Run()
1799 assert(N->getOpcode() != ISD::DELETED_NODE && in Run()
1805 if (N->getNumValues() == RV->getNumValues()) in Run()
1808 assert(N->getValueType(0) == RV.getValueType() && in Run()
1809 N->getNumValues() == 1 && "Type mismatch"); in Run()
1815 // out), because re-visiting the EntryToken and its users will not uncover in Run()
1834 // clang-format off in visit()
1835 switch (N->getOpcode()) { in visit()
1994 // clang-format on in visit()
2006 // If nothing happened, try a target-specific DAG combine. in combine()
2008 assert(N->getOpcode() != ISD::DELETED_NODE && in combine()
2011 if (N->getOpcode() >= ISD::BUILTIN_OP_END || in combine()
2012 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { in combine()
2024 switch (N->getOpcode()) { in combine()
2053 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode())) { in combine()
2054 SDValue N0 = N->getOperand(0); in combine()
2055 SDValue N1 = N->getOperand(1); in combine()
2060 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops, in combine()
2061 N->getFlags()); in combine()
2073 if (unsigned NumOps = N->getNumOperands()) { in getInputChainForNode()
2074 if (N->getOperand(0).getValueType() == MVT::Other) in getInputChainForNode()
2075 return N->getOperand(0); in getInputChainForNode()
2076 if (N->getOperand(NumOps-1).getValueType() == MVT::Other) in getInputChainForNode()
2077 return N->getOperand(NumOps-1); in getInputChainForNode()
2078 for (unsigned i = 1; i < NumOps-1; ++i) in getInputChainForNode()
2079 if (N->getOperand(i).getValueType() == MVT::Other) in getInputChainForNode()
2080 return N->getOperand(i); in getInputChainForNode()
2088 if (N->getNumOperands() == 2) { in visitTokenFactor()
2089 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1)) in visitTokenFactor()
2090 return N->getOperand(0); in visitTokenFactor()
2091 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0)) in visitTokenFactor()
2092 return N->getOperand(1); in visitTokenFactor()
2100 if (N->getNumOperands() > TokenFactorInlineLimit) in visitTokenFactor()
2106 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor) in visitTokenFactor()
2107 AddToWorklist(*(N->use_begin())); in visitTokenFactor()
2134 for (const SDValue &Op : TF->op_values()) { in visitTokenFactor()
2162 // Re-visit inlined Token Factors, to clean them up in case they have been in visitTokenFactor()
2168 // by walking up chains breath-first stopping when we've seen in visitTokenFactor()
2196 // Re-mark worklist from OrigOpNumber to OpNumber in visitTokenFactor()
2204 NumLeftToConsider--; in visitTokenFactor()
2221 switch (CurNode->getOpcode()) { in visitTokenFactor()
2230 for (const SDValue &Op : CurNode->op_values()) in visitTokenFactor()
2237 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber); in visitTokenFactor()
2241 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber); in visitTokenFactor()
2244 OpWorkCount[CurOpNumber]--; in visitTokenFactor()
2246 NumLeftToConsider--; in visitTokenFactor()
2285 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) in visitMERGE_VALUES()
2286 Ops.push_back(N->getOperand(i)); in visitMERGE_VALUES()
2288 } while (!N->use_empty()); in visitMERGE_VALUES()
2297 return Const != nullptr && !Const->isOpaque() ? Const : nullptr; in getAsNonOpaqueConstant()
2300 // isTruncateOf - If N is a truncate of some other value, return true, record
2306 if (N->getOpcode() == ISD::TRUNCATE) { in isTruncateOf()
2307 Op = N->getOperand(0); in isTruncateOf()
2329 if (LD->isIndexed() || LD->getBasePtr().getNode() != N) in canFoldInAddressingMode()
2331 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2332 AS = LD->getAddressSpace(); in canFoldInAddressingMode()
2334 if (ST->isIndexed() || ST->getBasePtr().getNode() != N) in canFoldInAddressingMode()
2336 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2337 AS = ST->getAddressSpace(); in canFoldInAddressingMode()
2339 if (LD->isIndexed() || LD->getBasePtr().getNode() != N) in canFoldInAddressingMode()
2341 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2342 AS = LD->getAddressSpace(); in canFoldInAddressingMode()
2344 if (ST->isIndexed() || ST->getBasePtr().getNode() != N) in canFoldInAddressingMode()
2346 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2347 AS = ST->getAddressSpace(); in canFoldInAddressingMode()
2353 if (N->getOpcode() == ISD::ADD) { in canFoldInAddressingMode()
2355 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); in canFoldInAddressingMode()
2357 // [reg +/- imm] in canFoldInAddressingMode()
2358 AM.BaseOffs = Offset->getSExtValue(); in canFoldInAddressingMode()
2360 // [reg +/- reg] in canFoldInAddressingMode()
2361 AM.Scale = 1; in canFoldInAddressingMode()
2362 } else if (N->getOpcode() == ISD::SUB) { in canFoldInAddressingMode()
2364 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); in canFoldInAddressingMode()
2366 // [reg +/- imm] in canFoldInAddressingMode()
2367 AM.BaseOffs = -Offset->getSExtValue(); in canFoldInAddressingMode()
2369 // [reg +/- reg] in canFoldInAddressingMode()
2370 AM.Scale = 1; in canFoldInAddressingMode()
2380 /// with an identity constant. Codegen improves if we re-use the variable
2386 // is only valid as operand 1 of a non-commutative binop. in foldSelectWithIdentityConstant()
2387 SDValue N0 = N->getOperand(0); in foldSelectWithIdentityConstant()
2388 SDValue N1 = N->getOperand(1); in foldSelectWithIdentityConstant()
2401 unsigned Opcode = N->getOpcode(); in foldSelectWithIdentityConstant()
2402 EVT VT = N->getValueType(0); in foldSelectWithIdentityConstant()
2408 // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal) in foldSelectWithIdentityConstant()
2410 if (isNeutralConstant(Opcode, N->getFlags(), TVal, OpNo)) { in foldSelectWithIdentityConstant()
2412 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags()); in foldSelectWithIdentityConstant()
2415 // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0 in foldSelectWithIdentityConstant()
2416 if (isNeutralConstant(Opcode, N->getFlags(), FVal, OpNo)) { in foldSelectWithIdentityConstant()
2418 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags()); in foldSelectWithIdentityConstant()
2426 assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && in foldBinOpIntoSelect()
2430 auto BinOpcode = BO->getOpcode(); in foldBinOpIntoSelect()
2431 EVT VT = BO->getValueType(0); in foldBinOpIntoSelect()
2436 if (TLI.isCommutativeBinOp(BO->getOpcode())) in foldBinOpIntoSelect()
2445 SDValue Sel = BO->getOperand(0); in foldBinOpIntoSelect()
2448 Sel = BO->getOperand(1); in foldBinOpIntoSelect()
2476 // The exception is "and" and "or" with either 0 or -1 in which case we can in foldBinOpIntoSelect()
2478 // and (select Cond, 0, -1), X --> select Cond, 0, X in foldBinOpIntoSelect()
2479 // or X, (select Cond, -1, 0) --> select Cond, -1, X in foldBinOpIntoSelect()
2485 SDValue CBO = BO->getOperand(SelOpNo ^ 1); in foldBinOpIntoSelect()
2508 // We have a select-of-constants followed by a binary operator with a in foldBinOpIntoSelect()
2510 // select. Example: add (select Cond, CT, CF), CBO --> select Cond, CT + in foldBinOpIntoSelect()
2524 SelectOp->setFlags(BO->getFlags()); in foldBinOpIntoSelect()
2530 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubBoolOfMaskedVal()
2536 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubBoolOfMaskedVal()
2537 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0); in foldAddSubBoolOfMaskedVal()
2538 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1); in foldAddSubBoolOfMaskedVal()
2554 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1)) in foldAddSubBoolOfMaskedVal()
2555 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1)) in foldAddSubBoolOfMaskedVal()
2558 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) in foldAddSubBoolOfMaskedVal()
2559 : DAG.getConstant(CN->getAPIntValue() - 1, DL, VT); in foldAddSubBoolOfMaskedVal()
2563 // Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1)
2565 SDValue N0 = N->getOperand(0); in foldSubToAvg()
2584 /// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2588 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubOfSignBit()
2593 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubOfSignBit()
2594 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0); in foldAddSubOfSignBit()
2595 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1); in foldAddSubOfSignBit()
2605 // The shift must be moving the sign bit to the least-significant-bit. in foldAddSubOfSignBit()
2609 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1)) in foldAddSubOfSignBit()
2613 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1) in foldAddSubOfSignBit()
2614 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1) in foldAddSubOfSignBit()
2636 SDValue N0 = N->getOperand(0); in visitADDLike()
2637 SDValue N1 = N->getOperand(1); in visitADDLike()
2641 // fold (add x, undef) -> undef in visitADDLike()
2647 // fold (add c1, c2) -> c1+c2 in visitADDLike()
2664 // fold (add x, 0) -> x, vector edition in visitADDLike()
2669 // fold (add x, 0) -> x in visitADDLike()
2677 // fold ((A-c1)+c2) -> (A+(c2-c1)) in visitADDLike()
2681 // fold ((c1-A)+c2) -> (c1+c2)-A in visitADDLike()
2686 // add (sext i1 X), 1 -> zext (not i1 X) in visitADDLike()
2688 // add (zext i1 X), -1 -> sext (not i1 X) in visitADDLike()
2702 // Fold (add (or x, c0), c1) -> (add x, (c0 + c1)) in visitADDLike()
2704 // Fold (add (xor x, c0), c1) -> (add x, (c0 + c1)) in visitADDLike()
2717 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) in visitADDLike()
2720 // Reassociate (add (or x, c), y) -> (add add(x, y), c)) if (or x, c) is in visitADDLike()
2722 // Reassociate (add (xor x, c), y) -> (add add(x, y), c)) if (xor x, c) is in visitADDLike()
2748 // Fold add(vecreduce(x), vecreduce(y)) -> vecreduce(add(x, y)) in visitADDLike()
2756 // fold ((0-A) + B) -> B-A in visitADDLike()
2760 // fold (A + (0-B)) -> A-B in visitADDLike()
2764 // fold (A+(B-A)) -> B in visitADDLike()
2768 // fold ((B-A)+A) -> B in visitADDLike()
2772 // fold ((A-B)+(C-A)) -> (C-B) in visitADDLike()
2777 // fold ((A-B)+(B-C)) -> (A-C) in visitADDLike()
2782 // fold (A+(B-(A+C))) to (B-C) in visitADDLike()
2783 // fold (A+(B-(C+A))) to (B-C) in visitADDLike()
2787 // fold (A+((B-A)+or-C)) to (B+or-C) in visitADDLike()
2793 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant in visitADDLike()
2801 // fold (add (umax X, C), -C) --> (usubsat X, C) in visitADDLike()
2805 (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue())); in visitADDLike()
2817 // fold (add (xor a, -1), 1) -> (sub 0, a) in visitADDLike()
2822 // fold (add (add (xor a, -1), b), 1) -> (sub b, a) in visitADDLike()
2841 // sub y, (xor x, -1) in visitADDLike()
2845 (Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2846 !N->getFlags().hasNoSignedWrap()))) { in visitADDLike()
2852 // (x - y) + -1 -> add (xor y, -1), x in visitADDLike()
2859 // Fold add(mul(add(A, CA), CM), CB) -> add(mul(A, CM), CM*CA+CB). in visitADDLike()
2867 (CA * CM + CB->getAPIntValue()).getSExtValue())) { in visitADDLike()
2871 if (N->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2872 N0->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2873 N0.getOperand(0)->getFlags().hasNoUnsignedWrap()) { in visitADDLike()
2875 if (N->getFlags().hasNoSignedWrap() && in visitADDLike()
2876 N0->getFlags().hasNoSignedWrap() && in visitADDLike()
2877 N0.getOperand(0)->getFlags().hasNoSignedWrap()) in visitADDLike()
2884 DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags); in visitADDLike()
2892 (CA * CM + CB->getAPIntValue()).getSExtValue())) { in visitADDLike()
2898 if (N->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2899 N0->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2900 OMul->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2901 OMul.getOperand(0)->getFlags().hasNoUnsignedWrap()) { in visitADDLike()
2903 if (N->getFlags().hasNoSignedWrap() && in visitADDLike()
2904 N0->getFlags().hasNoSignedWrap() && in visitADDLike()
2905 OMul->getFlags().hasNoSignedWrap() && in visitADDLike()
2906 OMul.getOperand(0)->getFlags().hasNoSignedWrap()) in visitADDLike()
2914 DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags); in visitADDLike()
2930 SDValue N0 = N->getOperand(0); in foldAddToAvg()
2951 SDValue N0 = N->getOperand(0); in visitADD()
2952 SDValue N1 = N->getOperand(1); in visitADD()
2969 // fold (a+b) -> (a|b) iff a and b share no bits. in visitADD()
2979 const APInt &C0 = N0->getConstantOperandAPInt(0); in visitADD()
2980 const APInt &C1 = N1->getConstantOperandAPInt(0); in visitADD()
2984 // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2) in visitADD()
2988 const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0); in visitADD()
2989 const APInt &VS1 = N1->getConstantOperandAPInt(0); in visitADD()
2997 const APInt &C0 = N0->getConstantOperandAPInt(0); in visitADD()
2998 const APInt &C1 = N1->getConstantOperandAPInt(0); in visitADD()
3007 const APInt &SV0 = N0.getOperand(1)->getConstantOperandAPInt(0); in visitADD()
3008 const APInt &SV1 = N1->getConstantOperandAPInt(0); in visitADD()
3018 unsigned Opcode = N->getOpcode(); in visitADDSAT()
3019 SDValue N0 = N->getOperand(0); in visitADDSAT()
3020 SDValue N1 = N->getOperand(1); in visitADDSAT()
3025 // fold (add_sat x, undef) -> -1 in visitADDSAT()
3029 // fold (add_sat c1, c2) -> c3 in visitADDSAT()
3043 // fold (add_sat x, 0) -> x, vector edition in visitADDSAT()
3048 // fold (add_sat x, 0) -> x in visitADDSAT()
3093 EVT VT = V->getValueType(0); in getAsCarry()
3109 /// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
3116 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1))) in foldAddSubMasked1()
3130 // add N0, (and (AssertSext X, i1), 1) --> sub N0, X in foldAddSubMasked1()
3131 // sub N0, (and (AssertSext X, i1), 1) --> add N0, X in foldAddSubMasked1()
3141 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n)) in visitADDLikeCommutative()
3153 // sub y, (xor x, -1) in visitADDLikeCommutative()
3157 (Level >= AfterLegalizeDAG || (!N0->getFlags().hasNoUnsignedWrap() && in visitADDLikeCommutative()
3158 !N0->getFlags().hasNoSignedWrap()))) { in visitADDLikeCommutative()
3164 // Hoist one-use subtraction by non-opaque constant: in visitADDLikeCommutative()
3165 // (x - C) + y -> (x + y) - C in visitADDLikeCommutative()
3166 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors. in visitADDLikeCommutative()
3171 // Hoist one-use subtraction from non-opaque constant: in visitADDLikeCommutative()
3172 // (C - x) + y -> (y - x) + C in visitADDLikeCommutative()
3179 // add (mul x, C), x -> mul x, C+1 in visitADDLikeCommutative()
3189 // rather than 'add 0/-1' (the zext should get folded). in visitADDLikeCommutative()
3190 // add (sext i1 Y), X --> sub X, (zext i1 Y) in visitADDLikeCommutative()
3198 // add X, (sextinreg Y i1) -> sub X, (and Y 1) in visitADDLikeCommutative()
3201 if (TN->getVT() == MVT::i1) { in visitADDLikeCommutative()
3208 // (add X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry) in visitADDLikeCommutative()
3211 return DAG.getNode(ISD::UADDO_CARRY, DL, N1->getVTList(), in visitADDLikeCommutative()
3214 // (add X, Carry) -> (uaddo_carry X, 0, Carry) in visitADDLikeCommutative()
3225 SDValue N0 = N->getOperand(0); in visitADDC()
3226 SDValue N1 = N->getOperand(1); in visitADDC()
3231 if (!N->hasAnyUseOfValue(1)) in visitADDC()
3239 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0); in visitADDC()
3241 // fold (addc x, 0) -> x + no carry out in visitADDC()
3279 IsFlip = Const->isOne(); in extractBooleanFlip()
3282 IsFlip = Const->isAllOnes(); in extractBooleanFlip()
3285 IsFlip = (Const->getAPIntValue() & 0x01) == 1; in extractBooleanFlip()
3297 SDValue N0 = N->getOperand(0); in visitADDO()
3298 SDValue N1 = N->getOperand(1); in visitADDO()
3300 bool IsSigned = (ISD::SADDO == N->getOpcode()); in visitADDO()
3302 EVT CarryVT = N->getValueType(1); in visitADDO()
3306 if (!N->hasAnyUseOfValue(1)) in visitADDO()
3313 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0); in visitADDO()
3315 // fold (addo x, 0) -> x + no carry out in visitADDO()
3325 // fold (saddo (xor a, -1), 1) -> (ssub 0, a). in visitADDO()
3327 return DAG.getNode(ISD::SSUBO, DL, N->getVTList(), in visitADDO()
3330 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry. in visitADDO()
3332 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(), in visitADDO()
3335 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1))); in visitADDO()
3353 // (uaddo X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry) in visitUADDOLike()
3359 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y, in visitUADDOLike()
3363 // (uaddo X, Carry) -> (uaddo_carry X, 0, Carry) in visitUADDOLike()
3366 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, in visitUADDOLike()
3373 SDValue N0 = N->getOperand(0); in visitADDE()
3374 SDValue N1 = N->getOperand(1); in visitADDE()
3375 SDValue CarryIn = N->getOperand(2); in visitADDE()
3381 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(), in visitADDE()
3384 // fold (adde x, y, false) -> (addc x, y) in visitADDE()
3386 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); in visitADDE()
3392 SDValue N0 = N->getOperand(0); in visitUADDO_CARRY()
3393 SDValue N1 = N->getOperand(1); in visitUADDO_CARRY()
3394 SDValue CarryIn = N->getOperand(2); in visitUADDO_CARRY()
3401 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitUADDO_CARRY()
3403 // fold (uaddo_carry x, y, false) -> (uaddo x, y) in visitUADDO_CARRY()
3406 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0))) in visitUADDO_CARRY()
3407 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); in visitUADDO_CARRY()
3410 // fold (uaddo_carry 0, 0, X) -> (and (ext/trunc X), 1) and no carry. in visitUADDO_CARRY()
3434 DAG.getNodeIfExists(ISD::UADDO_CARRY, N->getVTList(), Ops, N->getFlags()); in visitUADDO_CARRY()
3483 EVT VT = Carry0->getValueType(1); in combineUADDO_CARRYDiamond()
3494 DAG.getNode(ISD::UADDO_CARRY, DL, Carry0->getVTList(), A, B, Z); in combineUADDO_CARRYDiamond()
3496 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), X, in combineUADDO_CARRYDiamond()
3571 EVT CarryOutType = N->getValueType(0); in combineCarryDiamond()
3578 if (Carry1.getNode()->isOperandOf(Carry0.getNode())) in combineCarryDiamond()
3603 CarryIn = DAG.getBoolExtOrTrunc(CarryIn, DL, Carry1->getValueType(1), in combineCarryDiamond()
3604 Carry1->getValueType(0)); in combineCarryDiamond()
3606 DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0), in combineCarryDiamond()
3612 // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the in combineCarryDiamond()
3616 // 0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow) in combineCarryDiamond()
3623 if (N->getOpcode() == ISD::AND) in combineCarryDiamond()
3630 // fold (uaddo_carry (xor a, -1), b, c) -> (usubo_carry b, a, !c) and flip in visitUADDO_CARRYLike()
3635 SDValue Sub = DAG.getNode(ISD::USUBO_CARRY, DL, N->getVTList(), N1, in visitUADDO_CARRYLike()
3638 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1))); in visitUADDO_CARRYLike()
3642 // (uaddo_carry (add|uaddo X, Y), 0, Carry) -> (uaddo_carry X, Y, Carry) in visitUADDO_CARRYLike()
3648 isNullConstant(N1) && !N->hasAnyUseOfValue(1)) in visitUADDO_CARRYLike()
3649 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), in visitUADDO_CARRYLike()
3670 // fold (saddo_carry (xor a, -1), b, c) -> (ssubo_carry b, a, !c) in visitSADDO_CARRYLike()
3673 return DAG.getNode(ISD::SSUBO_CARRY, SDLoc(N), N->getVTList(), N1, in visitSADDO_CARRYLike()
3681 SDValue N0 = N->getOperand(0); in visitSADDO_CARRY()
3682 SDValue N1 = N->getOperand(1); in visitSADDO_CARRY()
3683 SDValue CarryIn = N->getOperand(2); in visitSADDO_CARRY()
3690 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitSADDO_CARRY()
3692 // fold (saddo_carry x, y, false) -> (saddo x, y) in visitSADDO_CARRY()
3695 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0))) in visitSADDO_CARRY()
3696 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1); in visitSADDO_CARRY()
3719 // If the LHS is zero-extended then we can perform the USUBSAT as DstVT by in getTruncatedUSUBSAT()
3736 // Try to find umax(a,b) - b or a - umin(a,b) patterns that may be converted to
3739 if (N->getOpcode() != ISD::SUB || in foldSubToUSubSat()
3743 EVT SubVT = N->getValueType(0); in foldSubToUSubSat()
3744 SDValue Op0 = N->getOperand(0); in foldSubToUSubSat()
3745 SDValue Op1 = N->getOperand(1); in foldSubToUSubSat()
3747 // Try to find umax(a,b) - b or a - umin(a,b) patterns in foldSubToUSubSat()
3767 // sub(a,trunc(umin(zext(a),b))) -> usubsat(a,trunc(umin(b,SatLimit))) in foldSubToUSubSat()
3796 SDValue N0 = N->getOperand(0); in visitSUB()
3797 SDValue N1 = N->getOperand(1); in visitSUB()
3803 if (N->getOpcode() == ISD::FREEZE && N.hasOneUse()) in visitSUB()
3804 return N->getOperand(0); in visitSUB()
3808 // fold (sub x, x) -> 0 in visitSUB()
3813 // fold (sub c1, c2) -> c3 in visitSUB()
3822 // fold (sub x, 0) -> x, vector edition in visitSUB()
3830 // fold (sub x, c) -> (add x, -c) in visitSUB()
3833 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUB()
3836 // Right-shifting everything out but the sign bit followed by negation is in visitSUB()
3838 // -(X >>u 31) -> (X >>s 31) in visitSUB()
3839 // -(X >>s 31) -> (X >>u 31) in visitSUB()
3840 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) { in visitSUB()
3842 if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) { in visitSUB()
3843 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA; in visitSUB()
3849 // 0 - X --> 0 if the sub is NUW. in visitSUB()
3850 if (N->getFlags().hasNoUnsignedWrap()) in visitSUB()
3856 if (N->getFlags().hasNoSignedWrap()) in visitSUB()
3859 // 0 - X --> X if X is 0 or the minimum signed value. in visitSUB()
3863 // Convert 0 - abs(x). in visitSUB()
3869 // Fold neg(splat(neg(x)) -> splat(x) in visitSUB()
3878 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) in visitSUB()
3882 // fold (A - (0-B)) -> A+B in visitSUB()
3886 // fold A-(A-B) -> B in visitSUB()
3890 // fold (A+B)-A -> B in visitSUB()
3894 // fold (A+B)-B -> A in visitSUB()
3898 // fold (A+C1)-C2 -> A+(C1-C2) in visitSUB()
3905 // fold C2-(A+C1) -> (C2-C1)-A in visitSUB()
3912 // fold (A-C1)-C2 -> A-(C1+C2) in visitSUB()
3919 // fold (c1-A)-c2 -> (c1-c2)-A in visitSUB()
3928 // fold ((A+(B+C))-B) -> A+C in visitSUB()
3932 // fold ((A+(B-C))-B) -> A-C in visitSUB()
3936 // fold ((A-(B-C))-C) -> A-B in visitSUB()
3940 // fold (A-(B-C)) -> A+(C-B) in visitSUB()
3945 // A - (A & B) -> A & (~B) in visitSUB()
3950 // fold (A - (-B * C)) -> (A + (B * C)) in visitSUB()
3977 // (A - B) - 1 -> add (xor B, -1), A in visitSUB()
3982 // sub y, (xor x, -1) in visitSUB()
3990 // Hoist one-use addition by non-opaque constant: in visitSUB()
3991 // (x + C) - y -> (x - y) + C in visitSUB()
3998 // y - (x + C) -> (y - x) - C in visitSUB()
4004 // (x - C) - y -> (x - y) - C in visitSUB()
4005 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors. in visitSUB()
4011 // (C - x) - y -> C - (x + y) in visitSUB()
4018 // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1' in visitSUB()
4020 // sub X, (zext i1 Y) --> add X, (sext i1 Y) in visitSUB()
4029 // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A) in visitSUB()
4031 sd_match(N1, m_Sra(m_Value(A), m_SpecificInt(BitWidth - 1))) && in visitSUB()
4038 // fold (sub Sym+c1, Sym+c2) -> c1-c2 in visitSUB()
4040 if (GA->getGlobal() == GB->getGlobal()) in visitSUB()
4041 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(), in visitSUB()
4045 // sub X, (sextinreg Y i1) -> add X, (and Y 1) in visitSUB()
4048 if (TN->getVT() == MVT::i1) { in visitSUB()
4055 // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C)) in visitSUB()
4058 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
4061 // canonicalize (sub X, step_vector(C)) to (add X, step_vector(-C)) in visitSUB()
4063 APInt NewStep = -N1.getConstantOperandAPInt(0); in visitSUB()
4069 // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1) in visitSUB()
4073 if (ShAmtC && ShAmtC->getAPIntValue() == (BitWidth - 1)) { in visitSUB()
4081 // N0 - (X << BW-1) --> N0 + (X << BW-1) in visitSUB()
4084 if (ShlC && ShlC->getAPIntValue() == (BitWidth - 1)) in visitSUB()
4088 // (sub (usubo_carry X, 0, Carry), Y) -> (usubo_carry X, Y, Carry) in visitSUB()
4091 return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(), in visitSUB()
4095 // (sub Carry, X) -> (uaddo_carry (sub 0, X), 0, Carry) in visitSUB()
4107 // sub C0, X --> xor X, C0 in visitSUB()
4109 if (!C0->isOpaque()) { in visitSUB()
4110 const APInt &C0Val = C0->getAPIntValue(); in visitSUB()
4112 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes)) in visitSUB()
4117 // smax(a,b) - smin(a,b) --> abds(a,b) in visitSUB()
4123 // umax(a,b) - umin(a,b) --> abdu(a,b) in visitSUB()
4133 unsigned Opcode = N->getOpcode(); in visitSUBSAT()
4134 SDValue N0 = N->getOperand(0); in visitSUBSAT()
4135 SDValue N1 = N->getOperand(1); in visitSUBSAT()
4140 // fold (sub_sat x, undef) -> 0 in visitSUBSAT()
4144 // fold (sub_sat x, x) -> 0 in visitSUBSAT()
4148 // fold (sub_sat c1, c2) -> c3 in visitSUBSAT()
4157 // fold (sub_sat x, 0) -> x, vector edition in visitSUBSAT()
4162 // fold (sub_sat x, 0) -> x in visitSUBSAT()
4174 SDValue N0 = N->getOperand(0); in visitSUBC()
4175 SDValue N1 = N->getOperand(1); in visitSUBC()
4180 if (!N->hasAnyUseOfValue(1)) in visitSUBC()
4184 // fold (subc x, x) -> 0 + no borrow in visitSUBC()
4189 // fold (subc x, 0) -> x + no borrow in visitSUBC()
4193 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow in visitSUBC()
4202 SDValue N0 = N->getOperand(0); in visitSUBO()
4203 SDValue N1 = N->getOperand(1); in visitSUBO()
4205 bool IsSigned = (ISD::SSUBO == N->getOpcode()); in visitSUBO()
4207 EVT CarryVT = N->getValueType(1); in visitSUBO()
4211 if (!N->hasAnyUseOfValue(1)) in visitSUBO()
4215 // fold (subo x, x) -> 0 + no borrow in visitSUBO()
4220 // fold (subox, c) -> (addo x, -c) in visitSUBO()
4222 if (IsSigned && !N1C->isMinSignedValue()) in visitSUBO()
4223 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, in visitSUBO()
4224 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUBO()
4226 // fold (subo x, 0) -> x + no borrow in visitSUBO()
4235 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow in visitSUBO()
4244 SDValue N0 = N->getOperand(0); in visitSUBE()
4245 SDValue N1 = N->getOperand(1); in visitSUBE()
4246 SDValue CarryIn = N->getOperand(2); in visitSUBE()
4248 // fold (sube x, y, false) -> (subc x, y) in visitSUBE()
4250 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); in visitSUBE()
4256 SDValue N0 = N->getOperand(0); in visitUSUBO_CARRY()
4257 SDValue N1 = N->getOperand(1); in visitUSUBO_CARRY()
4258 SDValue CarryIn = N->getOperand(2); in visitUSUBO_CARRY()
4260 // fold (usubo_carry x, y, false) -> (usubo x, y) in visitUSUBO_CARRY()
4263 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0))) in visitUSUBO_CARRY()
4264 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1); in visitUSUBO_CARRY()
4271 SDValue N0 = N->getOperand(0); in visitSSUBO_CARRY()
4272 SDValue N1 = N->getOperand(1); in visitSSUBO_CARRY()
4273 SDValue CarryIn = N->getOperand(2); in visitSSUBO_CARRY()
4275 // fold (ssubo_carry x, y, false) -> (ssubo x, y) in visitSSUBO_CARRY()
4278 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0))) in visitSSUBO_CARRY()
4279 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1); in visitSSUBO_CARRY()
4288 SDValue N0 = N->getOperand(0); in visitMULFIX()
4289 SDValue N1 = N->getOperand(1); in visitMULFIX()
4290 SDValue Scale = N->getOperand(2); in visitMULFIX() local
4293 // fold (mulfix x, undef, scale) -> 0 in visitMULFIX()
4300 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale); in visitMULFIX()
4302 // fold (mulfix x, 0, scale) -> 0 in visitMULFIX()
4310 SDValue N0 = N->getOperand(0); in visitMUL()
4311 SDValue N1 = N->getOperand(1); in visitMUL()
4318 // fold (mul x, undef) -> 0 in visitMUL()
4322 // fold (mul c1, c2) -> c1*c2 in visitMUL()
4348 ConstValue1 = N1->getAsAPIntVal(); in visitMUL()
4349 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque(); in visitMUL()
4353 // fold (mul x, 0) -> 0 in visitMUL()
4357 // fold (mul x, 1) -> x in visitMUL()
4365 // fold (mul x, -1) -> 0-x in visitMUL()
4369 // fold (mul x, (1 << c)) -> x << c in visitMUL()
4379 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c in visitMUL()
4381 unsigned Log2Val = (-ConstValue1).logBase2(); in visitMUL()
4384 // single-use add), we should put the negate there. in visitMUL()
4392 // hi result is in use in case we hit this mid-legalization. in visitMUL()
4399 if (LoHi->hasAnyUseOfValue(1)) in visitMUL()
4402 if (LoHi->hasAnyUseOfValue(1)) in visitMUL()
4409 // (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub. in visitMUL()
4410 // mul x, (2^N + 1) --> add (shl x, N), x in visitMUL()
4411 // mul x, (2^N - 1) --> sub (shl x, N), x in visitMUL()
4412 // Examples: x * 33 --> (x << 5) + x in visitMUL()
4413 // x * 15 --> (x << 4) - x in visitMUL()
4414 // x * -33 --> -((x << 5) + x) in visitMUL()
4415 // x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4) in visitMUL()
4416 // (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub. in visitMUL()
4417 // mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M)) in visitMUL()
4418 // mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M)) in visitMUL()
4419 // Examples: x * 0x8800 --> (x << 15) + (x << 11) in visitMUL()
4420 // x * 0xf800 --> (x << 16) - (x << 11) in visitMUL()
4421 // x * -0x8800 --> -((x << 15) + (x << 11)) in visitMUL()
4422 // x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16) in visitMUL()
4433 if ((MulC - 1).isPowerOf2()) in visitMUL()
4440 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2(); in visitMUL()
4443 "multiply-by-constant generated out of bounds shift"); in visitMUL()
4457 // (mul (shl X, c1), c2) -> (mul X, c2 << c1) in visitMUL()
4464 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one in visitMUL()
4484 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2) in visitMUL()
4498 const APInt &C1 = NC1->getAPIntValue(); in visitMUL()
4511 // Fold Y = sra (X, size(X)-1); mul (or (Y, 1), X) -> (abs X) in visitMUL()
4516 m_Mul(m_Or(m_Sra(m_Value(X), m_SpecificInt(BitWidth - 1)), m_One()), in visitMUL()
4521 // Fold ((mul x, 0/undef) -> 0, in visitMUL()
4522 // (mul x, 1) -> x) -> x) in visitMUL()
4523 // -> and(x, mask) in visitMUL()
4530 if (!V || V->isZero()) { in visitMUL()
4535 return V->isOne(); in visitMUL()
4554 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags())) in visitMUL()
4557 // Fold mul(vecreduce(x), vecreduce(y)) -> vecreduce(mul(x, y)) in visitMUL()
4564 // Simplify the operands using demanded-bits information. in visitMUL()
4575 EVT NodeType = Node->getValueType(0); in isDivRemLibcallAvailable()
4592 if (Node->use_empty()) in useDivRem()
4595 unsigned Opcode = Node->getOpcode(); in useDivRem()
4599 // DivMod lib calls can still work on non-legal types if using lib-calls. in useDivRem()
4600 EVT VT = Node->getValueType(0); in useDivRem()
4625 SDValue Op0 = Node->getOperand(0); in useDivRem()
4626 SDValue Op1 = Node->getOperand(1); in useDivRem()
4628 for (SDNode *User : Op0->uses()) { in useDivRem()
4629 if (User == Node || User->getOpcode() == ISD::DELETED_NODE || in useDivRem()
4630 User->use_empty()) in useDivRem()
4633 // otherwise, the DIVREM may get target-legalized into something in useDivRem()
4634 // target-specific that we won't be able to recognize. in useDivRem()
4635 unsigned UserOpc = User->getOpcode(); in useDivRem()
4637 User->getOperand(0) == Op0 && in useDivRem()
4638 User->getOperand(1) == Op1) { in useDivRem()
4660 SDValue N0 = N->getOperand(0); in simplifyDivRem()
4661 SDValue N1 = N->getOperand(1); in simplifyDivRem()
4662 EVT VT = N->getValueType(0); in simplifyDivRem()
4665 unsigned Opc = N->getOpcode(); in simplifyDivRem()
4669 // X / undef -> undef in simplifyDivRem()
4670 // X % undef -> undef in simplifyDivRem()
4671 // X / 0 -> undef in simplifyDivRem()
4672 // X % 0 -> undef in simplifyDivRem()
4677 // undef / X -> 0 in simplifyDivRem()
4678 // undef % X -> 0 in simplifyDivRem()
4682 // 0 / X -> 0 in simplifyDivRem()
4683 // 0 % X -> 0 in simplifyDivRem()
4685 if (N0C && N0C->isZero()) in simplifyDivRem()
4688 // X / X -> 1 in simplifyDivRem()
4689 // X % X -> 0 in simplifyDivRem()
4693 // X / 1 -> X in simplifyDivRem()
4694 // X % 1 -> 0 in simplifyDivRem()
4695 // If this is a boolean op (single-bit element type), we can't have in simplifyDivRem()
4696 // division-by-zero or remainder-by-zero, so assume the divisor is 1. in simplifyDivRem()
4697 // TODO: Similarly, if we're zero-extending a boolean divisor, then assume in simplifyDivRem()
4699 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1)) in simplifyDivRem()
4706 SDValue N0 = N->getOperand(0); in visitSDIV()
4707 SDValue N1 = N->getOperand(1); in visitSDIV()
4708 EVT VT = N->getValueType(0); in visitSDIV()
4712 // fold (sdiv c1, c2) -> c1/c2 in visitSDIV()
4721 // fold (sdiv X, -1) -> 0-X in visitSDIV()
4723 if (N1C && N1C->isAllOnes()) in visitSDIV()
4726 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0) in visitSDIV()
4727 if (N1C && N1C->isMinSignedValue()) in visitSDIV()
4739 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2 in visitSDIV()
4745 // (Dividend - (Quotient * Divisor). in visitSDIV()
4746 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(), in visitSDIV()
4757 // sdiv, srem -> sdivrem in visitSDIV()
4761 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr)) in visitSDIV()
4769 // Helper for determining whether a value is a power-2 constant scalar or a in isDivisorPowerOfTwo()
4772 if (C->isZero() || C->isOpaque()) in isDivisorPowerOfTwo()
4774 if (C->getAPIntValue().isPowerOf2()) in isDivisorPowerOfTwo()
4776 if (C->getAPIntValue().isNegatedPowerOf2()) in isDivisorPowerOfTwo()
4786 EVT VT = N->getValueType(0); in visitSDIVLike()
4790 // fold (sdiv X, pow2) -> simple ops after legalize in visitSDIVLike()
4792 // better results in that case. The target-specific lowering should learn how in visitSDIVLike()
4794 if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(N1)) { in visitSDIVLike()
4795 // Target-specific implementation of sdiv x, pow2. in visitSDIVLike()
4810 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy)); in visitSDIVLike()
4813 // Add (N0 < 0) ? abs2 - 1 : 0; in visitSDIVLike()
4821 // Special case: (sdiv X, 1) -> X in visitSDIVLike()
4822 // Special Case: (sdiv X, -1) -> 0-X in visitSDIVLike()
4835 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding. in visitSDIVLike()
4843 // trade-offs. in visitSDIVLike()
4846 !TLI.isIntDivCheap(N->getValueType(0), Attr)) in visitSDIVLike()
4854 SDValue N0 = N->getOperand(0); in visitUDIV()
4855 SDValue N1 = N->getOperand(1); in visitUDIV()
4856 EVT VT = N->getValueType(0); in visitUDIV()
4860 // fold (udiv c1, c2) -> c1/c2 in visitUDIV()
4869 // fold (udiv X, -1) -> select(X == -1, 1, 0) in visitUDIV()
4871 if (N1C && N1C->isAllOnes() && CCVT.isVector() == VT.isVector()) { in visitUDIV()
4885 // (Dividend - (Quotient * Divisor). in visitUDIV()
4886 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(), in visitUDIV()
4897 // sdiv, srem -> sdivrem in visitUDIV()
4901 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr)) in visitUDIV()
4910 EVT VT = N->getValueType(0); in visitUDIVLike()
4912 // fold (udiv x, (1 << c)) -> x >>u c in visitUDIVLike()
4924 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2 in visitUDIVLike()
4941 // fold (udiv x, c) -> alternate in visitUDIVLike()
4944 !TLI.isIntDivCheap(N->getValueType(0), Attr)) in visitUDIVLike()
4952 if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(N1) && in buildOptimizedSREM()
4953 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) { in buildOptimizedSREM()
4954 // Target-specific implementation of srem x, pow2. in buildOptimizedSREM()
4963 unsigned Opcode = N->getOpcode(); in visitREM()
4964 SDValue N0 = N->getOperand(0); in visitREM()
4965 SDValue N1 = N->getOperand(1); in visitREM()
4966 EVT VT = N->getValueType(0); in visitREM()
4972 // fold (rem c1, c2) -> c1%c2 in visitREM()
4976 // fold (urem X, -1) -> select(FX == -1, 0, FX) in visitREM()
4993 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15 in visitREM()
4998 // fold (urem x, pow2) -> (and x, pow2-1) in visitREM()
5004 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1)) in visitREM()
5005 // fold (urem x, (lshr pow2, y)) -> (and x, (add (lshr pow2, y), -1)) in visitREM()
5019 // If X/C can be simplified by the division-by-constant logic, lower in visitREM()
5020 // X%C to the equivalent of X-X/C*C. in visitREM()
5021 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the in visitREM()
5038 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(), in visitREM()
5049 // sdiv, srem -> sdivrem in visitREM()
5057 SDValue N0 = N->getOperand(0); in visitMULHS()
5058 SDValue N1 = N->getOperand(1); in visitMULHS()
5059 EVT VT = N->getValueType(0); in visitMULHS()
5069 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0); in visitMULHS()
5075 // fold (mulhs x, 0) -> 0 in visitMULHS()
5081 // fold (mulhs x, 0) -> 0 in visitMULHS()
5085 // fold (mulhs x, 1) -> (sra x, size(x)-1) in visitMULHS()
5089 DAG.getShiftAmountConstant(N0.getScalarValueSizeInBits() - 1, VT, DL)); in visitMULHS()
5091 // fold (mulhs x, undef) -> 0 in visitMULHS()
5116 SDValue N0 = N->getOperand(0); in visitMULHU()
5117 SDValue N1 = N->getOperand(1); in visitMULHU()
5118 EVT VT = N->getValueType(0); in visitMULHU()
5128 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0); in visitMULHU()
5134 // fold (mulhu x, 0) -> 0 in visitMULHU()
5140 // fold (mulhu x, 0) -> 0 in visitMULHU()
5144 // fold (mulhu x, 1) -> 0 in visitMULHU()
5148 // fold (mulhu x, undef) -> 0 in visitMULHU()
5152 // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c) in visitMULHU()
5182 // Simplify the operands using demanded-bits information. in visitMULHU()
5192 unsigned Opcode = N->getOpcode(); in visitAVG()
5193 SDValue N0 = N->getOperand(0); in visitAVG()
5194 SDValue N1 = N->getOperand(1); in visitAVG()
5195 EVT VT = N->getValueType(0); in visitAVG()
5206 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0); in visitAVG()
5212 // fold (avg x, undef) -> x in visitAVG()
5218 // fold (avg x, x) --> x in visitAVG()
5222 // fold (avgfloor x, 0) -> x >> 1 in visitAVG()
5231 // fold avgu(zext(x), zext(y)) -> zext(avgu(x, y)) in visitAVG()
5232 // fold avgs(sext(x), sext(y)) -> sext(avgs(x, y)) in visitAVG()
5248 // Fold avgflooru(x,y) -> avgceilu(x,y-1) iff y != 0 in visitAVG()
5249 // Fold avgflooru(x,y) -> avgceilu(x-1,y) iff x != 0 in visitAVG()
5267 unsigned Opcode = N->getOpcode(); in visitABD()
5268 SDValue N0 = N->getOperand(0); in visitABD()
5269 SDValue N1 = N->getOperand(1); in visitABD()
5270 EVT VT = N->getValueType(0); in visitABD()
5280 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0); in visitABD()
5286 // fold (abd x, undef) -> 0 in visitABD()
5292 // fold (abds x, 0) -> abs x in visitABD()
5297 // fold (abdu x, 0) -> x in visitABD()
5301 // fold (abds x, y) -> (abdu x, y) iff both args are known positive in visitABD()
5315 bool HiExists = N->hasAnyUseOfValue(1); in SimplifyNodeWithTwoResults()
5317 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) { in SimplifyNodeWithTwoResults()
5318 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops()); in SimplifyNodeWithTwoResults()
5323 bool LoExists = N->hasAnyUseOfValue(0); in SimplifyNodeWithTwoResults()
5325 TLI.isOperationLegalOrCustom(HiOp, N->getValueType(1)))) { in SimplifyNodeWithTwoResults()
5326 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops()); in SimplifyNodeWithTwoResults()
5336 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops()); in SimplifyNodeWithTwoResults()
5346 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops()); in SimplifyNodeWithTwoResults()
5362 SDValue N0 = N->getOperand(0); in visitSMUL_LOHI()
5363 SDValue N1 = N->getOperand(1); in visitSMUL_LOHI()
5364 EVT VT = N->getValueType(0); in visitSMUL_LOHI()
5369 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N0, N1); in visitSMUL_LOHI()
5374 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0); in visitSMUL_LOHI()
5403 SDValue N0 = N->getOperand(0); in visitUMUL_LOHI()
5404 SDValue N1 = N->getOperand(1); in visitUMUL_LOHI()
5405 EVT VT = N->getValueType(0); in visitUMUL_LOHI()
5410 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N0, N1); in visitUMUL_LOHI()
5415 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0); in visitUMUL_LOHI()
5417 // (umul_lohi N0, 0) -> (0, 0) in visitUMUL_LOHI()
5423 // (umul_lohi N0, 1) -> (N0, 0) in visitUMUL_LOHI()
5453 SDValue N0 = N->getOperand(0); in visitMULO()
5454 SDValue N1 = N->getOperand(1); in visitMULO()
5456 bool IsSigned = (ISD::SMULO == N->getOpcode()); in visitMULO()
5458 EVT CarryVT = N->getValueType(1); in visitMULO()
5470 IsSigned ? N0C->getAPIntValue().smul_ov(N1C->getAPIntValue(), Overflow) in visitMULO()
5471 : N0C->getAPIntValue().umul_ov(N1C->getAPIntValue(), Overflow); in visitMULO()
5479 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0); in visitMULO()
5481 // fold (mulo x, 0) -> 0 + no carry out in visitMULO()
5486 // (mulo x, 2) -> (addo x, x) in visitMULO()
5488 if (N1C && N1C->getAPIntValue() == 2 && in visitMULO()
5491 N->getVTList(), N0, N0); in visitMULO()
5510 // saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
5528 const APInt &C1 = N1C->getAPIntValue().trunc(N1.getScalarValueSizeInBits()); in isSaturatingMinMax()
5529 const APInt &C2 = N3C->getAPIntValue().trunc(N3.getScalarValueSizeInBits()); in isSaturatingMinMax()
5548 const fltSemantics &Semantics = InputTy->getFltSemantics(); in isSaturatingMinMax()
5574 N0CC = cast<CondCodeSDNode>(N0.getOperand(4))->get(); in isSaturatingMinMax()
5584 N0CC = cast<CondCodeSDNode>(N0.getOperand(0).getOperand(2))->get(); in isSaturatingMinMax()
5596 if (!MinCOp || !MaxCOp || MinCOp->getValueType(0) != MaxCOp->getValueType(0)) in isSaturatingMinMax()
5599 const APInt &MinC = MinCOp->getAPIntValue(); in isSaturatingMinMax()
5600 const APInt &MaxC = MaxCOp->getAPIntValue(); in isSaturatingMinMax()
5602 if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) { in isSaturatingMinMax()
5636 return DAG.getExtOrTrunc(!Unsigned, Sat, DL, N2->getValueType(0)); in PerformMinMaxFpToSatCombine()
5642 // We are looking for UMIN(FPTOUI(X), (2^n)-1), which may have come via a in PerformUMinFpToSatCombine()
5653 const APInt &C1 = N1C->getAPIntValue(); in PerformUMinFpToSatCombine()
5654 const APInt &C3 = N3C->getAPIntValue(); in PerformUMinFpToSatCombine()
5676 SDValue N0 = N->getOperand(0); in visitIMINMAX()
5677 SDValue N1 = N->getOperand(1); in visitIMINMAX()
5679 unsigned Opcode = N->getOpcode(); in visitIMINMAX()
5686 // If the operands are the same, this is a no-op. in visitIMINMAX()
5701 if (SDValue RMINMAX = reassociateOps(Opcode, DL, N0, N1, N->getFlags())) in visitIMINMAX()
5732 // Fold min/max(vecreduce(x), vecreduce(y)) -> vecreduce(min/max(x, y)) in visitIMINMAX()
5751 // Simplify the operands using demanded-bits information. in visitIMINMAX()
5761 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in hoistLogicOpWithSameOpcodeHands()
5763 unsigned LogicOpcode = N->getOpcode(); in hoistLogicOpWithSameOpcodeHands()
5775 // Handle size-changing casts (or sign_extend_inreg). in hoistLogicOpWithSameOpcodeHands()
5801 // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y) in hoistLogicOpWithSameOpcodeHands()
5808 // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y) in hoistLogicOpWithSameOpcodeHands()
5831 // logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z in hoistLogicOpWithSameOpcodeHands()
5842 // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y) in hoistLogicOpWithSameOpcodeHands()
5852 // logic_op (OP x, x1, s), (OP y, y1, s) --> in hoistLogicOpWithSameOpcodeHands()
5853 // --> OP (logic_op x, y), (logic_op, x1, y1), s in hoistLogicOpWithSameOpcodeHands()
5866 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B)) in hoistLogicOpWithSameOpcodeHands()
5885 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B)) in hoistLogicOpWithSameOpcodeHands()
5888 // The type-legalizer generates this pattern when loading illegal in hoistLogicOpWithSameOpcodeHands()
5906 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() || in hoistLogicOpWithSameOpcodeHands()
5907 !SVN0->getMask().equals(SVN1->getMask())) in hoistLogicOpWithSameOpcodeHands()
5916 // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C) in hoistLogicOpWithSameOpcodeHands()
5920 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
5929 // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B)) in hoistLogicOpWithSameOpcodeHands()
5933 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
5954 // If we're here post-legalization or the logic op type is not i1, the logic in foldLogicOfSetCCs()
5965 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get(); in foldLogicOfSetCCs()
5966 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get(); in foldLogicOfSetCCs()
5981 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0) in foldLogicOfSetCCs()
5982 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1) in foldLogicOfSetCCs()
5983 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0) in foldLogicOfSetCCs()
5984 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0) in foldLogicOfSetCCs()
6000 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1) in foldLogicOfSetCCs()
6001 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0) in foldLogicOfSetCCs()
6002 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1) in foldLogicOfSetCCs()
6003 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1) in foldLogicOfSetCCs()
6012 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2) in foldLogicOfSetCCs()
6028 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0 in foldLogicOfSetCCs()
6029 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0 in foldLogicOfSetCCs()
6040 // Match a shared variable operand and 2 non-opaque constant operands. in foldLogicOfSetCCs()
6044 APIntOps::umax(C0->getAPIntValue(), C1->getAPIntValue()); in foldLogicOfSetCCs()
6046 APIntOps::umin(C0->getAPIntValue(), C1->getAPIntValue()); in foldLogicOfSetCCs()
6047 return !C0->isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2(); in foldLogicOfSetCCs()
6050 // and/or (setcc X, CMax, ne), (setcc X, CMin, ne/eq) --> in foldLogicOfSetCCs()
6051 // setcc ((sub X, CMin), ~(CMax - CMin)), 0, ne/eq in foldLogicOfSetCCs()
6070 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC) in foldLogicOfSetCCs()
6071 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC) in foldLogicOfSetCCs()
6154 (LogicOp->getOpcode() == ISD::AND || LogicOp->getOpcode() == ISD::OR) && in foldAndOrOfSETCC()
6158 SDValue LHS = LogicOp->getOperand(0); in foldAndOrOfSETCC()
6159 SDValue RHS = LogicOp->getOperand(1); in foldAndOrOfSETCC()
6160 if (LHS->getOpcode() != ISD::SETCC || RHS->getOpcode() != ISD::SETCC || in foldAndOrOfSETCC()
6161 !LHS->hasOneUse() || !RHS->hasOneUse()) in foldAndOrOfSETCC()
6168 SDValue LHS0 = LHS->getOperand(0); in foldAndOrOfSETCC()
6169 SDValue RHS0 = RHS->getOperand(0); in foldAndOrOfSETCC()
6170 SDValue LHS1 = LHS->getOperand(1); in foldAndOrOfSETCC()
6171 SDValue RHS1 = RHS->getOperand(1); in foldAndOrOfSETCC()
6176 ISD::CondCode CCL = cast<CondCodeSDNode>(LHS.getOperand(2))->get(); in foldAndOrOfSETCC()
6177 ISD::CondCode CCR = cast<CondCodeSDNode>(RHS.getOperand(2))->get(); in foldAndOrOfSETCC()
6178 EVT VT = LogicOp->getValueType(0); in foldAndOrOfSETCC()
6183 // compare against the same value. Replace the and/or-cmp-cmp sequence with in foldAndOrOfSETCC()
6184 // min/max cmp sequence. If LHS1 is equal to RHS1, then the or-cmp-cmp in foldAndOrOfSETCC()
6185 // sequence will be replaced with min-cmp sequence: in foldAndOrOfSETCC()
6186 // (LHS0 < LHS1) | (RHS0 < RHS1) -> min(LHS0, RHS0) < LHS1 in foldAndOrOfSETCC()
6187 // and and-cmp-cmp will be replaced with max-cmp sequence: in foldAndOrOfSETCC()
6188 // (LHS0 < LHS1) & (RHS0 < RHS1) -> max(LHS0, RHS0) < LHS1 in foldAndOrOfSETCC()
6249 bool IsOr = (LogicOp->getOpcode() == ISD::OR); in foldAndOrOfSETCC()
6256 getMinMaxOpcodeForFP(Operand1, Operand2, CC, LogicOp->getOpcode(), in foldAndOrOfSETCC()
6271 CCL == (LogicOp->getOpcode() == ISD::AND ? ISD::SETNE : ISD::SETEQ) && in foldAndOrOfSETCC()
6273 const APInt &APLhs = LHS1C->getAPIntValue(); in foldAndOrOfSETCC()
6274 const APInt &APRhs = RHS1C->getAPIntValue(); in foldAndOrOfSETCC()
6278 if (APLhs == (-APRhs) && in foldAndOrOfSETCC()
6282 // (icmp eq A, C) | (icmp eq A, -C) in foldAndOrOfSETCC()
6283 // -> (icmp eq Abs(A), C) in foldAndOrOfSETCC()
6284 // (icmp ne A, C) & (icmp ne A, -C) in foldAndOrOfSETCC()
6285 // -> (icmp ne Abs(A), C) in foldAndOrOfSETCC()
6294 // IF IsPow2(smax(C0, C1)-smin(C0, C1)) in foldAndOrOfSETCC()
6295 // -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) == 0 in foldAndOrOfSETCC()
6297 // IF IsPow2(smax(C0, C1)-smin(C0, C1)) in foldAndOrOfSETCC()
6298 // -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) != 0 in foldAndOrOfSETCC()
6302 // IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1)) in foldAndOrOfSETCC()
6303 // -> ~A & smin(C0, C1) == 0 in foldAndOrOfSETCC()
6305 // IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1)) in foldAndOrOfSETCC()
6306 // -> ~A & smin(C0, C1) != 0 in foldAndOrOfSETCC()
6310 APInt Dif = MaxC - MinC; in foldAndOrOfSETCC()
6322 DAG.getConstant(-MinC, DL, OpVT)); in foldAndOrOfSETCC()
6335 // Combine `(select c, (X & 1), 0)` -> `(and (zext c), X)`.
6370 // fold (and x, undef) -> 0 in visitANDLike()
6378 // and(x, add) -> and(add, x) in visitANDLike()
6384 VT.isScalarInteger() && VT.getSizeInBits() <= 64 && N0->hasOneUse()) { in visitANDLike()
6391 APInt ADDC = ADDI->getAPIntValue(); in visitANDLike()
6392 APInt SRLC = SRLI->getAPIntValue(); in visitANDLike()
6419 if (!AndC->getAPIntValue().isMask()) in isAndLoadExtLoad()
6422 unsigned ActiveBits = AndC->getAPIntValue().countr_one(); in isAndLoadExtLoad()
6425 EVT LoadedVT = LoadN->getMemoryVT(); in isAndLoadExtLoad()
6436 if (!LoadN->isSimple()) in isAndLoadExtLoad()
6439 // Do not generate loads of non-round integer types since these can in isAndLoadExtLoad()
6463 // Do not generate loads of non-round integer types since these can in isLegalNarrowLdSt()
6469 if (!LDST->isSimple()) in isLegalNarrowLdSt()
6472 EVT LdStMemVT = LDST->getMemoryVT(); in isLegalNarrowLdSt()
6487 const Align LDSTAlign = LDST->getAlign(); in isLegalNarrowLdSt()
6490 LDST->getAddressSpace(), NarrowAlign, in isLegalNarrowLdSt()
6491 LDST->getMemOperand()->getFlags())) in isLegalNarrowLdSt()
6496 EVT PtrType = LDST->getBasePtr().getValueType(); in isLegalNarrowLdSt()
6508 !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT)) in isLegalNarrowLdSt()
6512 // (the value loaded and the chain). Don't transform a pre-increment in isLegalNarrowLdSt()
6516 if (Load->getNumValues() > 2) in isLegalNarrowLdSt()
6522 if (Load->getExtensionType() != ISD::NON_EXTLOAD && in isLegalNarrowLdSt()
6523 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt) in isLegalNarrowLdSt()
6532 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt) in isLegalNarrowLdSt()
6536 !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT)) in isLegalNarrowLdSt()
6549 for (SDValue Op : N->op_values()) { in SearchForAndLoads()
6555 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) && in SearchForAndLoads()
6556 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue()) in SearchForAndLoads()
6568 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) && in SearchForAndLoads()
6572 if (Load->getExtensionType() == ISD::ZEXTLOAD && in SearchForAndLoads()
6573 ExtVT.bitsGE(Load->getMemoryVT())) in SearchForAndLoads()
6577 if (ExtVT.bitsLE(Load->getMemoryVT())) in SearchForAndLoads()
6586 unsigned ActiveBits = Mask->getAPIntValue().countr_one(); in SearchForAndLoads()
6589 cast<VTSDNode>(Op.getOperand(1))->getVT() : in SearchForAndLoads()
6613 if (NodeToMask->getNumValues() > 1) { in SearchForAndLoads()
6615 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) { in SearchForAndLoads()
6632 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); in BackwardsPropagateMask()
6636 if (!Mask->getAPIntValue().isMask()) in BackwardsPropagateMask()
6640 if (isa<LoadSDNode>(N->getOperand(0))) in BackwardsPropagateMask()
6650 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump()); in BackwardsPropagateMask()
6651 SDValue MaskOp = N->getOperand(1); in BackwardsPropagateMask()
6656 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump()); in BackwardsPropagateMask()
6658 FixupNode->getValueType(0), in BackwardsPropagateMask()
6667 SDValue Op0 = LogicN->getOperand(0); in BackwardsPropagateMask()
6668 SDValue Op1 = LogicN->getOperand(1); in BackwardsPropagateMask()
6686 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump()); in BackwardsPropagateMask()
6687 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0), in BackwardsPropagateMask()
6698 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode()); in BackwardsPropagateMask()
6705 // x & (-1 'logical shift' y)
6710 assert(N->getOpcode() == ISD::AND); in unfoldExtremeBitClearingToShifts()
6712 SDValue N0 = N->getOperand(0); in unfoldExtremeBitClearingToShifts()
6713 SDValue N1 = N->getOperand(1); in unfoldExtremeBitClearingToShifts()
6719 // Try to match (-1 '[outer] logical shift' y) in unfoldExtremeBitClearingToShifts()
6723 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool { in unfoldExtremeBitClearingToShifts()
6726 OuterShift = M->getOpcode(); in unfoldExtremeBitClearingToShifts()
6733 if (!isAllOnesConstant(M->getOperand(0))) in unfoldExtremeBitClearingToShifts()
6735 Y = M->getOperand(1); in unfoldExtremeBitClearingToShifts()
6748 EVT VT = N->getValueType(0); in unfoldExtremeBitClearingToShifts()
6762 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op"); in combineShiftAnd1ToBitTest()
6765 SDValue And0 = And->getOperand(0), And1 = And->getOperand(1); in combineShiftAnd1ToBitTest()
6787 // Match a shift-right by constant. in combineShiftAnd1ToBitTest()
6801 if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(BitWidth)) in combineShiftAnd1ToBitTest()
6818 // Turn this into a bit-test pattern using mask op + setcc: in combineShiftAnd1ToBitTest()
6819 // and (not (srl X, C)), 1 --> (and X, 1<<C) == 0 in combineShiftAnd1ToBitTest()
6820 // and (srl (not X), C)), 1 --> (and X, 1<<C) == 0 in combineShiftAnd1ToBitTest()
6826 APInt::getOneBitSet(BitWidth, ShiftAmtC->getZExtValue()), DL, SrcVT); in combineShiftAnd1ToBitTest()
6830 return DAG.getZExtOrTrunc(Setcc, DL, And->getValueType(0)); in combineShiftAnd1ToBitTest()
6833 /// For targets that support usubsat, match a bit-hack form of that operation
6836 EVT VT = N->getValueType(0); in foldAndToUsubsat()
6840 // (i8 X ^ 128) & (i8 X s>> 7) --> usubsat X, 128 in foldAndToUsubsat()
6841 // (i8 X + 128) & (i8 X s>> 7) --> usubsat X, 128 in foldAndToUsubsat()
6846 m_SpecificInt(BitWidth - 1))))) && in foldAndToUsubsat()
6849 m_SpecificInt(BitWidth - 1)))))) in foldAndToUsubsat()
6859 /// ((X0 << Y) | Z) | (X1 << Y) --> ((X0 | X1) << Y) | Z
6862 unsigned LogicOpcode = N->getOpcode(); in foldLogicOfShifts()
6878 // LOGIC (LOGIC (SH X0, Y), Z), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z in foldLogicOfShifts()
6879 // LOGIC (LOGIC Z, (SH X0, Y)), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z in foldLogicOfShifts()
6895 EVT VT = N->getValueType(0); in foldLogicOfShifts()
6906 /// LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W) -->
6907 /// --> LOGIC (SH (LOGIC X0, X1), Y), (LOGIC Z, W)
6910 unsigned LogicOpcode = N->getOpcode(); in foldLogicTreeOfShifts()
6934 EVT VT = N->getValueType(0); in foldLogicTreeOfShifts()
6940 SDValue N0 = N->getOperand(0); in visitAND()
6941 SDValue N1 = N->getOperand(1); in visitAND()
6945 // x & x --> x in visitAND()
6949 // fold (and c1, c2) -> c1&c2 in visitAND()
6966 // fold (and x, 0) -> 0, vector edition in visitAND()
6972 // fold (and x, -1) -> x, vector edition in visitAND()
6979 if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat && in visitAND()
6981 EVT LoadVT = MLoad->getMemoryVT(); in visitAND()
6989 if (Splat->getAPIntValue().isMask(ElementSize)) { in visitAND()
6991 ExtVT, DL, MLoad->getChain(), MLoad->getBasePtr(), in visitAND()
6992 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(), in visitAND()
6993 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(), in visitAND()
6994 ISD::ZEXTLOAD, MLoad->isExpandingLoad()); in visitAND()
7005 // fold (and x, -1) -> x in visitAND()
7022 if (SDValue RAND = reassociateOps(ISD::AND, DL, N0, N1, N->getFlags())) in visitAND()
7025 // Fold and(vecreduce(x), vecreduce(y)) -> vecreduce(and(x, y)) in visitAND()
7030 // fold (and (or x, C), D) -> D if (C & D) == D in visitAND()
7032 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue()); in visitAND()
7042 APInt Mask = ~N1C->getAPIntValue(); in visitAND()
7045 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits. in visitAND()
7049 // fold (and (any_ext V), c) -> (zero_ext (and (trunc V), c)) if profitable. in visitAND()
7050 if (N1C->getAPIntValue().countLeadingZeros() >= (BitWidth - SrcBitWidth) && in visitAND()
7059 // fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2))) in visitAND()
7067 N0->hasOneUse() && N0Op0->hasOneUse()) { in visitAND()
7077 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) -> in visitAND()
7097 Constant = C->getAPIntValue(); in visitAND()
7099 unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits(); in visitAND()
7110 Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, in visitAND()
7134 Load->getValueType(0), in visitAND()
7135 Load->getMemoryVT()); in visitAND()
7140 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits()); in visitAND()
7143 switch (Load->getExtensionType()) { in visitAND()
7158 if (Load->getExtensionType() == ISD::EXTLOAD) { in visitAND()
7159 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD, in visitAND()
7160 Load->getValueType(0), SDLoc(Load), in visitAND()
7161 Load->getChain(), Load->getBasePtr(), in visitAND()
7162 Load->getOffset(), Load->getMemoryVT(), in visitAND()
7163 Load->getMemOperand()); in visitAND()
7165 if (Load->getNumValues() == 3) { in visitAND()
7190 EVT ExtVT = Ext->getValueType(0); in visitAND()
7191 SDValue Extendee = Ext->getOperand(0); in visitAND()
7194 if (N1C->getAPIntValue().isMask(ScalarWidth) && in visitAND()
7206 // fold (and (masked_gather x)) -> (zext_masked_gather x) in visitAND()
7208 EVT MemVT = GN0->getMemoryVT(); in visitAND()
7214 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(), in visitAND()
7215 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()}; in visitAND()
7218 DAG.getVTList(VT, MVT::Other), MemVT, DL, Ops, GN0->getMemOperand(), in visitAND()
7219 GN0->getIndexType(), ISD::ZEXTLOAD); in visitAND()
7228 // fold (and (load x), 255) -> (zextload x, i8) in visitAND()
7229 // fold (and (extload x, i16), 255) -> (zextload x, i8) in visitAND()
7246 // Simplify: (and (op x...), (op y...)) -> (op (and x, y)) in visitAND()
7256 // Masking the negated extension of a boolean is just the zero-extended in visitAND()
7258 // and (sub 0, zext(bool X)), 1 --> zext(bool X) in visitAND()
7259 // and (sub 0, sext(bool X)), 1 --> zext(bool X) in visitAND()
7261 // Note: the SimplifyDemandedBits fold below can make an information-losing in visitAND()
7263 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) { in visitAND()
7275 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) in visitAND()
7276 // fold (and (sra)) -> (and (srl)) when possible. in visitAND()
7280 // fold (zext_inreg (extload x)) -> (zextload x) in visitAND()
7281 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use in visitAND()
7286 EVT MemVT = LN0->getMemoryVT(); in visitAND()
7291 APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize); in visitAND()
7293 ((!LegalOperations && LN0->isSimple()) || in visitAND()
7296 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
7297 LN0->getBasePtr(), MemVT, LN0->getMemOperand()); in visitAND()
7304 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const) in visitAND()
7305 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { in visitAND()
7324 if (LHS->getOpcode() != ISD::SIGN_EXTEND) in visitAND()
7331 if (!C->getAPIntValue().isMask( in visitAND()
7361 EVT VT = N->getValueType(0); in MatchBSwapHWordLow()
7375 if (!N0->hasOneUse()) in MatchBSwapHWordLow()
7380 if (!N01C || (N01C->getZExtValue() != 0xFF00 && in MatchBSwapHWordLow()
7381 N01C->getZExtValue() != 0xFFFF)) in MatchBSwapHWordLow()
7388 if (!N1->hasOneUse()) in MatchBSwapHWordLow()
7391 if (!N11C || N11C->getZExtValue() != 0xFF) in MatchBSwapHWordLow()
7401 if (!N0->hasOneUse() || !N1->hasOneUse()) in MatchBSwapHWordLow()
7408 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8) in MatchBSwapHWordLow()
7412 SDValue N00 = N0->getOperand(0); in MatchBSwapHWordLow()
7414 if (!N00->hasOneUse()) in MatchBSwapHWordLow()
7417 if (!N001C || N001C->getZExtValue() != 0xFF) in MatchBSwapHWordLow()
7423 SDValue N10 = N1->getOperand(0); in MatchBSwapHWordLow()
7425 if (!N10->hasOneUse()) in MatchBSwapHWordLow()
7430 if (!N101C || (N101C->getZExtValue() != 0xFF00 && in MatchBSwapHWordLow()
7431 N101C->getZExtValue() != 0xFFFF)) in MatchBSwapHWordLow()
7444 // If the left-shift isn't masked out then the only way this is a bswap is in MatchBSwapHWordLow()
7466 DAG.getShiftAmountConstant(OpSizeInBits - 16, VT, DL)); in MatchBSwapHWordLow()
7471 /// Return true if the specified node is an element that makes up a 32-bit
7478 if (!N->hasOneUse()) in isBSwapHWordElement()
7500 switch (N1C->getZExtValue()) { in isBSwapHWordElement()
7525 if (!C || C->getZExtValue() != 8) in isBSwapHWordElement()
7533 if (!C || C->getZExtValue() != 8) in isBSwapHWordElement()
7542 if (!C || C->getZExtValue() != 8) in isBSwapHWordElement()
7550 if (!C || C->getZExtValue() != 8) in isBSwapHWordElement()
7569 if (!C || C->getAPIntValue() != 16) in isBSwapHWordPair()
7585 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
7592 if (!N0->hasOneUse() || !N1->hasOneUse()) in matchBSwapHWordOrAndAnd()
7598 if (Mask0->getAPIntValue() != 0xff00ff00 || in matchBSwapHWordOrAndAnd()
7599 Mask1->getAPIntValue() != 0x00ff00ff) in matchBSwapHWordOrAndAnd()
7609 if (ShiftAmt0->getAPIntValue() != 8 || ShiftAmt1->getAPIntValue() != 8) in matchBSwapHWordOrAndAnd()
7620 /// Match a 32-bit packed halfword bswap. That is
7630 EVT VT = N->getValueType(0); in MatchBSwapHWord()
7692 // fold (or x, undef) -> -1 in visitORLike()
7699 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible. in visitORLike()
7702 (N0->hasOneUse() || N1->hasOneUse())) { in visitORLike()
7711 const APInt &LHSMask = N0O1C->getAPIntValue(); in visitORLike()
7712 const APInt &RHSMask = N1O1C->getAPIntValue(); in visitORLike()
7725 // (or (and X, M), (and X, N)) -> (and X, (or M, N)) in visitORLike()
7730 (N0->hasOneUse() || N1->hasOneUse())) { in visitORLike()
7747 if (V->getOpcode() == ISD::ZERO_EXTEND || V->getOpcode() == ISD::TRUNCATE) in visitORCommutative()
7748 return V->getOperand(0); in visitORCommutative()
7758 // fold or (and x, y), x --> x in visitORCommutative()
7762 // fold (or (and X, (xor Y, -1)), Y) -> (or X, Y) in visitORCommutative()
7771 // fold (or (and (xor Y, -1), X), Y) -> (or X, Y) in visitORCommutative()
7782 // fold or (xor X, N1), N1 --> or X, N1 in visitORCommutative()
7786 // fold or (xor x, y), (x and/or y) --> or x, y in visitORCommutative()
7796 if (V->getOpcode() == ISD::ZERO_EXTEND) in visitORCommutative()
7797 return V->getOperand(0); in visitORCommutative()
7801 // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y in visitORCommutative()
7807 // (fshr ?, X, Y) | (srl X, Y) --> fshr ?, X, Y in visitORCommutative()
7813 // Attempt to match a legalized build_pair-esque pattern: in visitORCommutative()
7821 // Fold build_pair(not(Lo),not(Hi)) -> not(build_pair(Lo,Hi)). in visitORCommutative()
7837 SDValue N0 = N->getOperand(0); in visitOR()
7838 SDValue N1 = N->getOperand(1); in visitOR()
7842 // x | x --> x in visitOR()
7846 // fold (or c1, c2) -> c1|c2 in visitOR()
7860 // fold (or x, 0) -> x, vector edition in visitOR()
7864 // fold (or x, -1) -> -1, vector edition in visitOR()
7869 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask) in visitOR()
7884 SmallVector<int, 4> Mask(NumElts, -1); in visitOR()
7887 int M0 = SV0->getMaskElt(i); in visitOR()
7888 int M1 = SV1->getMaskElt(i); in visitOR()
7907 // We have a zero and non-zero element. If the non-zero came from in visitOR()
7926 // fold (or x, 0) -> x in visitOR()
7930 // fold (or x, -1) -> -1 in visitOR()
7937 // fold (or x, c) -> c iff (x & ~c) == 0 in visitOR()
7939 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue())) in visitOR()
7958 if (SDValue ROR = reassociateOps(ISD::OR, DL, N0, N1, N->getFlags())) in visitOR()
7961 // Fold or(vecreduce(x), vecreduce(y)) -> vecreduce(or(x, y)) in visitOR()
7966 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2) in visitOR()
7969 return !C1 || !C2 || C1->getAPIntValue().intersects(C2->getAPIntValue()); in visitOR()
7971 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() && in visitOR()
7986 // Simplify: (or (op x...), (op y...)) -> (op (or x, y)) in visitOR()
7998 // Simplify the operands using demanded-bits information. in visitOR()
8046 /// (or (add v v) (shrl v bitwidth-1)):
8047 /// expands (add v v) -> (shl v 1)
8050 /// expands (mul v c0) -> (shl (mul v c1) c3)
8053 /// expands (udiv v c0) -> (shrl (udiv v c1) c3)
8056 /// expands (shl v c0) -> (shl (shl v c1) c3)
8059 /// expands (shrl v c0) -> (shrl (shrl v c1) c3)
8078 // (add v v) -> (shl v 1) in extractShiftForRotate()
8084 OppShiftCst->getAPIntValue() == ShiftedVT.getScalarSizeInBits() - 1) in extractShiftForRotate()
8121 // TODO: We should be able to handle non-uniform constant vectors for these values in extractShiftForRotate()
8123 if (!OppShiftCst || !OppShiftCst->getAPIntValue() || in extractShiftForRotate()
8124 !OppLHSCst || !OppLHSCst->getAPIntValue() || in extractShiftForRotate()
8125 !ExtractFromCst || !ExtractFromCst->getAPIntValue()) in extractShiftForRotate()
8130 if (OppShiftCst->getAPIntValue().ugt(VTWidth)) in extractShiftForRotate()
8132 APInt NeededShiftAmt = VTWidth - OppShiftCst->getAPIntValue(); in extractShiftForRotate()
8134 APInt ExtractFromAmt = ExtractFromCst->getAPIntValue(); in extractShiftForRotate()
8135 APInt OppLHSAmt = OppLHSCst->getAPIntValue(); in extractShiftForRotate()
8143 // c2 / (1 << (bitwidth(op0 v c0) - c1)) == c0 in extractShiftForRotate()
8144 // c2 % (1 << (bitwidth(op0 v c0) - c1)) == 0 in extractShiftForRotate()
8155 // c2 - (bitwidth(op0 v c0) - c1) == c0 in extractShiftForRotate()
8156 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.zextOrTrunc( in extractShiftForRotate()
8169 // range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
8185 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1) in matchRotateSub()
8186 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize). in matchRotateSub()
8188 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check in matchRotateSub()
8191 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A] in matchRotateSub()
8193 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1) in matchRotateSub()
8198 // Neg == EltSize - Pos [B] in matchRotateSub()
8211 // always invokes undefined behavior for 32-bit X. in matchRotateSub()
8213 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise. in matchRotateSub()
8215 // un-demanded bits. in matchRotateSub()
8257 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask in matchRotateSub()
8270 Width = NegC->getAPIntValue(); in matchRotateSub()
8275 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask in matchRotateSub()
8279 // NegC & Mask == (EltSize - PosC) & Mask in matchRotateSub()
8283 Width = PosC->getAPIntValue() + NegC->getAPIntValue(); in matchRotateSub()
8291 // EltSize & Mask is 0 since Mask is EltSize - 1. in matchRotateSub()
8297 // shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
8307 // (srl x, (*ext (sub 32, y)))) -> in MatchRotatePosNeg()
8311 // (srl x, (*ext y))) -> in MatchRotatePosNeg()
8324 // shifts of N0 + N1. If Neg == <operand size> - Pos then the OR reduces
8338 // (srl x1, (*ext (sub 32, y)))) -> in MatchFunnelPosNeg()
8342 // (srl x1, (*ext y))) -> in MatchFunnelPosNeg()
8357 return Cst && (Cst->getAPIntValue() == Imm); in MatchFunnelPosNeg()
8361 // -> (fshl x0, x1, y) in MatchFunnelPosNeg()
8363 IsBinOpImm(InnerNeg, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8370 // -> (fshr x0, x1, y) in MatchFunnelPosNeg()
8372 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8379 // -> (fshr x0, x1, y) in MatchFunnelPosNeg()
8380 // TODO: Should add(x,x) -> shl(x,1) be a general DAG canonicalization? in MatchFunnelPosNeg()
8382 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8392 // MatchRotate - Handle an 'or' of two operands. If this is one of the many
8400 // We still try to match rotate by constant pre-legalization. in MatchRotate()
8401 // TODO: Support pre-legalization funnel-shift by constant. in MatchRotate()
8408 // lowering for rotate, allow matching rotate by non-constants. Only allow in MatchRotate()
8476 // Something has gone wrong - we've lost the shl/srl pair - bail. in MatchRotate()
8488 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits; in MatchRotate()
8514 // TODO: Support pre-legalization funnel-shift by constant. in MatchRotate()
8540 // (shl (X | Y), C1) | (srl X, C2) --> (rotl X, C1) | (shl Y, C1) in MatchRotate()
8545 // (shl X, C1) | (srl (X | Y), C2) --> (rotl X, C1) | (srl Y, C2) in MatchRotate()
8559 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1) in MatchRotate()
8560 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2) in MatchRotate()
8561 // fold (or (shl x, C1), (srl y, C2)) -> (fshl x, y, C1) in MatchRotate()
8562 // fold (or (shl x, C1), (srl y, C2)) -> (fshr x, y, C2) in MatchRotate()
8579 // Even pre-legalization, we can't easily rotate/funnel-shift by a variable in MatchRotate()
8589 // If the shift amount is sign/zext/any-extended just peel it off. in MatchRotate()
8642 /// However, there is a special case when dealing with vector loads -- we allow
8706 calculateByteProvider(Op->getOperand(0), Index, Depth + 1, VectorIndex); in calculateByteProvider()
8710 calculateByteProvider(Op->getOperand(1), Index, Depth + 1, VectorIndex); in calculateByteProvider()
8714 if (LHS->isConstantZero()) in calculateByteProvider()
8716 if (RHS->isConstantZero()) in calculateByteProvider()
8721 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1)); in calculateByteProvider()
8725 uint64_t BitShift = ShiftOp->getZExtValue(); in calculateByteProvider()
8736 : calculateByteProvider(Op->getOperand(0), Index - ByteShift, in calculateByteProvider()
8742 SDValue NarrowOp = Op->getOperand(0); in calculateByteProvider()
8757 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1, in calculateByteProvider()
8760 auto OffsetOp = dyn_cast<ConstantSDNode>(Op->getOperand(1)); in calculateByteProvider()
8764 VectorIndex = OffsetOp->getZExtValue(); in calculateByteProvider()
8766 SDValue NarrowOp = Op->getOperand(0); in calculateByteProvider()
8787 return calculateByteProvider(Op->getOperand(0), Index, Depth + 1, in calculateByteProvider()
8792 if (!L->isSimple() || L->isIndexed()) in calculateByteProvider()
8795 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits(); in calculateByteProvider()
8804 return L->getExtensionType() == ISD::ZEXTLOAD in calculateByteProvider()
8822 return BW - i - 1; in bigEndianByteAt()
8837 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset; in isBigEndian()
8892 // We only handle merging simple stores of 1-4 bytes. in mergeTruncStores()
8894 EVT MemVT = N->getMemoryVT(); in mergeTruncStores()
8896 !N->isSimple() || N->isIndexed()) in mergeTruncStores()
8900 SDValue Chain = N->getChain(); in mergeTruncStores()
8913 if (Store->getMemoryVT() != MemVT || !Store->isSimple() || in mergeTruncStores()
8914 Store->isIndexed() || !Store->hasOneUse()) in mergeTruncStores()
8917 Chain = Store->getChain(); in mergeTruncStores()
8943 SDValue Trunc = Store->getValue(); in mergeTruncStores()
8963 if (ShiftAmtC > WideVal.getScalarValueSizeInBits() - NarrowNumBits) in mergeTruncStores()
8993 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase)) in mergeTruncStores()
9015 *FirstStore->getMemOperand(), &Fast); in mergeTruncStores()
9027 for (unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j) in mergeTruncStores()
9038 // Special-case: check if byte offsets line up for the opposite endian. in mergeTruncStores()
9066 DAG.getStore(Chain, DL, SourceValue, FirstStore->getBasePtr(), in mergeTruncStores()
9067 FirstStore->getPointerInfo(), FirstStore->getAlign()); in mergeTruncStores()
9105 assert(N->getOpcode() == ISD::OR && in MatchLoadCombine()
9109 EVT VT = N->getValueType(0); in MatchLoadCombine()
9119 unsigned LoadBitWidth = Load->getMemoryVT().getScalarSizeInBits(); in MatchLoadCombine()
9139 for (int i = ByteWidth - 1; i >= 0; --i) { in MatchLoadCombine()
9146 if (P->isConstantZero()) { in MatchLoadCombine()
9148 // zero-extend the load. in MatchLoadCombine()
9149 if (++ZeroExtendedBytes != (ByteWidth - static_cast<unsigned>(i))) in MatchLoadCombine()
9153 assert(P->hasSrc() && "provenance should either be memory or zero"); in MatchLoadCombine()
9154 auto *L = cast<LoadSDNode>(P->Src.value()); in MatchLoadCombine()
9157 SDValue LChain = L->getChain(); in MatchLoadCombine()
9174 if (L->getMemoryVT().isVector()) { in MatchLoadCombine()
9175 unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits(); in MatchLoadCombine()
9178 unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8; in MatchLoadCombine()
9185 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase)) in MatchLoadCombine()
9209 EVT::getIntegerVT(*DAG.getContext(), (ByteWidth - ZeroExtendedBytes) * 8); in MatchLoadCombine()
9235 auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value()); in MatchLoadCombine()
9238 // replace it with a single (possibly zero-extended) load and bswap + shift if in MatchLoadCombine()
9247 // We do not introduce illegal bswaps when zero-extending as this tends to in MatchLoadCombine()
9263 *FirstLoad->getMemOperand(), &Fast); in MatchLoadCombine()
9269 Chain, FirstLoad->getBasePtr(), in MatchLoadCombine()
9270 FirstLoad->getPointerInfo(), MemVT, FirstLoad->getAlign()); in MatchLoadCombine()
9287 // If the target has andn, bsl, or a similar bit-select instruction,
9304 assert(N->getOpcode() == ISD::XOR); in unfoldMaskedMerge()
9306 // Don't touch 'not' (i.e. where y = -1). in unfoldMaskedMerge()
9307 if (isAllOnesOrAllOnesSplat(N->getOperand(1))) in unfoldMaskedMerge()
9310 EVT VT = N->getValueType(0); in unfoldMaskedMerge()
9323 // Don't touch 'not' (i.e. where y = -1). in unfoldMaskedMerge()
9336 SDValue N0 = N->getOperand(0); in unfoldMaskedMerge()
9337 SDValue N1 = N->getOperand(1); in unfoldMaskedMerge()
9387 SDValue N0 = N->getOperand(0); in visitXOR()
9388 SDValue N1 = N->getOperand(1); in visitXOR()
9392 // fold (xor undef, undef) -> 0. This is a common idiom (misuse). in visitXOR()
9396 // fold (xor x, undef) -> undef in visitXOR()
9402 // fold (xor c1, c2) -> c1^c2 in visitXOR()
9416 // fold (xor x, 0) -> x, vector edition in visitXOR()
9421 // fold (xor x, 0) -> x in visitXOR()
9429 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) in visitXOR()
9432 // Fold xor(vecreduce(x), vecreduce(y)) -> vecreduce(xor(x, y)) in visitXOR()
9437 // fold (a^b) -> (a|b) iff a and b share no bits. in visitXOR()
9445 // look for 'add-like' folds: in visitXOR()
9452 // fold !(x cc y) -> (x !cc y) in visitXOR()
9457 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), in visitXOR()
9488 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y))) in visitXOR()
9499 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc in visitXOR()
9511 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants in visitXOR()
9524 // fold (not (neg x)) -> (add X, -1) in visitXOR()
9525 // FIXME: This can be generalized to (not (sub Y, X)) -> (add X, ~Y) if in visitXOR()
9533 // fold (not (add X, -1)) -> (neg X) in visitXOR()
9539 // fold (xor (and x, y), y) -> (and (not x), y) in visitXOR()
9540 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) { in visitXOR()
9547 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X) in visitXOR()
9556 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitXOR()
9561 // fold (xor x, x) -> 0 in visitXOR()
9565 // fold (xor (shl 1, x), -1) -> (rotl ~1, x) in visitXOR()
9577 // - Try to see the operation as placing a single zero in a value of all ones. in visitXOR()
9578 // - There exists no value for x which would allow the result to contain zero. in visitXOR()
9579 // - Values of x larger than the bitwidth are undefined and do not require a in visitXOR()
9581 // - Pushing the zero left requires shifting one bits in from the right. in visitXOR()
9589 // Simplify: xor (op x...), (op y...) -> (op (xor x, y)) in visitXOR()
9605 // Simplify the expression using non-local knowledge. in visitXOR()
9615 /// If we have a shift-by-constant of a bitwise logic op that itself has a
9616 /// shift-by-constant operand with identical opcode, we may be able to convert
9620 // Match a one-use bitwise logic op. in combineShiftOfShiftedLogic()
9621 SDValue LogicOp = Shift->getOperand(0); in combineShiftOfShiftedLogic()
9630 // Find a matching one-use shift by constant. in combineShiftOfShiftedLogic()
9631 unsigned ShiftOpcode = Shift->getOpcode(); in combineShiftOfShiftedLogic()
9632 SDValue C1 = Shift->getOperand(1); in combineShiftOfShiftedLogic()
9635 const APInt &C1Val = C1Node->getAPIntValue(); in combineShiftOfShiftedLogic()
9647 ShiftAmtVal = &ShiftCNode->getAPIntValue(); in combineShiftOfShiftedLogic()
9651 if (ShiftAmtVal->getBitWidth() != C1Val.getBitWidth()) in combineShiftOfShiftedLogic()
9678 // shift (logic (shift X, C0), Y), C1 -> logic (shift X, C0+C1), (shift Y, C1) in combineShiftOfShiftedLogic()
9680 EVT VT = Shift->getValueType(0); in combineShiftOfShiftedLogic()
9681 EVT ShiftAmtVT = Shift->getOperand(1).getValueType(); in combineShiftOfShiftedLogic()
9686 LogicOp->getFlags()); in combineShiftOfShiftedLogic()
9696 assert(isConstOrConstSplat(N->getOperand(1)) && "Expected constant operand"); in visitShiftByConstant()
9699 if (isBitwiseNot(N->getOperand(0))) in visitShiftByConstant()
9702 // The inner binop must be one-use, since we want to replace it. in visitShiftByConstant()
9703 SDValue LHS = N->getOperand(0); in visitShiftByConstant()
9707 // Fold shift(bitop(shift(x,c1),y), c2) -> bitop(shift(x,c1+c2),shift(y,c2)). in visitShiftByConstant()
9723 if (N->getOpcode() != ISD::SHL) in visitShiftByConstant()
9742 if (IsCopyOrSelect && N->hasOneUse()) in visitShiftByConstant()
9747 EVT VT = N->getValueType(0); in visitShiftByConstant()
9749 N->getOpcode(), DL, VT, {LHS.getOperand(1), N->getOperand(1)})) { in visitShiftByConstant()
9750 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0), in visitShiftByConstant()
9751 N->getOperand(1)); in visitShiftByConstant()
9759 assert(N->getOpcode() == ISD::TRUNCATE); in distributeTruncateThroughAnd()
9760 assert(N->getOperand(0).getOpcode() == ISD::AND); in distributeTruncateThroughAnd()
9762 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC) in distributeTruncateThroughAnd()
9763 EVT TruncVT = N->getValueType(0); in distributeTruncateThroughAnd()
9764 if (N->hasOneUse() && N->getOperand(0).hasOneUse() && in distributeTruncateThroughAnd()
9766 SDValue N01 = N->getOperand(0).getOperand(1); in distributeTruncateThroughAnd()
9769 SDValue N00 = N->getOperand(0).getOperand(0); in distributeTruncateThroughAnd()
9783 SDValue N0 = N->getOperand(0); in visitRotate()
9784 SDValue N1 = N->getOperand(1); in visitRotate()
9785 EVT VT = N->getValueType(0); in visitRotate()
9788 // fold (rot x, 0) -> x in visitRotate()
9792 // fold (rot x, c) -> x iff (c % BitSize) == 0 in visitRotate()
9794 APInt ModuloMask(N1.getScalarValueSizeInBits(), Bitsize - 1); in visitRotate()
9799 // fold (rot x, c) -> (rot x, c % BitSize) in visitRotate()
9802 OutOfRange |= C->getAPIntValue().uge(Bitsize); in visitRotate()
9810 return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt); in visitRotate()
9813 // rot i16 X, 8 --> bswap X in visitRotate()
9815 if (RotAmtC && RotAmtC->getAPIntValue() == 8 && in visitRotate()
9819 // Simplify the operands using demanded-bits information. in visitRotate()
9823 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))). in visitRotate()
9827 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); in visitRotate()
9833 // -> (rot* x, ((c1 % bitsize) +- (c2 % bitsize) + bitsize) % bitsize) in visitRotate()
9837 if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) { in visitRotate()
9838 EVT ShiftVT = C1->getValueType(0); in visitRotate()
9839 bool SameSide = (N->getOpcode() == NextOp); in visitRotate()
9853 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0), in visitRotate()
9862 SDValue N0 = N->getOperand(0); in visitSHL()
9863 SDValue N1 = N->getOperand(1); in visitSHL()
9872 // fold (shl c1, c2) -> c1<<c2 in visitSHL()
9882 // If setcc produces all-one true value then: in visitSHL()
9883 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV) in visitSHL()
9884 if (N1CV && N1CV->isConstant()) { in visitSHL()
9886 SDValue N00 = N0->getOperand(0); in visitSHL()
9887 SDValue N01 = N0->getOperand(1); in visitSHL()
9890 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC && in visitSHL()
9908 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))). in visitSHL()
9915 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2)) in visitSHL()
9919 APInt c1 = LHS->getAPIntValue(); in visitSHL()
9920 APInt c2 = RHS->getAPIntValue(); in visitSHL()
9929 APInt c1 = LHS->getAPIntValue(); in visitSHL()
9930 APInt c2 = RHS->getAPIntValue(); in visitSHL()
9940 // fold (shl (ext (shl x, c1)), c2) -> (shl (ext x), (add c1, c2)) in visitSHL()
9956 APInt c1 = LHS->getAPIntValue(); in visitSHL()
9957 APInt c2 = RHS->getAPIntValue(); in visitSHL()
9959 return c2.uge(OpSizeInBits - InnerBitwidth) && in visitSHL()
9969 APInt c1 = LHS->getAPIntValue(); in visitSHL()
9970 APInt c2 = RHS->getAPIntValue(); in visitSHL()
9972 return c2.uge(OpSizeInBits - InnerBitwidth) && in visitSHL()
9985 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C)) in visitSHL()
9994 APInt c1 = LHS->getAPIntValue(); in visitSHL()
9995 APInt c2 = RHS->getAPIntValue(); in visitSHL()
10013 const APInt &LHSC = LHS->getAPIntValue(); in visitSHL()
10014 const APInt &RHSC = RHS->getAPIntValue(); in visitSHL()
10019 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2 in visitSHL()
10020 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 >= C2 in visitSHL()
10021 if (N0->getFlags().hasExact()) { in visitSHL()
10038 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or in visitSHL()
10040 // Only fold this if the inner shift has no other uses -- if it does, in visitSHL()
10069 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1)) in visitSHL()
10077 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2) in visitSHL()
10078 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2) in visitSHL()
10082 N0->hasOneUse() && TLI.isDesirableToCommuteWithShift(N, Level)) { in visitSHL()
10090 if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint()) in visitSHL()
10096 // fold (shl (sext (add_nsw x, c1)), c2) -> (add (shl (sext x), c2), c1 << c2) in visitSHL()
10101 N0.getOperand(0)->getFlags().hasNoSignedWrap() && N0->hasOneUse() && in visitSHL()
10102 N0.getOperand(0)->hasOneUse() && in visitSHL()
10117 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2) in visitSHL()
10118 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) { in visitSHL()
10126 if (N1C && !N1C->isOpaque()) in visitSHL()
10130 // fold (shl X, cttz(Y)) -> (mul (Y & -Y), X) if cttz is unsupported on the in visitSHL()
10151 const APInt &C1 = N1C->getAPIntValue(); in visitSHL()
10169 // Transform a right shift of a multiply into a multiply-high.
10171 // (srl (mul (zext i32:$a to i64), (zext i32:$a to i64)), 32) -> (mulhu $a, $b)
10172 // (sra (mul (sext i32:$a to i64), (sext i32:$a to i64)), 32) -> (mulhs $a, $b)
10175 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && in combineShiftToMULH()
10180 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N->getOperand(1)); in combineShiftToMULH()
10185 SDValue ShiftOperand = N->getOperand(0); in combineShiftToMULH()
10204 if (U->getOpcode() != ISD::SRL && U->getOpcode() != ISD::SRA) { in combineShiftToMULH()
10207 ConstantSDNode *UShiftAmtSrc = isConstOrConstSplat(U->getOperand(1)); in combineShiftToMULH()
10211 unsigned UShiftAmt = UShiftAmtSrc->getZExtValue(); in combineShiftToMULH()
10220 llvm::any_of(ShiftOperand->uses(), UserOfLowerBits)) { in combineShiftToMULH()
10227 ? Constant->getAPIntValue().getSignificantBits() in combineShiftToMULH()
10228 : Constant->getAPIntValue().getActiveBits(); in combineShiftToMULH()
10232 Constant->getAPIntValue().trunc(NarrowVT.getScalarSizeInBits()), DL, in combineShiftToMULH()
10256 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue(); in combineShiftToMULH()
10279 bool IsSigned = N->getOpcode() == ISD::SRA; in combineShiftToMULH()
10283 // fold (bswap (logic_op(bswap(x),y))) -> logic_op(x,bswap(y))
10286 unsigned Opcode = N->getOpcode(); in foldBitOrderCrossLogicOp()
10290 SDValue N0 = N->getOperand(0); in foldBitOrderCrossLogicOp()
10291 EVT VT = N->getValueType(0); in foldBitOrderCrossLogicOp()
10320 SDValue N0 = N->getOperand(0); in visitSRA()
10321 SDValue N1 = N->getOperand(1); in visitSRA()
10329 // fold (sra c1, c2) -> (sra c1, c2) in visitSRA()
10333 // Arithmetic shifting an all-sign-bit value is a no-op. in visitSRA()
10334 // fold (sra 0, x) -> 0 in visitSRA()
10335 // fold (sra -1, x) -> -1 in visitSRA()
10349 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2)) in visitSRA()
10357 APInt c1 = LHS->getAPIntValue(); in visitSRA()
10358 APInt c2 = RHS->getAPIntValue(); in visitSRA()
10362 Sum.uge(OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue(); in visitSRA()
10382 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for in visitSRA()
10383 // result_size - n != m. in visitSRA()
10392 EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue()); in visitSRA()
10397 // Determine the residual right-shift amount. in visitSRA()
10398 int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue(); in visitSRA()
10400 // If the shift is not a no-op (in which case this should be just a sign in visitSRA()
10414 N->getValueType(0), Trunc); in visitSRA()
10420 // sra (add (shl X, N1C), AddC), N1C --> in visitSRA()
10421 // sext (add (trunc X to (width - N1C)), AddC') in visitSRA()
10422 // sra (sub AddC, (shl X, N1C)), N1C --> in visitSRA()
10423 // sext (sub AddC1',(trunc X to (width - N1C))) in visitSRA()
10436 unsigned ShiftAmt = N1C->getZExtValue(); in visitSRA()
10437 EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - ShiftAmt); in visitSRA()
10442 // implementation and/or target-specific overrides (because in visitSRA()
10443 // non-simple types likely require masking when legalized), but in visitSRA()
10449 DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).trunc( in visitSRA()
10463 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))). in visitSRA()
10470 // fold (sra (trunc (sra x, c1)), c2) -> (trunc (sra x, c1 + c2)) in visitSRA()
10471 // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2)) in visitSRA()
10473 // TODO - support non-uniform vector shift amounts. in visitSRA()
10482 unsigned TruncBits = LargeVT.getScalarSizeInBits() - OpSizeInBits; in visitSRA()
10483 if (LargeShift->getAPIntValue() == TruncBits) { in visitSRA()
10503 if (N1C && !N1C->isOpaque()) in visitSRA()
10507 // Try to transform this shift into a multiply-high if in visitSRA()
10512 // Attempt to convert a sra of a load into a narrower sign-extending load. in visitSRA()
10520 SDValue N0 = N->getOperand(0); in visitSRL()
10521 SDValue N1 = N->getOperand(1); in visitSRL()
10530 // fold (srl c1, c2) -> c1 >>u c2 in visitSRL()
10548 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2)) in visitSRL()
10552 APInt c1 = LHS->getAPIntValue(); in visitSRL()
10553 APInt c2 = RHS->getAPIntValue(); in visitSRL()
10562 APInt c1 = LHS->getAPIntValue(); in visitSRL()
10563 APInt c2 = RHS->getAPIntValue(); in visitSRL()
10576 // TODO - support non-uniform vector shift amounts. in visitSRL()
10578 uint64_t c1 = N001C->getZExtValue(); in visitSRL()
10579 uint64_t c2 = N1C->getZExtValue(); in visitSRL()
10583 // srl (trunc (srl x, c1)), c2 --> 0 or (trunc (srl x, (add c1, c2))) in visitSRL()
10594 // srl (trunc (srl x, c1)), c2 --> trunc (and (srl x, (c1+c2)), Mask) in visitSRL()
10601 OpSizeInBits - c2), in visitSRL()
10609 // fold (srl (shl x, c1), c2) -> (and (shl x, (sub c1, c2), MASK) or in visitSRL()
10612 (N0.getOperand(1) == N1 || N0->hasOneUse()) && in visitSRL()
10616 const APInt &LHSC = LHS->getAPIntValue(); in visitSRL()
10617 const APInt &RHSC = RHS->getAPIntValue(); in visitSRL()
10644 // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask) in visitSRL()
10645 // TODO - support non-uniform vector shift amounts. in visitSRL()
10650 if (N1C->getAPIntValue().uge(BitSize)) in visitSRL()
10654 uint64_t ShiftAmt = N1C->getZExtValue(); in visitSRL()
10660 APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt); in visitSRL()
10667 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign in visitSRL()
10669 if (N1C && N1C->getAPIntValue() == (OpSizeInBits - 1)) { in visitSRL()
10674 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit), and x has a power in visitSRL()
10678 N1C->getAPIntValue() == Log2_32(OpSizeInBits)) { in visitSRL()
10709 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))). in visitSRL()
10721 if (N1C && !N1C->isOpaque()) in visitSRL()
10725 // Attempt to convert a srl of a load into a narrower zero-extending load. in visitSRL()
10753 if (N->hasOneUse()) { in visitSRL()
10754 SDNode *Use = *N->use_begin(); in visitSRL()
10757 if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) in visitSRL()
10758 Use = *Use->use_begin(); in visitSRL()
10760 if (Use->getOpcode() == ISD::BRCOND || Use->getOpcode() == ISD::AND || in visitSRL()
10761 Use->getOpcode() == ISD::OR || Use->getOpcode() == ISD::XOR) in visitSRL()
10765 // Try to transform this shift into a multiply-high if in visitSRL()
10774 EVT VT = N->getValueType(0); in visitFunnelShift()
10775 SDValue N0 = N->getOperand(0); in visitFunnelShift()
10776 SDValue N1 = N->getOperand(1); in visitFunnelShift()
10777 SDValue N2 = N->getOperand(2); in visitFunnelShift()
10778 bool IsFSHL = N->getOpcode() == ISD::FSHL; in visitFunnelShift()
10782 // fold (fshl N0, N1, 0) -> N0 in visitFunnelShift()
10783 // fold (fshr N0, N1, 0) -> N1 in visitFunnelShift()
10786 N2, APInt(N2.getScalarValueSizeInBits(), BitWidth - 1))) in visitFunnelShift()
10793 // TODO - support non-uniform vector shift amounts. in visitFunnelShift()
10797 // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth) in visitFunnelShift()
10798 if (Cst->getAPIntValue().uge(BitWidth)) { in visitFunnelShift()
10799 uint64_t RotAmt = Cst->getAPIntValue().urem(BitWidth); in visitFunnelShift()
10800 return DAG.getNode(N->getOpcode(), DL, VT, N0, N1, in visitFunnelShift()
10804 unsigned ShAmt = Cst->getZExtValue(); in visitFunnelShift()
10808 // fold fshl(undef_or_zero, N1, C) -> lshr(N1, BW-C) in visitFunnelShift()
10809 // fold fshr(undef_or_zero, N1, C) -> lshr(N1, C) in visitFunnelShift()
10810 // fold fshl(N0, undef_or_zero, C) -> shl(N0, C) in visitFunnelShift()
10811 // fold fshr(N0, undef_or_zero, C) -> shl(N0, BW-C) in visitFunnelShift()
10815 DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt, DL, ShAmtTy)); in visitFunnelShift()
10819 DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt, DL, ShAmtTy)); in visitFunnelShift()
10821 // fold (fshl ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive. in visitFunnelShift()
10822 // fold (fshr ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive. in visitFunnelShift()
10823 // TODO - bigendian support once we have test coverage. in visitFunnelShift()
10824 // TODO - can we merge this with CombineConseutiveLoads/MatchLoadCombine? in visitFunnelShift()
10825 // TODO - permit LHS EXTLOAD if extensions are shifted out. in visitFunnelShift()
10830 if (LHS && RHS && LHS->isSimple() && RHS->isSimple() && in visitFunnelShift()
10831 LHS->getAddressSpace() == RHS->getAddressSpace() && in visitFunnelShift()
10832 (LHS->hasOneUse() || RHS->hasOneUse()) && ISD::isNON_EXTLoad(RHS) && in visitFunnelShift()
10837 IsFSHL ? (((BitWidth - ShAmt) % BitWidth) / 8) : (ShAmt / 8); in visitFunnelShift()
10838 Align NewAlign = commonAlignment(RHS->getAlign(), PtrOff); in visitFunnelShift()
10841 RHS->getAddressSpace(), NewAlign, in visitFunnelShift()
10842 RHS->getMemOperand()->getFlags(), &Fast) && in visitFunnelShift()
10845 RHS->getBasePtr(), TypeSize::getFixed(PtrOff), DL); in visitFunnelShift()
10848 VT, DL, RHS->getChain(), NewPtr, in visitFunnelShift()
10849 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign, in visitFunnelShift()
10850 RHS->getMemOperand()->getFlags(), RHS->getAAInfo()); in visitFunnelShift()
10861 // fold fshr(undef_or_zero, N1, N2) -> lshr(N1, N2) in visitFunnelShift()
10862 // fold fshl(N0, undef_or_zero, N2) -> shl(N0, N2) in visitFunnelShift()
10866 APInt ModuloBits(N2.getScalarValueSizeInBits(), BitWidth - 1); in visitFunnelShift()
10873 // fold (fshl N0, N0, N2) -> (rotl N0, N2) in visitFunnelShift()
10874 // fold (fshr N0, N0, N2) -> (rotr N0, N2) in visitFunnelShift()
10877 // non-constant (BW - N2). in visitFunnelShift()
10890 SDValue N0 = N->getOperand(0); in visitSHLSAT()
10891 SDValue N1 = N->getOperand(1); in visitSHLSAT()
10898 // fold (*shlsat c1, c2) -> c1<<c2 in visitSHLSAT()
10899 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1})) in visitSHLSAT()
10905 // fold (sshlsat x, c) -> (shl x, c) in visitSHLSAT()
10906 if (N->getOpcode() == ISD::SSHLSAT && N1C && in visitSHLSAT()
10907 N1C->getAPIntValue().ult(DAG.ComputeNumSignBits(N0))) in visitSHLSAT()
10910 // fold (ushlsat x, c) -> (shl x, c) in visitSHLSAT()
10911 if (N->getOpcode() == ISD::USHLSAT && N1C && in visitSHLSAT()
10912 N1C->getAPIntValue().ule( in visitSHLSAT()
10925 EVT SrcVT = N->getValueType(0); in foldABSToABD()
10927 if (N->getOpcode() == ISD::TRUNCATE) in foldABSToABD()
10928 N = N->getOperand(0).getNode(); in foldABSToABD()
10930 if (N->getOpcode() != ISD::ABS) in foldABSToABD()
10933 EVT VT = N->getValueType(0); in foldABSToABD()
10934 SDValue AbsOp1 = N->getOperand(0); in foldABSToABD()
10945 // Check if the operands of the sub are (zero|sign)-extended. in foldABSToABD()
10950 // fold (abs (sub nsw x, y)) -> abds(x, y) in foldABSToABD()
10951 if (AbsOp1->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) && in foldABSToABD()
10961 VT0 = cast<VTSDNode>(Op0.getOperand(1))->getVT(); in foldABSToABD()
10962 VT1 = cast<VTSDNode>(Op1.getOperand(1))->getVT(); in foldABSToABD()
10969 // fold abs(sext(x) - sext(y)) -> zext(abds(x, y)) in foldABSToABD()
10970 // fold abs(zext(x) - zext(y)) -> zext(abdu(x, y)) in foldABSToABD()
10972 if ((VT0 == MaxVT || Op0->hasOneUse()) && in foldABSToABD()
10973 (VT1 == MaxVT || Op1->hasOneUse()) && hasOperation(ABDOpcode, MaxVT)) { in foldABSToABD()
10981 // fold abs(sext(x) - sext(y)) -> abds(sext(x), sext(y)) in foldABSToABD()
10982 // fold abs(zext(x) - zext(y)) -> abdu(zext(x), zext(y)) in foldABSToABD()
10992 SDValue N0 = N->getOperand(0); in visitABS()
10993 EVT VT = N->getValueType(0); in visitABS()
10996 // fold (abs c1) -> c2 in visitABS()
10999 // fold (abs (abs x)) -> (abs x) in visitABS()
11002 // fold (abs x) -> x iff not-negative in visitABS()
11009 // fold (abs (sign_extend_inreg x)) -> (zero_extend (abs (truncate x))) in visitABS()
11012 EVT ExtVT = cast<VTSDNode>(N0.getOperand(1))->getVT(); in visitABS()
11027 SDValue N0 = N->getOperand(0); in visitBSWAP()
11028 EVT VT = N->getValueType(0); in visitBSWAP()
11031 // fold (bswap c1) -> c2 in visitBSWAP()
11034 // fold (bswap (bswap x)) -> x in visitBSWAP()
11038 // Canonicalize bswap(bitreverse(x)) -> bitreverse(bswap(x)). If bitreverse in visitBSWAP()
11047 // fold (bswap shl(x,c)) -> (zext(bswap(trunc(shl(x,sub(c,bw/2)))))) in visitBSWAP()
11053 if (ShAmt && ShAmt->getAPIntValue().ult(BW) && in visitBSWAP()
11054 ShAmt->getZExtValue() >= (BW / 2) && in visitBSWAP()
11055 (ShAmt->getZExtValue() % 16) == 0 && TLI.isTypeLegal(HalfVT) && in visitBSWAP()
11059 if (uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2))) in visitBSWAP()
11068 // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as in visitBSWAP()
11069 // inverse-shift-of-bswap: in visitBSWAP()
11070 // bswap (X u<< C) --> (bswap X) u>> C in visitBSWAP()
11071 // bswap (X u>> C) --> (bswap X) u<< C in visitBSWAP()
11075 if (ShAmt && ShAmt->getAPIntValue().ult(BW) && in visitBSWAP()
11076 ShAmt->getZExtValue() % 8 == 0) { in visitBSWAP()
11090 SDValue N0 = N->getOperand(0); in visitBITREVERSE()
11091 EVT VT = N->getValueType(0); in visitBITREVERSE()
11094 // fold (bitreverse c1) -> c2 in visitBITREVERSE()
11098 // fold (bitreverse (bitreverse x)) -> x in visitBITREVERSE()
11104 // fold (bitreverse (lshr (bitreverse x), y)) -> (shl x, y) in visitBITREVERSE()
11109 // fold (bitreverse (shl (bitreverse x), y)) -> (lshr x, y) in visitBITREVERSE()
11118 SDValue N0 = N->getOperand(0); in visitCTLZ()
11119 EVT VT = N->getValueType(0); in visitCTLZ()
11122 // fold (ctlz c1) -> c2 in visitCTLZ()
11135 SDValue N0 = N->getOperand(0); in visitCTLZ_ZERO_UNDEF()
11136 EVT VT = N->getValueType(0); in visitCTLZ_ZERO_UNDEF()
11139 // fold (ctlz_zero_undef c1) -> c2 in visitCTLZ_ZERO_UNDEF()
11147 SDValue N0 = N->getOperand(0); in visitCTTZ()
11148 EVT VT = N->getValueType(0); in visitCTTZ()
11151 // fold (cttz c1) -> c2 in visitCTTZ()
11164 SDValue N0 = N->getOperand(0); in visitCTTZ_ZERO_UNDEF()
11165 EVT VT = N->getValueType(0); in visitCTTZ_ZERO_UNDEF()
11168 // fold (cttz_zero_undef c1) -> c2 in visitCTTZ_ZERO_UNDEF()
11176 SDValue N0 = N->getOperand(0); in visitCTPOP()
11177 EVT VT = N->getValueType(0); in visitCTPOP()
11181 // fold (ctpop c1) -> c2 in visitCTPOP()
11189 const APInt &Amt = AmtC->getAPIntValue(); in visitCTPOP()
11300 // select (setcc x, K) (fneg x), -K -> fneg(minnum(x, K)) in combineMinNumMaxNum()
11322 /// If a (v)select has a condition value that is a sign-bit test, try to smear
11323 /// the condition operand sign-bit across the value width and use it as a mask.
11326 SDValue Cond = N->getOperand(0); in foldSelectOfConstantsUsingSra()
11327 SDValue C1 = N->getOperand(1); in foldSelectOfConstantsUsingSra()
11328 SDValue C2 = N->getOperand(2); in foldSelectOfConstantsUsingSra()
11332 EVT VT = N->getValueType(0); in foldSelectOfConstantsUsingSra()
11337 // The inverted-condition + commuted-select variants of these patterns are in foldSelectOfConstantsUsingSra()
11341 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in foldSelectOfConstantsUsingSra()
11344 // i32 X > -1 ? C1 : -1 --> (X >>s 31) | C1 in foldSelectOfConstantsUsingSra()
11345 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
11350 // i8 X < 0 ? C1 : 0 --> (X >>s 7) & C1 in foldSelectOfConstantsUsingSra()
11351 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
11363 if (Cond.getOpcode() != ISD::SETCC || !Cond->hasOneUse()) in shouldConvertSelectOfConstantsToMath()
11368 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in shouldConvertSelectOfConstantsToMath()
11378 SDValue Cond = N->getOperand(0); in foldSelectOfConstants()
11379 SDValue N1 = N->getOperand(1); in foldSelectOfConstants()
11380 SDValue N2 = N->getOperand(2); in foldSelectOfConstants()
11381 EVT VT = N->getValueType(0); in foldSelectOfConstants()
11394 // fold (select Cond, 0, 1) -> (xor Cond, 1) in foldSelectOfConstants()
11397 // have an integer-based boolean or a floating-point-based boolean unless we in foldSelectOfConstants()
11408 C1->isZero() && C2->isOne()) { in foldSelectOfConstants()
11419 // Only do this before legalization to avoid conflicting with target-specific in foldSelectOfConstants()
11421 // is also a target-independent combine here in DAGCombiner in the other in foldSelectOfConstants()
11422 // direction for (select Cond, -1, 0) when the condition is not i1. in foldSelectOfConstants()
11425 // select Cond, 1, 0 --> zext (Cond) in foldSelectOfConstants()
11426 if (C1->isOne() && C2->isZero()) in foldSelectOfConstants()
11429 // select Cond, -1, 0 --> sext (Cond) in foldSelectOfConstants()
11430 if (C1->isAllOnes() && C2->isZero()) in foldSelectOfConstants()
11433 // select Cond, 0, 1 --> zext (!Cond) in foldSelectOfConstants()
11434 if (C1->isZero() && C2->isOne()) { in foldSelectOfConstants()
11440 // select Cond, 0, -1 --> sext (!Cond) in foldSelectOfConstants()
11441 if (C1->isZero() && C2->isAllOnes()) { in foldSelectOfConstants()
11454 const APInt &C1Val = C1->getAPIntValue(); in foldSelectOfConstants()
11455 const APInt &C2Val = C2->getAPIntValue(); in foldSelectOfConstants()
11457 // select Cond, C1, C1-1 --> add (zext Cond), C1-1 in foldSelectOfConstants()
11458 if (C1Val - 1 == C2Val) { in foldSelectOfConstants()
11463 // select Cond, C1, C1+1 --> add (sext Cond), C1+1 in foldSelectOfConstants()
11469 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2) in foldSelectOfConstants()
11477 // select Cond, -1, C --> or (sext Cond), C in foldSelectOfConstants()
11478 if (C1->isAllOnes()) { in foldSelectOfConstants()
11483 // select Cond, C, -1 --> or (sext (not Cond)), C in foldSelectOfConstants()
11484 if (C2->isAllOnes()) { in foldSelectOfConstants()
11499 assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT || in foldBoolSelectToLogic()
11500 N->getOpcode() == ISD::VP_SELECT) && in foldBoolSelectToLogic()
11502 SDValue Cond = N->getOperand(0); in foldBoolSelectToLogic()
11503 SDValue T = N->getOperand(1), F = N->getOperand(2); in foldBoolSelectToLogic()
11504 EVT VT = N->getValueType(0); in foldBoolSelectToLogic()
11511 // select Cond, Cond, F --> or Cond, freeze(F) in foldBoolSelectToLogic()
11512 // select Cond, 1, F --> or Cond, freeze(F) in foldBoolSelectToLogic()
11516 // select Cond, T, Cond --> and Cond, freeze(T) in foldBoolSelectToLogic()
11517 // select Cond, T, 0 --> and Cond, freeze(T) in foldBoolSelectToLogic()
11521 // select Cond, T, 1 --> or (not Cond), freeze(T) in foldBoolSelectToLogic()
11528 // select Cond, 0, F --> and (not Cond), freeze(F) in foldBoolSelectToLogic()
11539 SDValue N0 = N->getOperand(0); in foldVSelectToSignBitSplatMask()
11540 SDValue N1 = N->getOperand(1); in foldVSelectToSignBitSplatMask()
11541 SDValue N2 = N->getOperand(2); in foldVSelectToSignBitSplatMask()
11542 EVT VT = N->getValueType(0); in foldVSelectToSignBitSplatMask()
11552 // compare is inverted from that pattern ("Cond0 s> -1"). in foldVSelectToSignBitSplatMask()
11560 // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & freeze(N1) in foldVSelectToSignBitSplatMask()
11563 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
11568 // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | freeze(N2) in foldVSelectToSignBitSplatMask()
11571 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
11578 // (Cond0 s< -0) ? 0 : N2 --> ~(Cond0 s>> BW-1) & freeze(N2) in foldVSelectToSignBitSplatMask()
11582 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
11590 // (Cond0 s> -1) ? -1 : N2 --> ~(Cond0 s>> BW-1) | freeze(N2) in foldVSelectToSignBitSplatMask()
11596 SDValue N0 = N->getOperand(0); in visitSELECT()
11597 SDValue N1 = N->getOperand(1); in visitSELECT()
11598 SDValue N2 = N->getOperand(2); in visitSELECT()
11599 EVT VT = N->getValueType(0); in visitSELECT()
11602 SDNodeFlags Flags = N->getFlags(); in visitSELECT()
11610 // select (not Cond), N1, N2 -> select Cond, N2, N1 in visitSELECT()
11613 SelectOp->setFlags(Flags); in visitSELECT()
11636 // -> select Cond0, (select Cond1, X, Y), Y in visitSELECT()
11637 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { in visitSELECT()
11638 SDValue Cond0 = N0->getOperand(0); in visitSELECT()
11639 SDValue Cond1 = N0->getOperand(1); in visitSELECT()
11649 // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y) in visitSELECT()
11650 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { in visitSELECT()
11651 SDValue Cond0 = N0->getOperand(0); in visitSELECT()
11652 SDValue Cond1 = N0->getOperand(1); in visitSELECT()
11663 // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y in visitSELECT()
11664 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) { in visitSELECT()
11665 SDValue N1_0 = N1->getOperand(0); in visitSELECT()
11666 SDValue N1_1 = N1->getOperand(1); in visitSELECT()
11667 SDValue N1_2 = N1->getOperand(2); in visitSELECT()
11682 // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y in visitSELECT()
11683 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) { in visitSELECT()
11684 SDValue N2_0 = N2->getOperand(0); in visitSELECT()
11685 SDValue N2_1 = N2->getOperand(1); in visitSELECT()
11686 SDValue N2_2 = N2->getOperand(2); in visitSELECT()
11705 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSELECT()
11707 // select (fcmp lt x, y), x, y -> fminnum x, y in visitSELECT()
11708 // select (fcmp gt x, y), x, y -> fmaxnum x, y in visitSELECT()
11717 // This is conservatively limited to pre-legal-operations to give targets in visitSELECT()
11726 if (C && NotC && C->getAPIntValue() == ~NotC->getAPIntValue()) { in visitSELECT()
11727 // select (setcc Cond0, ~C, ugt), -1, (add Cond0, C) --> in visitSELECT()
11728 // uaddo Cond0, C; select uaddo.1, -1, uaddo.0 in visitSELECT()
11733 // %r = select %c, -1, %a in visitSELECT()
11738 // %r = select %u1, -1, %u0 in visitSELECT()
11750 Flags = N0->getFlags(); in visitSELECT()
11753 SelectNode->setFlags(Flags); in visitSELECT()
11775 SDValue Cond = N->getOperand(0); in ConvertSelectToConcatVector()
11776 SDValue LHS = N->getOperand(1); in ConvertSelectToConcatVector()
11777 SDValue RHS = N->getOperand(2); in ConvertSelectToConcatVector()
11778 EVT VT = N->getValueType(0); in ConvertSelectToConcatVector()
11786 if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2) in ConvertSelectToConcatVector()
11793 // length of the BV and see if all the non-undef nodes are the same. in ConvertSelectToConcatVector()
11796 if (Cond->getOperand(i)->isUndef()) in ConvertSelectToConcatVector()
11801 else if (Cond->getOperand(i).getNode() != BottomHalf) in ConvertSelectToConcatVector()
11808 if (Cond->getOperand(i)->isUndef()) in ConvertSelectToConcatVector()
11813 else if (Cond->getOperand(i).getNode() != TopHalf) in ConvertSelectToConcatVector()
11822 BottomHalf->isZero() ? RHS->getOperand(0) : LHS->getOperand(0), in ConvertSelectToConcatVector()
11823 TopHalf->isZero() ? RHS->getOperand(1) : LHS->getOperand(1)); in ConvertSelectToConcatVector()
11895 SDValue Mask = MSC->getMask(); in visitVPSCATTER()
11896 SDValue Chain = MSC->getChain(); in visitVPSCATTER()
11897 SDValue Index = MSC->getIndex(); in visitVPSCATTER()
11898 SDValue Scale = MSC->getScale(); in visitVPSCATTER() local
11899 SDValue StoreVal = MSC->getValue(); in visitVPSCATTER()
11900 SDValue BasePtr = MSC->getBasePtr(); in visitVPSCATTER()
11901 SDValue VL = MSC->getVectorLength(); in visitVPSCATTER()
11902 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitVPSCATTER()
11909 if (refineUniformBase(BasePtr, Index, MSC->isIndexScaled(), DAG, DL)) { in visitVPSCATTER()
11910 SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL}; in visitVPSCATTER()
11911 return DAG.getScatterVP(DAG.getVTList(MVT::Other), MSC->getMemoryVT(), in visitVPSCATTER()
11912 DL, Ops, MSC->getMemOperand(), IndexType); in visitVPSCATTER()
11916 SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL}; in visitVPSCATTER()
11917 return DAG.getScatterVP(DAG.getVTList(MVT::Other), MSC->getMemoryVT(), in visitVPSCATTER()
11918 DL, Ops, MSC->getMemOperand(), IndexType); in visitVPSCATTER()
11926 SDValue Mask = MSC->getMask(); in visitMSCATTER()
11927 SDValue Chain = MSC->getChain(); in visitMSCATTER()
11928 SDValue Index = MSC->getIndex(); in visitMSCATTER()
11929 SDValue Scale = MSC->getScale(); in visitMSCATTER() local
11930 SDValue StoreVal = MSC->getValue(); in visitMSCATTER()
11931 SDValue BasePtr = MSC->getBasePtr(); in visitMSCATTER()
11932 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitMSCATTER()
11939 if (refineUniformBase(BasePtr, Index, MSC->isIndexScaled(), DAG, DL)) { in visitMSCATTER()
11940 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale}; in visitMSCATTER()
11941 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(), in visitMSCATTER()
11942 DL, Ops, MSC->getMemOperand(), IndexType, in visitMSCATTER()
11943 MSC->isTruncatingStore()); in visitMSCATTER()
11947 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale}; in visitMSCATTER()
11948 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(), in visitMSCATTER()
11949 DL, Ops, MSC->getMemOperand(), IndexType, in visitMSCATTER()
11950 MSC->isTruncatingStore()); in visitMSCATTER()
11958 SDValue Mask = MST->getMask(); in visitMSTORE()
11959 SDValue Chain = MST->getChain(); in visitMSTORE()
11960 SDValue Value = MST->getValue(); in visitMSTORE()
11961 SDValue Ptr = MST->getBasePtr(); in visitMSTORE()
11970 if (MST->isUnindexed() && MST->isSimple() && MST1->isUnindexed() && in visitMSTORE()
11971 MST1->isSimple() && MST1->getBasePtr() == Ptr && in visitMSTORE()
11972 !MST->getBasePtr().isUndef() && in visitMSTORE()
11973 ((Mask == MST1->getMask() && MST->getMemoryVT().getStoreSize() == in visitMSTORE()
11974 MST1->getMemoryVT().getStoreSize()) || in visitMSTORE()
11976 TypeSize::isKnownLE(MST1->getMemoryVT().getStoreSize(), in visitMSTORE()
11977 MST->getMemoryVT().getStoreSize())) { in visitMSTORE()
11978 CombineTo(MST1, MST1->getChain()); in visitMSTORE()
11979 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
11987 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() && in visitMSTORE()
11988 !MST->isCompressingStore() && !MST->isTruncatingStore()) in visitMSTORE()
11989 return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(), in visitMSTORE()
11990 MST->getBasePtr(), MST->getPointerInfo(), in visitMSTORE()
11991 MST->getOriginalAlign(), in visitMSTORE()
11992 MST->getMemOperand()->getFlags(), MST->getAAInfo()); in visitMSTORE()
11998 if (MST->isTruncatingStore() && MST->isUnindexed() && in visitMSTORE()
12001 !cast<ConstantSDNode>(Value)->isOpaque())) { in visitMSTORE()
12004 MST->getMemoryVT().getScalarSizeInBits()); in visitMSTORE()
12009 // Re-visit the store if anything changed and the store hasn't been merged in visitMSTORE()
12011 // node back to the worklist if necessary, but we also need to re-visit in visitMSTORE()
12013 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
12023 if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() && in visitMSTORE()
12024 MST->isUnindexed() && !MST->isCompressingStore() && in visitMSTORE()
12026 MST->getMemoryVT(), LegalOperations)) { in visitMSTORE()
12027 auto Mask = TLI.promoteTargetBoolean(DAG, MST->getMask(), in visitMSTORE()
12030 MST->getOffset(), Mask, MST->getMemoryVT(), in visitMSTORE()
12031 MST->getMemOperand(), MST->getAddressingMode(), in visitMSTORE()
12040 EVT EltVT = SST->getValue().getValueType().getVectorElementType(); in visitVP_STRIDED_STORE()
12041 // Combine strided stores with unit-stride to a regular VP store. in visitVP_STRIDED_STORE()
12042 if (auto *CStride = dyn_cast<ConstantSDNode>(SST->getStride()); in visitVP_STRIDED_STORE()
12043 CStride && CStride->getZExtValue() == EltVT.getStoreSize()) { in visitVP_STRIDED_STORE()
12044 return DAG.getStoreVP(SST->getChain(), SDLoc(N), SST->getValue(), in visitVP_STRIDED_STORE()
12045 SST->getBasePtr(), SST->getOffset(), SST->getMask(), in visitVP_STRIDED_STORE()
12046 SST->getVectorLength(), SST->getMemoryVT(), in visitVP_STRIDED_STORE()
12047 SST->getMemOperand(), SST->getAddressingMode(), in visitVP_STRIDED_STORE()
12048 SST->isTruncatingStore(), SST->isCompressingStore()); in visitVP_STRIDED_STORE()
12055 SDValue Vec = N->getOperand(0); in visitVECTOR_COMPRESS()
12056 SDValue Mask = N->getOperand(1); in visitVECTOR_COMPRESS()
12057 SDValue Passthru = N->getOperand(2); in visitVECTOR_COMPRESS()
12104 SDValue Mask = MGT->getMask(); in visitVPGATHER()
12105 SDValue Chain = MGT->getChain(); in visitVPGATHER()
12106 SDValue Index = MGT->getIndex(); in visitVPGATHER()
12107 SDValue Scale = MGT->getScale(); in visitVPGATHER() local
12108 SDValue BasePtr = MGT->getBasePtr(); in visitVPGATHER()
12109 SDValue VL = MGT->getVectorLength(); in visitVPGATHER()
12110 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitVPGATHER()
12113 if (refineUniformBase(BasePtr, Index, MGT->isIndexScaled(), DAG, DL)) { in visitVPGATHER()
12114 SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL}; in visitVPGATHER()
12116 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL, in visitVPGATHER()
12117 Ops, MGT->getMemOperand(), IndexType); in visitVPGATHER()
12120 if (refineIndexType(Index, IndexType, N->getValueType(0), DAG)) { in visitVPGATHER()
12121 SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL}; in visitVPGATHER()
12123 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL, in visitVPGATHER()
12124 Ops, MGT->getMemOperand(), IndexType); in visitVPGATHER()
12132 SDValue Mask = MGT->getMask(); in visitMGATHER()
12133 SDValue Chain = MGT->getChain(); in visitMGATHER()
12134 SDValue Index = MGT->getIndex(); in visitMGATHER()
12135 SDValue Scale = MGT->getScale(); in visitMGATHER() local
12136 SDValue PassThru = MGT->getPassThru(); in visitMGATHER()
12137 SDValue BasePtr = MGT->getBasePtr(); in visitMGATHER()
12138 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitMGATHER()
12143 return CombineTo(N, PassThru, MGT->getChain()); in visitMGATHER()
12145 if (refineUniformBase(BasePtr, Index, MGT->isIndexScaled(), DAG, DL)) { in visitMGATHER()
12146 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale}; in visitMGATHER()
12148 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL, in visitMGATHER()
12149 Ops, MGT->getMemOperand(), IndexType, MGT->getExtensionType()); in visitMGATHER()
12152 if (refineIndexType(Index, IndexType, N->getValueType(0), DAG)) { in visitMGATHER()
12153 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale}; in visitMGATHER()
12155 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL, in visitMGATHER()
12156 Ops, MGT->getMemOperand(), IndexType, MGT->getExtensionType()); in visitMGATHER()
12164 SDValue Mask = MLD->getMask(); in visitMLOAD()
12169 return CombineTo(N, MLD->getPassThru(), MLD->getChain()); in visitMLOAD()
12173 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() && in visitMLOAD()
12174 !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) { in visitMLOAD()
12176 N->getValueType(0), SDLoc(N), MLD->getChain(), MLD->getBasePtr(), in visitMLOAD()
12177 MLD->getPointerInfo(), MLD->getOriginalAlign(), in visitMLOAD()
12178 MLD->getMemOperand()->getFlags(), MLD->getAAInfo(), MLD->getRanges()); in visitMLOAD()
12191 EVT EltVT = SLD->getValueType(0).getVectorElementType(); in visitVP_STRIDED_LOAD()
12192 // Combine strided loads with unit-stride to a regular VP load. in visitVP_STRIDED_LOAD()
12193 if (auto *CStride = dyn_cast<ConstantSDNode>(SLD->getStride()); in visitVP_STRIDED_LOAD()
12194 CStride && CStride->getZExtValue() == EltVT.getStoreSize()) { in visitVP_STRIDED_LOAD()
12196 SLD->getAddressingMode(), SLD->getExtensionType(), SLD->getValueType(0), in visitVP_STRIDED_LOAD()
12197 SDLoc(N), SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(), in visitVP_STRIDED_LOAD()
12198 SLD->getMask(), SLD->getVectorLength(), SLD->getMemoryVT(), in visitVP_STRIDED_LOAD()
12199 SLD->getMemOperand(), SLD->isExpandingLoad()); in visitVP_STRIDED_LOAD()
12208 SDValue Cond = N->getOperand(0); in foldVSelectOfConstants()
12209 SDValue N1 = N->getOperand(1); in foldVSelectOfConstants()
12210 SDValue N2 = N->getOperand(2); in foldVSelectOfConstants()
12211 EVT VT = N->getValueType(0); in foldVSelectOfConstants()
12235 const APInt &C1 = N1Elt->getAsAPIntVal(); in foldVSelectOfConstants()
12236 const APInt &C2 = N2Elt->getAsAPIntVal(); in foldVSelectOfConstants()
12239 if (C1 != C2 - 1) in foldVSelectOfConstants()
12243 // Further simplifications for the extra-special cases where the constants are in foldVSelectOfConstants()
12244 // all 0 or all -1 should be implemented as folds of these patterns. in foldVSelectOfConstants()
12247 // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C in foldVSelectOfConstants()
12248 // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C in foldVSelectOfConstants()
12254 // select Cond, Pow2C, 0 --> (zext Cond) << log2(Pow2C) in foldVSelectOfConstants()
12266 // The general case for select-of-constants: in foldVSelectOfConstants()
12267 // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2 in foldVSelectOfConstants()
12269 // leave that to a machine-specific pass. in foldVSelectOfConstants()
12274 SDValue N0 = N->getOperand(0); in visitVP_SELECT()
12275 SDValue N1 = N->getOperand(1); in visitVP_SELECT()
12276 SDValue N2 = N->getOperand(2); in visitVP_SELECT()
12289 SDValue N0 = N->getOperand(0); in visitVSELECT()
12290 SDValue N1 = N->getOperand(1); in visitVSELECT()
12291 SDValue N2 = N->getOperand(2); in visitVSELECT()
12292 EVT VT = N->getValueType(0); in visitVSELECT()
12301 // vselect (not Cond), N1, N2 -> vselect Cond, N2, N1 in visitVSELECT()
12305 // select (sext m), (add X, C), X --> (add X, (and C, (sext m)))) in visitVSELECT()
12306 if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N2 && N1->hasOneUse() && in visitVSELECT()
12317 // vselect (setg[te] X, 0), X, -X -> in visitVSELECT()
12318 // vselect (setgt X, -1), X, -X -> in visitVSELECT()
12319 // vselect (setl[te] X, 0), -X, X -> in visitVSELECT()
12320 // Y = sra (X, size(X)-1); xor (add (X, Y), Y) in visitVSELECT()
12323 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitVSELECT()
12341 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, DL)); in visitVSELECT()
12348 // vselect x, y (fcmp lt x, y) -> fminnum x, y in visitVSELECT()
12349 // vselect x, y (fcmp gt x, y) -> fmaxnum x, y in visitVSELECT()
12355 isLegalToCombineMinNumMaxNum(DAG, LHS, RHS, N->getFlags(), TLI)) { in visitVSELECT()
12368 // TODO: This could be extended to handle non-loading patterns, but that in visitVSELECT()
12384 // vselect (ext (setcc load(X), C)), N1, N2 --> in visitVSELECT()
12396 // (vselect (setcc a, b, set?gt), (sub a, b), (sub b, a)) --> (abd? a, b) in visitVSELECT()
12397 // (vselect (setcc a, b, set?ge), (sub a, b), (sub b, a)) --> (abd? a, b) in visitVSELECT()
12398 // (vselect (setcc a, b, set?lt), (sub b, a), (sub a, b)) --> (abd? a, b) in visitVSELECT()
12399 // (vselect (setcc a, b, set?le), (sub b, a), (sub a, b)) --> (abd? a, b) in visitVSELECT()
12451 // x <= x+y ? x+y : ~0 --> uaddsat x, y in visitVSELECT()
12452 // x+y >= x ? x+y : ~0 --> uaddsat x, y in visitVSELECT()
12463 // x >= ~C ? x+C : ~0 --> uaddsat x, C in visitVSELECT()
12465 return Cond->getAPIntValue() == ~Op->getAPIntValue(); in visitVSELECT()
12487 // zext(x) >= y ? trunc(zext(x) - y) : 0 in visitVSELECT()
12488 // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit))) in visitVSELECT()
12489 // zext(x) > y ? trunc(zext(x) - y) : 0 in visitVSELECT()
12490 // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit))) in visitVSELECT()
12508 // x >= y ? x-y : 0 --> usubsat x, y in visitVSELECT()
12509 // x > y ? x-y : 0 --> usubsat x, y in visitVSELECT()
12520 // x > C-1 ? x+-C : 0 --> usubsat x, C in visitVSELECT()
12524 Cond->getAPIntValue() == (-Op->getAPIntValue() - 1)); in visitVSELECT()
12537 // x s< 0 ? x^C : 0 --> usubsat x, C in visitVSELECT()
12558 // Fold (vselect all_ones, N1, N2) -> N1 in visitVSELECT()
12561 // Fold (vselect all_zeros, N1, N2) -> N2 in visitVSELECT()
12589 SDValue N0 = N->getOperand(0); in visitSELECT_CC()
12590 SDValue N1 = N->getOperand(1); in visitSELECT_CC()
12591 SDValue N2 = N->getOperand(2); in visitSELECT_CC()
12592 SDValue N3 = N->getOperand(3); in visitSELECT_CC()
12593 SDValue N4 = N->getOperand(4); in visitSELECT_CC()
12594 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get(); in visitSELECT_CC()
12597 // fold select_cc lhs, rhs, x, x, cc -> x in visitSELECT_CC()
12601 // select_cc bool, 0, x, y, seteq -> select bool, y, x in visitSELECT_CC()
12611 // cond always true -> true val in visitSELECT_CC()
12612 // cond always false -> false val in visitSELECT_CC()
12614 return SCCC->isZero() ? N3 : N2; in visitSELECT_CC()
12618 if (SCC->isUndef()) in visitSELECT_CC()
12626 SelectOp->setFlags(SCC->getFlags()); in visitSELECT_CC()
12644 N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND; in visitSETCC()
12646 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in visitSETCC()
12647 EVT VT = N->getValueType(0); in visitSETCC()
12648 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in visitSETCC()
12714 auto GetAPIntValue = [](SDValue Op) -> std::optional<APInt> { in visitSETCC()
12719 return CNode->getAPIntValue(); in visitSETCC()
12728 if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(NumBits)) { in visitSETCC()
12736 CanTransform &= (*ShiftCAmt + AndCMask->popcount()) == NumBits; in visitSETCC()
12739 ShiftOpc == ISD::SHL ? (~*AndCMask).isMask() : AndCMask->isMask(); in visitSETCC()
12744 OpVT, ShiftOpc, ShiftCAmt->isPowerOf2(), *ShiftCAmt, AndCMask); in visitSETCC()
12756 NumBits - ShiftCAmt->getZExtValue()) in visitSETCC()
12758 NumBits - ShiftCAmt->getZExtValue()); in visitSETCC()
12775 SDValue LHS = N->getOperand(0); in visitSETCCCARRY()
12776 SDValue RHS = N->getOperand(1); in visitSETCCCARRY()
12777 SDValue Carry = N->getOperand(2); in visitSETCCCARRY()
12778 SDValue Cond = N->getOperand(3); in visitSETCCCARRY()
12782 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond); in visitSETCCCARRY()
12802 ISD::LoadExtType LoadExt = Load->getExtensionType(); in isCompatibleLoad()
12816 /// (sext (select c, load x, load y)) -> (select c, sextload x, sextload y)
12817 /// (zext (select c, load x, load y)) -> (select c, zextload x, zextload y)
12818 /// (aext (select c, load x, load y)) -> (select c, extload x, extload y)
12824 unsigned Opcode = N->getOpcode(); in tryToFoldExtendSelectLoad()
12825 SDValue N0 = N->getOperand(0); in tryToFoldExtendSelectLoad()
12826 EVT VT = N->getValueType(0); in tryToFoldExtendSelectLoad()
12831 if (!(N0->getOpcode() == ISD::SELECT || N0->getOpcode() == ISD::VSELECT) || in tryToFoldExtendSelectLoad()
12835 SDValue Op1 = N0->getOperand(1); in tryToFoldExtendSelectLoad()
12836 SDValue Op2 = N0->getOperand(2); in tryToFoldExtendSelectLoad()
12850 if (!TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load1->getMemoryVT()) || in tryToFoldExtendSelectLoad()
12851 !TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load2->getMemoryVT()) || in tryToFoldExtendSelectLoad()
12852 (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes && in tryToFoldExtendSelectLoad()
12858 return DAG.getSelect(DL, VT, N0->getOperand(0), Ext1, Ext2); in tryToFoldExtendSelectLoad()
12870 unsigned Opcode = N->getOpcode(); in tryToFoldExtendOfConstant()
12871 SDValue N0 = N->getOperand(0); in tryToFoldExtendOfConstant()
12872 EVT VT = N->getValueType(0); in tryToFoldExtendOfConstant()
12877 // fold (sext c1) -> c1 in tryToFoldExtendOfConstant()
12878 // fold (zext c1) -> c1 in tryToFoldExtendOfConstant()
12879 // fold (aext c1) -> c1 in tryToFoldExtendOfConstant()
12883 // fold (sext (select cond, c1, c2)) -> (select cond, sext c1, sext c2) in tryToFoldExtendOfConstant()
12884 // fold (zext (select cond, c1, c2)) -> (select cond, zext c1, zext c2) in tryToFoldExtendOfConstant()
12885 // fold (aext (select cond, c1, c2)) -> (select cond, sext c1, sext c2) in tryToFoldExtendOfConstant()
12886 if (N0->getOpcode() == ISD::SELECT) { in tryToFoldExtendOfConstant()
12887 SDValue Op1 = N0->getOperand(1); in tryToFoldExtendOfConstant()
12888 SDValue Op2 = N0->getOperand(2); in tryToFoldExtendOfConstant()
12894 // t1: i8 = select t0, Constant:i8<-1>, Constant:i8<0> in tryToFoldExtendOfConstant()
12896 // --> in tryToFoldExtendOfConstant()
12897 // t3: i64 = select t0, Constant:i64<-1>, Constant:i64<0> in tryToFoldExtendOfConstant()
12898 // --> in tryToFoldExtendOfConstant()
12903 return DAG.getSelect(DL, VT, N0->getOperand(0), in tryToFoldExtendOfConstant()
12909 // fold (sext (build_vector AllConstants) -> (build_vector AllConstants) in tryToFoldExtendOfConstant()
12910 // fold (zext (build_vector AllConstants) -> (build_vector AllConstants) in tryToFoldExtendOfConstant()
12911 // fold (aext (build_vector AllConstants) -> (build_vector AllConstants) in tryToFoldExtendOfConstant()
12919 unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits(); in tryToFoldExtendOfConstant()
12936 APInt C = Op->getAsAPIntVal().zextOrTrunc(EVTBits); in tryToFoldExtendOfConstant()
12946 // ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
12947 // "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
12956 for (SDNode::use_iterator UI = N0->use_begin(), UE = N0->use_end(); UI != UE; in ExtendUsesToFormExtLoad()
12964 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) { in ExtendUsesToFormExtLoad()
12965 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in ExtendUsesToFormExtLoad()
12971 SDValue UseOp = User->getOperand(i); in ExtendUsesToFormExtLoad()
12986 // Remember if this value is live-out. in ExtendUsesToFormExtLoad()
12987 if (User->getOpcode() == ISD::CopyToReg) in ExtendUsesToFormExtLoad()
12993 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); in ExtendUsesToFormExtLoad()
12996 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) { in ExtendUsesToFormExtLoad()
13018 SDValue SOp = SetCC->getOperand(j); in ExtendSetCCUses()
13022 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp)); in ExtendSetCCUses()
13025 Ops.push_back(SetCC->getOperand(2)); in ExtendSetCCUses()
13026 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops)); in ExtendSetCCUses()
13032 SDValue N0 = N->getOperand(0); in CombineExtLoad()
13033 EVT DstVT = N->getValueType(0); in CombineExtLoad()
13036 assert((N->getOpcode() == ISD::SIGN_EXTEND || in CombineExtLoad()
13037 N->getOpcode() == ISD::ZERO_EXTEND) && in CombineExtLoad()
13054 // All legal types, and illegal non-vector types, are handled elsewhere. in CombineExtLoad()
13057 if (N0->getOpcode() != ISD::LOAD) in CombineExtLoad()
13063 !N0.hasOneUse() || !LN0->isSimple() || in CombineExtLoad()
13069 if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI)) in CombineExtLoad()
13073 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in CombineExtLoad()
13096 SDValue BasePtr = LN0->getBasePtr(); in CombineExtLoad()
13101 DAG.getExtLoad(ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), in CombineExtLoad()
13102 BasePtr, LN0->getPointerInfo().getWithOffset(Offset), in CombineExtLoad()
13103 SplitSrcVT, LN0->getOriginalAlign(), in CombineExtLoad()
13104 LN0->getMemOperand()->getFlags(), LN0->getAAInfo()); in CombineExtLoad()
13124 ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode()); in CombineExtLoad()
13129 // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
13132 assert(N->getOpcode() == ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
13133 EVT VT = N->getValueType(0); in CombineZExtLogicopShiftLoad()
13134 EVT OrigVT = N->getOperand(0).getValueType(); in CombineZExtLogicopShiftLoad()
13139 SDValue N0 = N->getOperand(0); in CombineZExtLogicopShiftLoad()
13146 SDValue N1 = N0->getOperand(0); in CombineZExtLogicopShiftLoad()
13156 EVT MemVT = Load->getMemoryVT(); in CombineZExtLogicopShiftLoad()
13158 Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed()) in CombineZExtLogicopShiftLoad()
13177 Load->getChain(), Load->getBasePtr(), in CombineZExtLogicopShiftLoad()
13178 Load->getMemoryVT(), Load->getMemOperand()); in CombineZExtLogicopShiftLoad()
13195 Load->getValueType(0), ExtLoad); in CombineZExtLogicopShiftLoad()
13210 unsigned CastOpcode = Cast->getOpcode(); in matchVSelectOpSizesWithSetCC()
13217 // obfuscated by target-specific operations after legalization. Do not create in matchVSelectOpSizesWithSetCC()
13219 EVT VT = Cast->getValueType(0); in matchVSelectOpSizesWithSetCC()
13223 SDValue VSel = Cast->getOperand(0); in matchVSelectOpSizesWithSetCC()
13234 // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B) in matchVSelectOpSizesWithSetCC()
13241 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
13242 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
13250 // fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
13251 // fold ([s|z]ext ( extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
13264 EVT MemVT = LN0->getMemoryVT(); in tryToFoldExtOfExtload()
13265 if ((LegalOperations || !LN0->isSimple() || in tryToFoldExtOfExtload()
13271 DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfExtload()
13272 LN0->getBasePtr(), MemVT, LN0->getMemOperand()); in tryToFoldExtOfExtload()
13275 if (LN0->use_empty()) in tryToFoldExtOfExtload()
13280 // fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
13297 for (SDNode *User : N0->uses()) { in tryToFoldExtOfLoad()
13298 if (User->getOpcode() == ISD::SETCC) { in tryToFoldExtOfLoad()
13299 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in tryToFoldExtOfLoad()
13313 !cast<LoadSDNode>(N0)->isSimple()) && in tryToFoldExtOfLoad()
13327 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfLoad()
13328 LN0->getBasePtr(), N0.getValueType(), in tryToFoldExtOfLoad()
13329 LN0->getMemOperand()); in tryToFoldExtOfLoad()
13353 if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD) in tryToFoldExtOfMaskedLoad()
13356 if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->isSimple()) && in tryToFoldExtOfMaskedLoad()
13357 !TLI.isLoadExtLegalOrCustom(ExtLoadType, VT, Ld->getValueType(0))) in tryToFoldExtOfMaskedLoad()
13364 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru()); in tryToFoldExtOfMaskedLoad()
13366 VT, dl, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(), in tryToFoldExtOfMaskedLoad()
13367 PassThru, Ld->getMemoryVT(), Ld->getMemOperand(), Ld->getAddressingMode(), in tryToFoldExtOfMaskedLoad()
13368 ExtLoadType, Ld->isExpandingLoad()); in tryToFoldExtOfMaskedLoad()
13373 // fold ([s|z]ext (atomic_load)) -> ([s|z]ext (truncate ([s|z]ext atomic_load)))
13379 if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD) in tryToFoldExtOfAtomicLoad()
13381 EVT MemoryVT = ALoad->getMemoryVT(); in tryToFoldExtOfAtomicLoad()
13385 ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType(); in tryToFoldExtOfAtomicLoad()
13390 EVT OrigVT = ALoad->getValueType(0); in tryToFoldExtOfAtomicLoad()
13393 ISD::ATOMIC_LOAD, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(), in tryToFoldExtOfAtomicLoad()
13394 ALoad->getBasePtr(), ALoad->getMemOperand())); in tryToFoldExtOfAtomicLoad()
13395 NewALoad->setExtensionType(ExtLoadType); in tryToFoldExtOfAtomicLoad()
13406 assert((N->getOpcode() == ISD::SIGN_EXTEND || in foldExtendedSignBitTest()
13407 N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext"); in foldExtendedSignBitTest()
13409 SDValue SetCC = N->getOperand(0); in foldExtendedSignBitTest()
13416 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get(); in foldExtendedSignBitTest()
13417 EVT VT = N->getValueType(0); in foldExtendedSignBitTest()
13424 // sext i1 (setgt iN X, -1) --> sra (not X), (N - 1) in foldExtendedSignBitTest()
13425 // zext i1 (setgt iN X, -1) --> srl (not X), (N - 1) in foldExtendedSignBitTest()
13427 unsigned ShCt = VT.getSizeInBits() - 1; in foldExtendedSignBitTest()
13433 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL; in foldExtendedSignBitTest()
13441 SDValue N0 = N->getOperand(0); in foldSextSetcc()
13447 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldSextSetcc()
13448 EVT VT = N->getValueType(0); in foldSextSetcc()
13452 // Propagate fast-math-flags. in foldSextSetcc()
13453 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags()); in foldSextSetcc()
13496 // Match a simple, non-extended load that can be converted to a in foldSextSetcc()
13497 // legal {z/s}ext-load. in foldSextSetcc()
13498 // TODO: Allow widening of an existing {z/s}ext-load? in foldSextSetcc()
13501 cast<LoadSDNode>(V)->isSimple() && in foldSextSetcc()
13505 // Non-chain users of this value must either be the setcc in this in foldSextSetcc()
13506 // sequence or extends that can be folded into the new {z/s}ext-load. in foldSextSetcc()
13507 for (SDNode::use_iterator UI = V->use_begin(), UE = V->use_end(); in foldSextSetcc()
13516 if (User->getOpcode() != ExtOpcode || User->getValueType(0) != VT) in foldSextSetcc()
13530 // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0) in foldSextSetcc()
13531 // Here, T can be 1 or -1, depending on the type of the setcc and in foldSextSetcc()
13538 // sext(i1 1), that is, -1. in foldSextSetcc()
13566 SDValue N0 = N->getOperand(0); in visitSIGN_EXTEND()
13567 EVT VT = N->getValueType(0); in visitSIGN_EXTEND()
13581 // fold (sext (sext x)) -> (sext x) in visitSIGN_EXTEND()
13582 // fold (sext (aext x)) -> (sext x) in visitSIGN_EXTEND()
13586 // fold (sext (aext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x) in visitSIGN_EXTEND()
13587 // fold (sext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x) in visitSIGN_EXTEND()
13593 // fold (sext (sext_inreg x)) -> (sext (trunc x)) in visitSIGN_EXTEND()
13596 EVT ExtVT = cast<VTSDNode>(N0->getOperand(1))->getVT(); in visitSIGN_EXTEND()
13605 // fold (sext (truncate (load x))) -> (sext (smaller load x)) in visitSIGN_EXTEND()
13606 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n))) in visitSIGN_EXTEND()
13626 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign in visitSIGN_EXTEND()
13628 if (NumSignBits > DestBits-MidBits) in visitSIGN_EXTEND()
13631 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign in visitSIGN_EXTEND()
13633 if (NumSignBits > OpBits-MidBits) in visitSIGN_EXTEND()
13636 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign in visitSIGN_EXTEND()
13638 if (NumSignBits > OpBits-MidBits) in visitSIGN_EXTEND()
13642 // fold (sext (truncate x)) -> (sextinreg x). in visitSIGN_EXTEND()
13680 // fold (sext (and/or/xor (load x), cst)) -> in visitSIGN_EXTEND()
13687 EVT MemVT = LN00->getMemoryVT(); in visitSIGN_EXTEND()
13689 LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) { in visitSIGN_EXTEND()
13695 LN00->getChain(), LN00->getBasePtr(), in visitSIGN_EXTEND()
13696 LN00->getMemoryVT(), in visitSIGN_EXTEND()
13697 LN00->getMemOperand()); in visitSIGN_EXTEND()
13715 LN00->getValueType(0), ExtLoad); in visitSIGN_EXTEND()
13729 // fold (sext x) -> (zext x) if the sign bit is known zero. in visitSIGN_EXTEND()
13742 // sext i32 (0 - (zext i8 X to i32)) to i64 --> 0 - (zext i8 X to i64) in visitSIGN_EXTEND()
13751 // sext i32 ((zext i8 X to i32) + (-1)) to i64 --> (zext i8 X to i64) + (-1) in visitSIGN_EXTEND()
13760 // fold sext (not i1 X) -> add (zext i1 X), -1 in visitSIGN_EXTEND()
13767 // Returning N0 is a form of in-visit replacement that may have in visitSIGN_EXTEND()
13789 /// Given an extending node with a pop-count operand, if the target does not
13790 /// support a pop-count in the narrow source type but does support it in the
13791 /// destination type, widen the pop-count to the destination type.
13793 assert((Extend->getOpcode() == ISD::ZERO_EXTEND || in widenCtPop()
13794 Extend->getOpcode() == ISD::ANY_EXTEND) && in widenCtPop()
13797 SDValue CtPop = Extend->getOperand(0); in widenCtPop()
13801 EVT VT = Extend->getValueType(0); in widenCtPop()
13807 // zext (ctpop X) --> ctpop (zext X) in widenCtPop()
13815 assert(Extend->getOpcode() == ISD::ZERO_EXTEND && "Expected zero extend."); in widenAbs()
13817 EVT VT = Extend->getValueType(0); in widenAbs()
13821 SDValue Abs = Extend->getOperand(0); in widenAbs()
13840 SDValue N0 = N->getOperand(0); in visitZERO_EXTEND()
13841 EVT VT = N->getValueType(0); in visitZERO_EXTEND()
13855 // fold (zext (zext x)) -> (zext x) in visitZERO_EXTEND()
13856 // fold (zext (aext x)) -> (zext x) in visitZERO_EXTEND()
13860 Flags.setNonNeg(N0->getFlags().hasNonNeg()); in visitZERO_EXTEND()
13864 // fold (zext (aext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x) in visitZERO_EXTEND()
13865 // fold (zext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x) in visitZERO_EXTEND()
13870 // fold (zext (truncate x)) -> (zext x) or in visitZERO_EXTEND()
13871 // (zext (truncate x)) -> (truncate x) in visitZERO_EXTEND()
13891 // fold (zext (truncate x)) -> (and x, mask) in visitZERO_EXTEND()
13893 // fold (zext (truncate (load x))) -> (zext (smaller load x)) in visitZERO_EXTEND()
13894 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n))) in visitZERO_EXTEND()
13908 if (N->getFlags().hasNonNeg()) { in visitZERO_EXTEND()
13916 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign in visitZERO_EXTEND()
13918 if (NumSignBits > DestBits - MidBits) in visitZERO_EXTEND()
13921 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign in visitZERO_EXTEND()
13924 if (NumSignBits > OpBits - MidBits) in visitZERO_EXTEND()
13927 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign in visitZERO_EXTEND()
13929 if (NumSignBits > OpBits - MidBits) in visitZERO_EXTEND()
13935 // possibly over several sub-vectors. in visitZERO_EXTEND()
13960 // Fold (zext (and (trunc x), cst)) -> (and x, cst), in visitZERO_EXTEND()
13977 ISD::ZERO_EXTEND, N->getFlags().hasNonNeg())) in visitZERO_EXTEND()
13995 // fold (zext (and/or/xor (load x), cst)) -> in visitZERO_EXTEND()
14004 EVT MemVT = LN00->getMemoryVT(); in visitZERO_EXTEND()
14006 LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) { in visitZERO_EXTEND()
14012 EVT LoadResultTy = AndC->getValueType(0); in visitZERO_EXTEND()
14023 LN00->getChain(), LN00->getBasePtr(), in visitZERO_EXTEND()
14024 LN00->getMemoryVT(), in visitZERO_EXTEND()
14025 LN00->getMemOperand()); in visitZERO_EXTEND()
14043 LN00->getValueType(0), ExtLoad); in visitZERO_EXTEND()
14051 // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) -> in visitZERO_EXTEND()
14065 // Propagate fast-math-flags. in visitZERO_EXTEND()
14066 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags()); in visitZERO_EXTEND()
14081 // zext(setcc) -> zext_in_reg(vsetcc) for vectors. in visitZERO_EXTEND()
14098 // zext(setcc x,y,cc) -> zext(select x, y, true, false, cc) in visitZERO_EXTEND()
14105 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true)) in visitZERO_EXTEND()
14109 // (zext (shl (zext x), cst)) -> (shl (zext x), cst) in visitZERO_EXTEND()
14119 unsigned KnownZeroBits = ShVal.getValueSizeInBits() - in visitZERO_EXTEND()
14121 if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) { in visitZERO_EXTEND()
14127 ShAmtC->getAPIntValue().getZExtValue()); in visitZERO_EXTEND()
14158 if (N->getFlags().hasNonNeg() && !TLI.isZExtFree(N0.getValueType(), VT)) { in visitZERO_EXTEND()
14159 SDNode *CSENode = DAG.getNodeIfExists(ISD::SIGN_EXTEND, N->getVTList(), N0); in visitZERO_EXTEND()
14168 SDValue N0 = N->getOperand(0); in visitANY_EXTEND()
14169 EVT VT = N->getValueType(0); in visitANY_EXTEND()
14179 // fold (aext (aext x)) -> (aext x) in visitANY_EXTEND()
14180 // fold (aext (zext x)) -> (zext x) in visitANY_EXTEND()
14181 // fold (aext (sext x)) -> (sext x) in visitANY_EXTEND()
14186 Flags.setNonNeg(N0->getFlags().hasNonNeg()); in visitANY_EXTEND()
14190 // fold (aext (aext_extend_vector_inreg x)) -> (aext_extend_vector_inreg x) in visitANY_EXTEND()
14191 // fold (aext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x) in visitANY_EXTEND()
14192 // fold (aext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x) in visitANY_EXTEND()
14198 // fold (aext (truncate (load x))) -> (aext (smaller load x)) in visitANY_EXTEND()
14199 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n))) in visitANY_EXTEND()
14216 // Fold (aext (and (trunc x), cst)) -> (and x, cst) in visitANY_EXTEND()
14228 // fold (aext (load x)) -> (aext (truncate (extload x))) in visitANY_EXTEND()
14247 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(), in visitANY_EXTEND()
14248 LN0->getBasePtr(), N0.getValueType(), in visitANY_EXTEND()
14249 LN0->getMemOperand()); in visitANY_EXTEND()
14266 // fold (aext (zextload x)) -> (aext (truncate (zextload x))) in visitANY_EXTEND()
14267 // fold (aext (sextload x)) -> (aext (truncate (sextload x))) in visitANY_EXTEND()
14268 // fold (aext ( extload x)) -> (aext (truncate (extload x))) in visitANY_EXTEND()
14272 ISD::LoadExtType ExtType = LN0->getExtensionType(); in visitANY_EXTEND()
14273 EVT MemVT = LN0->getMemoryVT(); in visitANY_EXTEND()
14276 DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), LN0->getBasePtr(), in visitANY_EXTEND()
14277 MemVT, LN0->getMemOperand()); in visitANY_EXTEND()
14286 // Propagate fast-math-flags. in visitANY_EXTEND()
14287 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags()); in visitANY_EXTEND()
14290 // aext(setcc) -> vsetcc in visitANY_EXTEND()
14291 // aext(setcc) -> truncate(vsetcc) in visitANY_EXTEND()
14292 // aext(setcc) -> aext(vsetcc) in visitANY_EXTEND()
14306 cast<CondCodeSDNode>(N0.getOperand(2))->get()); in visitANY_EXTEND()
14314 cast<CondCodeSDNode>(N0.getOperand(2))->get()); in visitANY_EXTEND()
14318 // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc in visitANY_EXTEND()
14322 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true)) in visitANY_EXTEND()
14336 unsigned Opcode = N->getOpcode(); in visitAssertExt()
14337 SDValue N0 = N->getOperand(0); in visitAssertExt()
14338 SDValue N1 = N->getOperand(1); in visitAssertExt()
14339 EVT AssertVT = cast<VTSDNode>(N1)->getVT(); in visitAssertExt()
14341 // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt) in visitAssertExt()
14343 AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT()) in visitAssertExt()
14351 // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN in visitAssertExt()
14352 // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN in visitAssertExt()
14355 EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT(); in visitAssertExt()
14360 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
14370 EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT(); in visitAssertExt()
14375 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
14385 Align AL = cast<AssertAlignSDNode>(N)->getAlign(); in visitAssertAlign()
14386 SDValue N0 = N->getOperand(0); in visitAssertAlign()
14388 // Fold (assertalign (assertalign x, AL0), AL1) -> in visitAssertAlign()
14392 std::max(AL, AAN->getAlign())); in visitAssertAlign()
14425 unsigned Opc = N->getOpcode(); in reduceLoadWidth()
14428 SDValue N0 = N->getOperand(0); in reduceLoadWidth()
14429 EVT VT = N->getValueType(0); in reduceLoadWidth()
14442 // to indicate that the narrowed load should be left-shifted ShAmt bits to get in reduceLoadWidth()
14449 ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT(); in reduceLoadWidth()
14451 // Another special-case: SRL/SRA is basically zero/sign-extending a narrower in reduceLoadWidth()
14458 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)); in reduceLoadWidth()
14463 ShAmt = N1C->getZExtValue(); in reduceLoadWidth()
14464 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits(); in reduceLoadWidth()
14469 ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShAmt); in reduceLoadWidth()
14474 if ((LN->getExtensionType() == ISD::SEXTLOAD || in reduceLoadWidth()
14475 LN->getExtensionType() == ISD::ZEXTLOAD) && in reduceLoadWidth()
14476 LN->getExtensionType() != ExtType) in reduceLoadWidth()
14479 // An AND with a constant mask is the same as a truncate + zero-extend. in reduceLoadWidth()
14480 auto AndC = dyn_cast<ConstantSDNode>(N->getOperand(1)); in reduceLoadWidth()
14484 const APInt &Mask = AndC->getAPIntValue(); in reduceLoadWidth()
14506 // (undocumented) reasons. Maybe intent was to guard the AND-masking below in reduceLoadWidth()
14507 // check below? And maybe it could be non-profitable to do the transform in in reduceLoadWidth()
14523 ShAmt = SRL1C->getZExtValue(); in reduceLoadWidth()
14524 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits(); in reduceLoadWidth()
14528 // Because a SRL must be assumed to *need* to zero-extend the high bits in reduceLoadWidth()
14531 if (LN->getExtensionType() == ISD::SEXTLOAD) in reduceLoadWidth()
14537 // (i64 (truncate (i96 (srl (load x), 64)))) -> in reduceLoadWidth()
14539 if (ExtVT.getScalarSizeInBits() > MemoryWidth - ShAmt) { in reduceLoadWidth()
14545 ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShAmt); in reduceLoadWidth()
14550 SDNode *Mask = *(SRL->use_begin()); in reduceLoadWidth()
14551 if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND && in reduceLoadWidth()
14552 isa<ConstantSDNode>(Mask->getOperand(1))) { in reduceLoadWidth()
14554 const APInt& ShiftMask = Mask->getConstantOperandAPInt(1); in reduceLoadWidth()
14583 // (truncate (shl (load x), c))) -> (shl (narrow load x), c) in reduceLoadWidth()
14589 ShLeftAmt = N01->getZExtValue(); in reduceLoadWidth()
14601 if (!LN0->isSimple() || in reduceLoadWidth()
14607 LN0->getMemoryVT().getStoreSizeInBits().getFixedValue(); in reduceLoadWidth()
14609 return LVTStoreBits - EVTStoreBits - ShAmt; in reduceLoadWidth()
14623 LN0->getBasePtr(), TypeSize::getFixed(PtrOff), DL, Flags); in reduceLoadWidth()
14628 Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr, in reduceLoadWidth()
14629 LN0->getPointerInfo().getWithOffset(PtrOff), in reduceLoadWidth()
14630 LN0->getOriginalAlign(), in reduceLoadWidth()
14631 LN0->getMemOperand()->getFlags(), LN0->getAAInfo()); in reduceLoadWidth()
14633 Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr, in reduceLoadWidth()
14634 LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT, in reduceLoadWidth()
14635 LN0->getOriginalAlign(), in reduceLoadWidth()
14636 LN0->getMemOperand()->getFlags(), LN0->getAAInfo()); in reduceLoadWidth()
14671 SDValue N0 = N->getOperand(0); in visitSIGN_EXTEND_INREG()
14672 SDValue N1 = N->getOperand(1); in visitSIGN_EXTEND_INREG()
14673 EVT VT = N->getValueType(0); in visitSIGN_EXTEND_INREG()
14674 EVT ExtVT = cast<VTSDNode>(N1)->getVT(); in visitSIGN_EXTEND_INREG()
14682 // fold (sext_in_reg c1) -> c1 in visitSIGN_EXTEND_INREG()
14690 // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2 in visitSIGN_EXTEND_INREG()
14692 ExtVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT())) in visitSIGN_EXTEND_INREG()
14696 // fold (sext_in_reg (sext x)) -> (sext x) in visitSIGN_EXTEND_INREG()
14697 // fold (sext_in_reg (aext x)) -> (sext x) in visitSIGN_EXTEND_INREG()
14709 // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x) in visitSIGN_EXTEND_INREG()
14727 // fold (sext_in_reg (zext x)) -> (sext x) in visitSIGN_EXTEND_INREG()
14736 // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero. in visitSIGN_EXTEND_INREG()
14737 if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, ExtVTBits - 1))) in visitSIGN_EXTEND_INREG()
14745 // fold (sext_in_reg (load x)) -> (smaller sextload x) in visitSIGN_EXTEND_INREG()
14746 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits)) in visitSIGN_EXTEND_INREG()
14750 // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24) in visitSIGN_EXTEND_INREG()
14751 // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible. in visitSIGN_EXTEND_INREG()
14752 // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above. in visitSIGN_EXTEND_INREG()
14755 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) { in visitSIGN_EXTEND_INREG()
14759 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits) in visitSIGN_EXTEND_INREG()
14765 // fold (sext_inreg (extload x)) -> (sextload x) in visitSIGN_EXTEND_INREG()
14771 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() && in visitSIGN_EXTEND_INREG()
14772 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() && in visitSIGN_EXTEND_INREG()
14777 LN0->getChain(), in visitSIGN_EXTEND_INREG()
14778 LN0->getBasePtr(), ExtVT, in visitSIGN_EXTEND_INREG()
14779 LN0->getMemOperand()); in visitSIGN_EXTEND_INREG()
14786 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use in visitSIGN_EXTEND_INREG()
14789 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() && in visitSIGN_EXTEND_INREG()
14790 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) && in visitSIGN_EXTEND_INREG()
14794 LN0->getChain(), in visitSIGN_EXTEND_INREG()
14795 LN0->getBasePtr(), ExtVT, in visitSIGN_EXTEND_INREG()
14796 LN0->getMemOperand()); in visitSIGN_EXTEND_INREG()
14802 // fold (sext_inreg (masked_load x)) -> (sext_masked_load x) in visitSIGN_EXTEND_INREG()
14805 if (ExtVT == Ld->getMemoryVT() && N0.hasOneUse() && in visitSIGN_EXTEND_INREG()
14806 Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD && in visitSIGN_EXTEND_INREG()
14809 VT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), in visitSIGN_EXTEND_INREG()
14810 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(), in visitSIGN_EXTEND_INREG()
14811 Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad()); in visitSIGN_EXTEND_INREG()
14818 // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x) in visitSIGN_EXTEND_INREG()
14821 ExtVT == GN0->getMemoryVT() && in visitSIGN_EXTEND_INREG()
14823 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(), in visitSIGN_EXTEND_INREG()
14824 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()}; in visitSIGN_EXTEND_INREG()
14828 GN0->getMemOperand(), GN0->getIndexType(), ISD::SEXTLOAD); in visitSIGN_EXTEND_INREG()
14847 // -> (extract_subvector (signext iN_v to iM)) in visitSIGN_EXTEND_INREG()
14851 EVT InnerExtVT = InnerExt->getValueType(0); in visitSIGN_EXTEND_INREG()
14852 SDValue Extendee = InnerExt->getOperand(0); in visitSIGN_EXTEND_INREG()
14870 unsigned InregOpcode = N->getOpcode(); in foldExtendVectorInregToExtendOfSubvector()
14873 SDValue Src = N->getOperand(0); in foldExtendVectorInregToExtendOfSubvector()
14874 EVT VT = N->getValueType(0); in foldExtendVectorInregToExtendOfSubvector()
14882 // Profitability check: our operand must be an one-use CONCAT_VECTORS. in foldExtendVectorInregToExtendOfSubvector()
14883 // FIXME: one-use check may be overly restrictive in foldExtendVectorInregToExtendOfSubvector()
14900 SDValue N0 = N->getOperand(0); in visitEXTEND_VECTOR_INREG()
14901 EVT VT = N->getValueType(0); in visitEXTEND_VECTOR_INREG()
14907 return N->getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG in visitEXTEND_VECTOR_INREG()
14926 SDValue N0 = N->getOperand(0); in visitTRUNCATE()
14927 EVT VT = N->getValueType(0); in visitTRUNCATE()
14936 // fold (truncate (truncate x)) -> (truncate x) in visitTRUNCATE()
14940 // fold (truncate c1) -> c1 in visitTRUNCATE()
14944 // fold (truncate (ext x)) -> (ext x) or (truncate x) or x in visitTRUNCATE()
14959 // Try to narrow a truncate-of-sext_in_reg to the destination type: in visitTRUNCATE()
14960 // trunc (sign_ext_inreg X, iM) to iN --> sign_ext_inreg (trunc X to iN), iM in visitTRUNCATE()
14965 EVT ExtVT = cast<VTSDNode>(ExtVal)->getVT(); in visitTRUNCATE()
14973 if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND)) in visitTRUNCATE()
14976 // Fold extract-and-trunc into a narrow extract. For example: in visitTRUNCATE()
14979 // -- becomes -- in visitTRUNCATE()
14987 LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) { in visitTRUNCATE()
14990 EVT TrTy = N->getValueType(0); in visitTRUNCATE()
14999 SDValue EltNo = N0->getOperand(1); in visitTRUNCATE()
15001 int Elt = EltNo->getAsZExtVal(); in visitTRUNCATE()
15002 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1)); in visitTRUNCATE()
15009 // trunc (select c, a, b) -> select c, (trunc a), (trunc b) in visitTRUNCATE()
15021 // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits() in visitTRUNCATE()
15045 // Attempt to pre-truncate BUILD_VECTOR sources. in visitTRUNCATE()
15053 for (const SDValue &Op : N0->op_values()) { in visitTRUNCATE()
15060 // trunc (splat_vector x) -> splat_vector (trunc x) in visitTRUNCATE()
15066 VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0))); in visitTRUNCATE()
15099 // fold (truncate (load x)) -> (smaller load x) in visitTRUNCATE()
15100 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits)) in visitTRUNCATE()
15109 if (LN0->isSimple() && LN0->getMemoryVT().bitsLE(VT)) { in visitTRUNCATE()
15111 LN0->getExtensionType(), SDLoc(LN0), VT, LN0->getChain(), in visitTRUNCATE()
15112 LN0->getBasePtr(), LN0->getMemoryVT(), LN0->getMemOperand()); in visitTRUNCATE()
15119 // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)), in visitTRUNCATE()
15134 // Stop if more than one members are non-undef. in visitTRUNCATE()
15165 // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx in visitTRUNCATE()
15172 unsigned Idx = isLE ? 0 : VecSrcVT.getVectorNumElements() - 1; in visitTRUNCATE()
15178 // Simplify the operands using demanded-bits information. in visitTRUNCATE()
15182 // fold (truncate (extract_subvector(ext x))) -> in visitTRUNCATE()
15191 if (N00.getOperand(0)->getValueType(0).getVectorElementType() == in visitTRUNCATE()
15193 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
15201 // Narrow a suitable binary operation with a non-opaque constant operand by in visitTRUNCATE()
15202 // moving it ahead of the truncate. This is limited to pre-legalization in visitTRUNCATE()
15215 // TODO: We already restricted this to pre-legalization, but for vectors in visitTRUNCATE()
15217 // Target-specific changes are likely needed to avoid regressions here. in visitTRUNCATE()
15227 // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry) in visitTRUNCATE()
15228 // (trunc uaddo_carry(X, Y, Carry)) -> in visitTRUNCATE()
15234 N0.hasOneUse() && !N0->hasAnyUseOfValue(1)) { in visitTRUNCATE()
15237 SDVTList VTs = DAG.getVTList(VT, N0->getValueType(1)); in visitTRUNCATE()
15242 // Truncate the USUBSAT only if LHS is a known zero-extension, its not in visitTRUNCATE()
15260 SDValue Elt = N->getOperand(i); in getBuildPairElt()
15266 /// build_pair (load, load) -> load
15269 assert(N->getOpcode() == ISD::BUILD_PAIR); in CombineConsecutiveLoads()
15281 !LD1->hasOneUse() || !LD2->hasOneUse() || in CombineConsecutiveLoads()
15282 LD1->getAddressSpace() != LD2->getAddressSpace()) in CombineConsecutiveLoads()
15286 EVT LD1VT = LD1->getValueType(0); in CombineConsecutiveLoads()
15291 *LD1->getMemOperand(), &LD1Fast) && LD1Fast) in CombineConsecutiveLoads()
15292 return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), in CombineConsecutiveLoads()
15293 LD1->getPointerInfo(), LD1->getAlign()); in CombineConsecutiveLoads()
15299 // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi in getPPCf128HiElementSelector()
15300 // and Lo parts; on big-endian machines it doesn't. in getPPCf128HiElementSelector()
15307 // IEEE754-compliant FP logic, we're done. in foldBitcastedFPLogic()
15308 EVT VT = N->getValueType(0); in foldBitcastedFPLogic()
15309 SDValue N0 = N->getOperand(0); in foldBitcastedFPLogic()
15352 // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X in foldBitcastedFPLogic()
15353 // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X in foldBitcastedFPLogic()
15354 // Fold (bitcast int (or (bitcast fp X to int), 0x8000...) to fp) -> in foldBitcastedFPLogic()
15358 if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask && in foldBitcastedFPLogic()
15372 SDValue N0 = N->getOperand(0); in visitBITCAST()
15373 EVT VT = N->getValueType(0); in visitBITCAST()
15388 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() && in visitBITCAST()
15389 cast<BuildVectorSDNode>(N0)->isConstant()) in visitBITCAST()
15396 // a fp -> int or int -> conversion and that the resulting operation will in visitBITCAST()
15409 // (conv (conv x, t1), t2) -> (conv x, t2) in visitBITCAST()
15413 // fold (conv (logicop (conv x), (c))) -> (logicop x, (conv c)) in visitBITCAST()
15421 V->hasOneUse()); in visitBITCAST()
15429 // fold (conv (load x)) -> (load (conv*)x) in visitBITCAST()
15440 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) || in visitBITCAST()
15445 *LN0->getMemOperand())) { in visitBITCAST()
15447 DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in visitBITCAST()
15448 LN0->getMemOperand()); in visitBITCAST()
15457 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit) in visitBITCAST()
15458 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit)) in visitBITCAST()
15461 // fold (bitcast (fneg x)) -> in visitBITCAST()
15465 // fold (bitcast (fabs x)) -> in visitBITCAST()
15471 N0->hasOneUse() && VT.isInteger() && !VT.isVector() && in visitBITCAST()
15509 // fold (bitconvert (fcopysign cst, x)) -> in visitBITCAST()
15515 // fold (bitcast (fcopysign cst, x)) -> in visitBITCAST()
15520 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitBITCAST()
15540 DAG.getConstant(OrigXWidth-VTWidth, DL, in visitBITCAST()
15583 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive. in visitBITCAST()
15588 // Remove double bitcasts from shuffles - this is often a legacy of in visitBITCAST()
15591 // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1) in visitBITCAST()
15593 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() && in visitBITCAST()
15612 SDValue SV0 = PeekThroughBitcast(N0->getOperand(0)); in visitBITCAST()
15613 SDValue SV1 = PeekThroughBitcast(N0->getOperand(1)); in visitBITCAST()
15620 for (int M : SVN->getMask()) in visitBITCAST()
15622 NewMask.push_back(M < 0 ? -1 : M * MaskScale + i); in visitBITCAST()
15634 EVT VT = N->getValueType(0); in visitBUILD_PAIR()
15639 SDValue N0 = N->getOperand(0); in visitFREEZE()
15650 // Fold freeze(op(x, ...)) -> op(freeze(x), ...). in visitFREEZE()
15653 // conditions 1) one-use, 2) does not produce poison, and 3) has all but one in visitFREEZE()
15654 // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push in visitFREEZE()
15655 // the freeze through to the operands that are not guaranteed non-poison. in visitFREEZE()
15656 // NOTE: we will strip poison-generating flags, so ignore them here. in visitFREEZE()
15659 N0->getNumValues() != 1 || !N0->hasOneUse()) in visitFREEZE()
15683 for (const SDValue &Op : N0->op_values()) in visitFREEZE()
15692 for (auto [OpNo, Op] : enumerate(N0->ops())) { in visitFREEZE()
15703 // Multiple maybe-poison ops when not allowed - bail out. in visitFREEZE()
15708 // it could create undef or poison due to it's poison-generating flags. in visitFREEZE()
15709 // So not finding any maybe-poison operands is fine. in visitFREEZE()
15722 SDValue MaybePoisonOperand = N->getOperand(0).getOperand(OpNo); in visitFREEZE()
15740 if (N->getOpcode() == ISD::DELETED_NODE) in visitFREEZE()
15744 // may no longer be valid. Re-fetch the operand we're `freeze`ing. in visitFREEZE()
15745 N0 = N->getOperand(0); in visitFREEZE()
15749 SmallVector<SDValue> Ops(N0->op_begin(), N0->op_end()); in visitFREEZE()
15750 // Special-handle ISD::UNDEF, each single one of them can be it's own thing. in visitFREEZE()
15760 SVN->getMask()); in visitFREEZE()
15763 R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops); in visitFREEZE()
15774 EVT SrcEltVT = BV->getValueType(0).getVectorElementType(); in ConstantFoldBITCASTofBUILD_VECTOR()
15783 // type, convert each element. This handles FP<->INT cases. in ConstantFoldBITCASTofBUILD_VECTOR()
15786 for (SDValue Op : BV->op_values()) { in ConstantFoldBITCASTofBUILD_VECTOR()
15795 BV->getValueType(0).getVectorNumElements()); in ConstantFoldBITCASTofBUILD_VECTOR()
15831 if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements)) in ConstantFoldBITCASTofBUILD_VECTOR()
15847 // Returns true if floating point contraction is allowed on the FMUL-SDValue
15853 N->getFlags().hasAllowContract(); in isContractableFMUL()
15858 return Options.NoInfsFPMath || N->getFlags().hasNoInfs(); in hasNoInfs()
15864 SDValue N0 = N->getOperand(0); in visitFADDForFMACombine()
15865 SDValue N1 = N->getOperand(1); in visitFADDForFMACombine()
15866 EVT VT = N->getValueType(0); in visitFADDForFMACombine()
15873 // Floating-point multiply-add with intermediate rounding. in visitFADDForFMACombine()
15878 // Floating-point multiply-add without intermediate rounding. in visitFADDForFMACombine()
15890 if (!AllowFusionGlobally && !N->getFlags().hasAllowContract()) in visitFADDForFMACombine()
15893 // Folding fadd (fmul x, y), (fmul x, y) -> fma x, y, (fmul x, y) is never in visitFADDForFMACombine()
15916 return AllowFusionGlobally || N->getFlags().hasAllowContract(); in visitFADDForFMACombine()
15921 if (N0->use_size() > N1->use_size()) in visitFADDForFMACombine()
15925 // fold (fadd (fmul x, y), z) -> (fma x, y, z) in visitFADDForFMACombine()
15926 if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) { in visitFADDForFMACombine()
15931 // fold (fadd x, (fmul y, z)) -> (fma y, z, x) in visitFADDForFMACombine()
15933 if (isContractableFMUL(N1) && (Aggressive || N1->hasOneUse())) { in visitFADDForFMACombine()
15938 // fadd (fma A, B, (fmul C, D)), E --> fma A, B, (fma C, D, E) in visitFADDForFMACombine()
15939 // fadd E, (fma A, B, (fmul C, D)) --> fma A, B, (fma C, D, E) in visitFADDForFMACombine()
15941 // fadd (fma A, B, (fma (C, D, (fmul (E, F))))), G --> in visitFADDForFMACombine()
15943 // fadd (G, (fma A, B, (fma (C, D, (fmul (E, F)))))) --> in visitFADDForFMACombine()
15947 Options.UnsafeFPMath || N->getFlags().hasAllowReassociation(); in visitFADDForFMACombine()
15960 SDValue FMul = TmpFMA->getOperand(2); in visitFADDForFMACombine()
15971 TmpFMA = TmpFMA->getOperand(2); in visitFADDForFMACombine()
15977 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) in visitFADDForFMACombine()
15990 // fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x) in visitFADDForFMACombine()
16007 // -> (fma x, y, (fma (fpext u), (fpext v), z)) in visitFADDForFMACombine()
16031 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z)) in visitFADDForFMACombine()
16032 // FIXME: This turns two single-precision and one double-precision in visitFADDForFMACombine()
16033 // operation into two double-precision operations, which might not be in visitFADDForFMACombine()
16060 // -> (fma y, z, (fma (fpext u), (fpext v), x)) in visitFADDForFMACombine()
16076 // -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x)) in visitFADDForFMACombine()
16077 // FIXME: This turns two single-precision and one double-precision in visitFADDForFMACombine()
16078 // operation into two double-precision operations, which might not be in visitFADDForFMACombine()
16101 SDValue N0 = N->getOperand(0); in visitFSUBForFMACombine()
16102 SDValue N1 = N->getOperand(1); in visitFSUBForFMACombine()
16103 EVT VT = N->getValueType(0); in visitFSUBForFMACombine()
16110 // Floating-point multiply-add with intermediate rounding. in visitFSUBForFMACombine()
16115 // Floating-point multiply-add without intermediate rounding. in visitFSUBForFMACombine()
16124 const SDNodeFlags Flags = N->getFlags(); in visitFSUBForFMACombine()
16129 if (!AllowFusionGlobally && !N->getFlags().hasAllowContract()) in visitFSUBForFMACombine()
16145 return AllowFusionGlobally || N->getFlags().hasAllowContract(); in visitFSUBForFMACombine()
16148 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z)) in visitFSUBForFMACombine()
16150 if (isContractableFMUL(XY) && (Aggressive || XY->hasOneUse())) { in visitFSUBForFMACombine()
16158 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x) in visitFSUBForFMACombine()
16161 if (isContractableFMUL(YZ) && (Aggressive || YZ->hasOneUse())) { in visitFSUBForFMACombine()
16173 (N0->use_size() > N1->use_size())) { in visitFSUBForFMACombine()
16174 // fold (fsub (fmul a, b), (fmul c, d)) -> (fma (fneg c), d, (fmul a, b)) in visitFSUBForFMACombine()
16177 // fold (fsub (fmul a, b), (fmul c, d)) -> (fma a, b, (fneg (fmul c, d))) in visitFSUBForFMACombine()
16181 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z)) in visitFSUBForFMACombine()
16184 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x) in visitFSUBForFMACombine()
16189 // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) in visitFSUBForFMACombine()
16191 (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) { in visitFSUBForFMACombine()
16202 // -> (fma (fpext x), (fpext y), (fneg z)) in visitFSUBForFMACombine()
16217 // -> (fma (fneg (fpext y)), (fpext z), x) in visitFSUBForFMACombine()
16234 // -> (fneg (fma (fpext x), (fpext y), z)) in visitFSUBForFMACombine()
16237 // orthogonal flags -fp-contract=fast and -enable-unsafe-fp-math prevent in visitFSUBForFMACombine()
16258 // -> (fneg (fma (fpext x)), (fpext y), z) in visitFSUBForFMACombine()
16261 // orthogonal flags -fp-contract=fast and -enable-unsafe-fp-math prevent in visitFSUBForFMACombine()
16282 return Options.UnsafeFPMath || N->getFlags().hasAllowReassociation(); in visitFSUBForFMACombine()
16296 bool CanFuse = Options.UnsafeFPMath || N->getFlags().hasAllowContract(); in visitFSUBForFMACombine()
16298 // -> (fma x, y (fma u, v, (fneg z))) in visitFSUBForFMACombine()
16301 N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) { in visitFSUBForFMACombine()
16311 // -> (fma (fneg y), z, (fma (fneg u), v, x)) in visitFSUBForFMACombine()
16314 N1->hasOneUse() && NoSignedZero) { in visitFSUBForFMACombine()
16326 // -> (fma x, y (fma (fpext u), (fpext v), (fneg z))) in visitFSUBForFMACombine()
16327 if (isFusedOp(N0) && N0->hasOneUse()) { in visitFSUBForFMACombine()
16346 // -> (fma (fpext x), (fpext y), in visitFSUBForFMACombine()
16348 // FIXME: This turns two single-precision and one double-precision in visitFSUBForFMACombine()
16349 // operation into two double-precision operations, which might not be in visitFSUBForFMACombine()
16372 // -> (fma (fneg y), z, (fma (fneg (fpext u)), (fpext v), x)) in visitFSUBForFMACombine()
16374 N1->hasOneUse()) { in visitFSUBForFMACombine()
16394 // -> (fma (fneg (fpext y)), (fpext z), in visitFSUBForFMACombine()
16396 // FIXME: This turns two single-precision and one double-precision in visitFSUBForFMACombine()
16397 // operation into two double-precision operations, which might not be in visitFSUBForFMACombine()
16430 SDValue N0 = N->getOperand(0); in visitFMULForFMADistributiveCombine()
16431 SDValue N1 = N->getOperand(1); in visitFMULForFMADistributiveCombine()
16432 EVT VT = N->getValueType(0); in visitFMULForFMADistributiveCombine()
16435 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation"); in visitFMULForFMADistributiveCombine()
16445 // Floating-point multiply-add without intermediate rounding. in visitFMULForFMADistributiveCombine()
16451 // Floating-point multiply-add with intermediate rounding. This can result in visitFMULForFMADistributiveCombine()
16464 // fold (fmul (fadd x0, +1.0), y) -> (fma x0, y, y) in visitFMULForFMADistributiveCombine()
16465 // fold (fmul (fadd x0, -1.0), y) -> (fma x0, y, (fneg y)) in visitFMULForFMADistributiveCombine()
16467 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
16469 if (C->isExactlyValue(+1.0)) in visitFMULForFMADistributiveCombine()
16472 if (C->isExactlyValue(-1.0)) in visitFMULForFMADistributiveCombine()
16485 // fold (fmul (fsub +1.0, x1), y) -> (fma (fneg x1), y, y) in visitFMULForFMADistributiveCombine()
16486 // fold (fmul (fsub -1.0, x1), y) -> (fma (fneg x1), y, (fneg y)) in visitFMULForFMADistributiveCombine()
16487 // fold (fmul (fsub x0, +1.0), y) -> (fma x0, y, (fneg y)) in visitFMULForFMADistributiveCombine()
16488 // fold (fmul (fsub x0, -1.0), y) -> (fma x0, y, y) in visitFMULForFMADistributiveCombine()
16490 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
16492 if (C0->isExactlyValue(+1.0)) in visitFMULForFMADistributiveCombine()
16496 if (C0->isExactlyValue(-1.0)) in visitFMULForFMADistributiveCombine()
16502 if (C1->isExactlyValue(+1.0)) in visitFMULForFMADistributiveCombine()
16505 if (C1->isExactlyValue(-1.0)) in visitFMULForFMADistributiveCombine()
16524 // FADD -> FMA combines: in visitVP_FADD()
16534 SDValue N0 = N->getOperand(0); in visitFADD()
16535 SDValue N1 = N->getOperand(1); in visitFADD()
16538 EVT VT = N->getValueType(0); in visitFADD()
16541 SDNodeFlags Flags = N->getFlags(); in visitFADD()
16544 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFADD()
16547 // fold (fadd c1, c2) -> c1 + c2 in visitFADD()
16560 // N0 + -0.0 --> N0 (also allowed with +0.0 and fast-math) in visitFADD()
16562 if (N1C && N1C->isZero()) in visitFADD()
16563 if (N1C->isNegative() || Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) in visitFADD()
16569 // fold (fadd A, (fneg B)) -> (fsub A, B) in visitFADD()
16575 // fold (fadd (fneg A), B) -> (fsub B, A) in visitFADD()
16585 return C && C->isExactlyValue(-2.0); in visitFADD()
16588 // fadd (fmul B, -2.0), A --> fsub A, (fadd B, B) in visitFADD()
16594 // fadd A, (fmul B, -2.0) --> fsub A, (fadd B, B) in visitFADD()
16607 // If allowed, fold (fadd (fneg x), x) -> 0.0 in visitFADD()
16611 // If allowed, fold (fadd x, (fneg x)) -> 0.0 in visitFADD()
16622 // fadd (fadd x, c1), c2 -> fadd x, c1 + c2 in visitFADD()
16639 // (fadd (fmul x, c), x) -> (fmul x, c+1) in visitFADD()
16646 // (fadd (fmul x, c), (fadd x, x)) -> (fmul x, c+2) in visitFADD()
16662 // (fadd x, (fmul x, c)) -> (fmul x, c+1) in visitFADD()
16669 // (fadd (fadd x, x), (fmul x, c)) -> (fmul x, c+2) in visitFADD()
16682 // (fadd (fadd x, x), x) -> (fmul x, 3.0) in visitFADD()
16693 // (fadd x, (fadd x, x)) -> (fmul x, 3.0) in visitFADD()
16701 // (fadd (fadd x, x), (fadd x, x)) -> (fmul x, 4.0) in visitFADD()
16711 // Fold fadd(vecreduce(x), vecreduce(y)) -> vecreduce(fadd(x, y)) in visitFADD()
16715 } // enable-unsafe-fp-math in visitFADD()
16717 // FADD -> FMA combines: in visitFADD()
16727 SDValue Chain = N->getOperand(0); in visitSTRICT_FADD()
16728 SDValue N0 = N->getOperand(1); in visitSTRICT_FADD()
16729 SDValue N1 = N->getOperand(2); in visitSTRICT_FADD()
16730 EVT VT = N->getValueType(0); in visitSTRICT_FADD()
16731 EVT ChainVT = N->getValueType(1); in visitSTRICT_FADD()
16735 // fold (strict_fadd A, (fneg B)) -> (strict_fsub A, B) in visitSTRICT_FADD()
16743 // fold (strict_fadd (fneg A), B) -> (strict_fsub B, A) in visitSTRICT_FADD()
16754 SDValue N0 = N->getOperand(0); in visitFSUB()
16755 SDValue N1 = N->getOperand(1); in visitFSUB()
16758 EVT VT = N->getValueType(0); in visitFSUB()
16761 const SDNodeFlags Flags = N->getFlags(); in visitFSUB()
16764 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFSUB()
16767 // fold (fsub c1, c2) -> c1-c2 in visitFSUB()
16779 // (fsub A, 0) -> A in visitFSUB()
16780 if (N1CFP && N1CFP->isZero()) { in visitFSUB()
16781 if (!N1CFP->isNegative() || Options.NoSignedZerosFPMath || in visitFSUB()
16788 // (fsub x, x) -> 0.0 in visitFSUB()
16793 // (fsub -0.0, N1) -> -N1 in visitFSUB()
16794 if (N0CFP && N0CFP->isZero()) { in visitFSUB()
16795 if (N0CFP->isNegative() || in visitFSUB()
16797 // We cannot replace an FSUB(+-0.0,X) with FNEG(X) when denormals are in visitFSUB()
16815 // X - (X + Y) -> -Y in visitFSUB()
16816 if (N0 == N1->getOperand(0)) in visitFSUB()
16817 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
16818 // X - (Y + X) -> -Y in visitFSUB()
16819 if (N0 == N1->getOperand(1)) in visitFSUB()
16820 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
16823 // fold (fsub A, (fneg B)) -> (fadd A, B) in visitFSUB()
16828 // FSUB -> FMA combines: in visitFSUB()
16839 // -> (bitcast_to_FP (add (bitcast_to_INT C), Log2(Pow2) << mantissa))
16841 // -> (bitcast_to_FP (sub (bitcast_to_INT C), Log2(Pow2) << mantissa))
16853 EVT VT = N->getValueType(0); in combineFMulOrFDivWithIntPow2()
16858 if (ConstOpIdx == 1 && N->getOpcode() == ISD::FDIV) in combineFMulOrFDivWithIntPow2()
16861 ConstOp = peekThroughBitcasts(N->getOperand(ConstOpIdx)); in combineFMulOrFDivWithIntPow2()
16862 Pow2Op = N->getOperand(1 - ConstOpIdx); in combineFMulOrFDivWithIntPow2()
16878 const APFloat &APF = CFP->getValueAPF(); in combineFMulOrFDivWithIntPow2()
16889 N->getOpcode() == ISD::FMUL ? CurExp : (CurExp - MaxExpChange); in combineFMulOrFDivWithIntPow2()
16892 N->getOpcode() == ISD::FDIV ? CurExp : (CurExp + MaxExpChange); in combineFMulOrFDivWithIntPow2()
16898 int ThisMantissa = APFloat::semanticsPrecision(APF.getSemantics()) - 1; in combineFMulOrFDivWithIntPow2()
16937 DAG.getNode(N->getOpcode() == ISD::FMUL ? ISD::ADD : ISD::SUB, DL, in combineFMulOrFDivWithIntPow2()
16944 SDValue N0 = N->getOperand(0); in visitFMUL()
16945 SDValue N1 = N->getOperand(1); in visitFMUL()
16947 EVT VT = N->getValueType(0); in visitFMUL()
16950 const SDNodeFlags Flags = N->getFlags(); in visitFMUL()
16953 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFMUL()
16956 // fold (fmul c1, c2) -> c1*c2 in visitFMUL()
16974 // fmul (fmul X, C1), C2 -> fmul X, C1 * C2 in visitFMUL()
16988 // Match a special-case: we convert X * 2.0 into fadd. in visitFMUL()
16989 // fmul (fadd X, X), C -> fmul X, 2.0 * C in visitFMUL()
16997 // Fold fmul(vecreduce(x), vecreduce(y)) -> vecreduce(fmul(x, y)) in visitFMUL()
17003 // fold (fmul X, 2.0) -> (fadd X, X) in visitFMUL()
17004 if (N1CFP && N1CFP->isExactlyValue(+2.0)) in visitFMUL()
17007 // fold (fmul X, -1.0) -> (fsub -0.0, X) in visitFMUL()
17008 if (N1CFP && N1CFP->isExactlyValue(-1.0)) { in visitFMUL()
17011 DAG.getConstantFP(-0.0, DL, VT), N0, Flags); in visitFMUL()
17015 // -N0 * -N1 --> N0 * N1 in visitFMUL()
17031 // fold (fmul X, (select (fcmp X > 0.0), -1.0, 1.0)) -> (fneg (fabs X)) in visitFMUL()
17032 // fold (fmul X, (select (fcmp X > 0.0), 1.0, -1.0)) -> (fabs X) in visitFMUL()
17047 cast<ConstantFPSDNode>(Cond.getOperand(1))->isExactlyValue(0.0)) { in visitFMUL()
17048 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in visitFMUL()
17065 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) && in visitFMUL()
17069 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0)) in visitFMUL()
17077 // FMUL -> FMA combines: in visitFMUL()
17083 // Don't do `combineFMulOrFDivWithIntPow2` until after FMUL -> FMA has been in visitFMUL()
17092 SDValue N0 = N->getOperand(0); in visitFMA()
17093 SDValue N1 = N->getOperand(1); in visitFMA()
17094 SDValue N2 = N->getOperand(2); in visitFMA()
17097 EVT VT = N->getValueType(0); in visitFMA()
17111 // (-N0 * -N1) + N2 --> (N0 * N1) + N2 in visitFMA()
17129 if (N0CFP && N0CFP->isZero()) in visitFMA()
17131 if (N1CFP && N1CFP->isZero()) in visitFMA()
17136 if (N0CFP && N0CFP->isExactlyValue(1.0)) in visitFMA()
17138 if (N1CFP && N1CFP->isExactlyValue(1.0)) in visitFMA()
17141 // Canonicalize (fma c, x, y) -> (fma x, c, y) in visitFMA()
17147 Options.UnsafeFPMath || N->getFlags().hasAllowReassociation(); in visitFMA()
17149 // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2) in visitFMA()
17158 // (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y) in visitFMA()
17168 // (fma x, -1, y) -> (fadd (fneg x), y) in visitFMA()
17171 if (N1CFP->isExactlyValue(1.0)) in visitFMA()
17174 if (N1CFP->isExactlyValue(-1.0) && in visitFMA()
17181 // fma (fneg x), K, y -> fma x -K, y in visitFMA()
17185 !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, ForCodeSize)))) { in visitFMA()
17193 // (fma x, c, x) -> (fmul x, (c+1)) in visitFMA()
17200 // (fma x, c, (fneg x)) -> (fmul x, (c-1)) in visitFMA()
17204 DAG.getConstantFP(-1.0, DL, VT))); in visitFMA()
17208 // fold ((fma (fneg X), Y, (fneg Z)) -> fneg (fma X, Y, Z)) in visitFMA()
17209 // fold ((fma X, (fneg Y), (fneg Z)) -> fneg (fma X, Y, Z)) in visitFMA()
17218 SDValue N0 = N->getOperand(0); in visitFMAD()
17219 SDValue N1 = N->getOperand(1); in visitFMAD()
17220 SDValue N2 = N->getOperand(2); in visitFMAD()
17221 EVT VT = N->getValueType(0); in visitFMAD()
17234 // E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip)
17240 // TODO: Limit this transform based on optsize/minsize - it always creates at in combineRepeatedFPDivisors()
17244 const SDNodeFlags Flags = N->getFlags(); in combineRepeatedFPDivisors()
17248 // Skip if current node is a reciprocal/fneg-reciprocal. in combineRepeatedFPDivisors()
17249 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in combineRepeatedFPDivisors()
17251 if (N0CFP && (N0CFP->isExactlyValue(1.0) || N0CFP->isExactlyValue(-1.0))) in combineRepeatedFPDivisors()
17258 // For splat vectors, scale the number of uses by the splat factor. If we can in combineRepeatedFPDivisors()
17261 EVT VT = N->getValueType(0); in combineRepeatedFPDivisors()
17265 if (!MinUses || (N1->use_size() * NumElts) < MinUses) in combineRepeatedFPDivisors()
17271 for (auto *U : N1->uses()) { in combineRepeatedFPDivisors()
17272 if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1) { in combineRepeatedFPDivisors()
17274 if (U->getOperand(1).getOpcode() == ISD::FSQRT && in combineRepeatedFPDivisors()
17275 U->getOperand(0) == U->getOperand(1).getOperand(0) && in combineRepeatedFPDivisors()
17276 U->getFlags().hasAllowReassociation() && in combineRepeatedFPDivisors()
17277 U->getFlags().hasNoSignedZeros()) in combineRepeatedFPDivisors()
17282 if (UnsafeMath || U->getFlags().hasAllowReciprocal()) in combineRepeatedFPDivisors()
17296 // Dividend / Divisor -> Dividend * Reciprocal in combineRepeatedFPDivisors()
17298 SDValue Dividend = U->getOperand(0); in combineRepeatedFPDivisors()
17304 // In the absence of fast-math-flags, this user node is always the in combineRepeatedFPDivisors()
17313 SDValue N0 = N->getOperand(0); in visitFDIV()
17314 SDValue N1 = N->getOperand(1); in visitFDIV()
17315 EVT VT = N->getValueType(0); in visitFDIV()
17318 SDNodeFlags Flags = N->getFlags(); in visitFDIV()
17321 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFDIV()
17324 // fold (fdiv c1, c2) -> c1/c2 in visitFDIV()
17339 // fold (fdiv X, c2) -> (fmul X, 1/c2) if there is no loss in precision, or in visitFDIV()
17343 const APFloat &N1APF = N1CFP->getValueAPF(); in visitFDIV()
17363 // into a target-specific square root estimate instruction. in visitFDIV()
17398 N1->getFlags().hasAllowReassociation() && Sqrt.hasOneUse()) { in visitFDIV()
17405 // X / (fabs(A) * sqrt(Z)) --> X / sqrt(A*A*Z) --> X * rsqrt(A*A*Z) in visitFDIV()
17406 // X / (A * sqrt(A)) --> X / sqrt(A*A*A) --> X * rsqrt(A*A*A) in visitFDIV()
17419 // X / (Y * sqrt(Z)) -> X * (rsqrt(Z) / Y) in visitFDIV()
17434 // Fold X/Sqrt(X) -> Sqrt(X) in visitFDIV()
17440 // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y) in visitFDIV()
17463 SDValue N0 = N->getOperand(0); in visitFREM()
17464 SDValue N1 = N->getOperand(1); in visitFREM()
17465 EVT VT = N->getValueType(0); in visitFREM()
17466 SDNodeFlags Flags = N->getFlags(); in visitFREM()
17470 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFREM()
17473 // fold (frem c1, c2) -> fmod(c1,c2) in visitFREM()
17480 // Lower frem N0, N1 => x - trunc(N0 / N1) * N1, providing N1 is an integer in visitFREM()
17506 SDNodeFlags Flags = N->getFlags(); in visitFSQRT()
17515 SDValue N0 = N->getOperand(0); in visitFSQRT()
17521 // transform the fdiv, we may produce a sub-optimal estimate sequence in visitFSQRT()
17527 /// copysign(x, fp_extend(y)) -> copysign(x, y)
17528 /// copysign(x, fp_round(y)) -> copysign(x, y)
17531 // Always fold no-op FP casts. in CanCombineFCOPYSIGN_EXTEND_ROUND()
17546 SDValue N1 = N->getOperand(1); in CanCombineFCOPYSIGN_EXTEND_ROUND()
17550 EVT N1VT = N1->getValueType(0); in CanCombineFCOPYSIGN_EXTEND_ROUND()
17551 EVT N1Op0VT = N1->getOperand(0).getValueType(); in CanCombineFCOPYSIGN_EXTEND_ROUND()
17556 SDValue N0 = N->getOperand(0); in visitFCOPYSIGN()
17557 SDValue N1 = N->getOperand(1); in visitFCOPYSIGN()
17558 EVT VT = N->getValueType(0); in visitFCOPYSIGN()
17561 // fold (fcopysign c1, c2) -> fcopysign(c1,c2) in visitFCOPYSIGN()
17565 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N->getOperand(1))) { in visitFCOPYSIGN()
17566 const APFloat &V = N1C->getValueAPF(); in visitFCOPYSIGN()
17567 // copysign(x, c1) -> fabs(x) iff ispos(c1) in visitFCOPYSIGN()
17568 // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1) in visitFCOPYSIGN()
17579 // copysign(fabs(x), y) -> copysign(x, y) in visitFCOPYSIGN()
17580 // copysign(fneg(x), y) -> copysign(x, y) in visitFCOPYSIGN()
17581 // copysign(copysign(x,z), y) -> copysign(x, y) in visitFCOPYSIGN()
17586 // copysign(x, abs(y)) -> abs(x) in visitFCOPYSIGN()
17590 // copysign(x, copysign(y,z)) -> copysign(x, z) in visitFCOPYSIGN()
17594 // copysign(x, fp_extend(y)) -> copysign(x, y) in visitFCOPYSIGN()
17595 // copysign(x, fp_round(y)) -> copysign(x, y) in visitFCOPYSIGN()
17605 // We only take the non-sign bits from the value operand in visitFCOPYSIGN()
17614 ConstantFPSDNode *ExponentC = isConstOrConstSplatFP(N->getOperand(1)); in visitFPOW()
17623 EVT VT = N->getValueType(0); in visitFPOW()
17624 if ((VT == MVT::f32 && ExponentC->getValueAPF().isExactlyValue(1.0f/3.0f)) || in visitFPOW()
17625 (VT == MVT::f64 && ExponentC->getValueAPF().isExactlyValue(1.0/3.0))) { in visitFPOW()
17626 // pow(-0.0, 1/3) = +0.0; cbrt(-0.0) = -0.0. in visitFPOW()
17627 // pow(-inf, 1/3) = +inf; cbrt(-inf) = -inf. in visitFPOW()
17628 // pow(-val, 1/3) = nan; cbrt(-val) = -num. in visitFPOW()
17632 SDNodeFlags Flags = N->getFlags(); in visitFPOW()
17644 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
17650 // power-of-2 fractional exponents. in visitFPOW()
17651 bool ExponentIs025 = ExponentC->getValueAPF().isExactlyValue(0.25); in visitFPOW()
17652 bool ExponentIs075 = ExponentC->getValueAPF().isExactlyValue(0.75); in visitFPOW()
17654 // pow(-0.0, 0.25) = +0.0; sqrt(sqrt(-0.0)) = -0.0. in visitFPOW()
17655 // pow(-inf, 0.25) = +inf; sqrt(sqrt(-inf)) = NaN. in visitFPOW()
17656 // pow(-0.0, 0.75) = +0.0; sqrt(-0.0) * sqrt(sqrt(-0.0)) = +0.0. in visitFPOW()
17657 // pow(-inf, 0.75) = +inf; sqrt(-inf) * sqrt(sqrt(-inf)) = NaN. in visitFPOW()
17661 SDNodeFlags Flags = N->getFlags(); in visitFPOW()
17677 // pow(X, 0.25) --> sqrt(sqrt(X)) in visitFPOW()
17679 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
17683 // pow(X, 0.75) --> sqrt(X) * sqrt(sqrt(X)) in visitFPOW()
17693 // replacing casts with a libcall. We also must be allowed to ignore -0.0 in foldFPToIntToFP()
17694 // because FTRUNC will return -0.0 for (-1.0, -0.0), but using integer in foldFPToIntToFP()
17696 // FIXME: We should be able to use node-level FMF here. in foldFPToIntToFP()
17698 EVT VT = N->getValueType(0); in foldFPToIntToFP()
17704 // back is the same as an 'ftrunc': [us]itofp (fpto[us]i X) --> ftrunc X in foldFPToIntToFP()
17705 SDValue N0 = N->getOperand(0); in foldFPToIntToFP()
17706 if (N->getOpcode() == ISD::SINT_TO_FP && N0.getOpcode() == ISD::FP_TO_SINT && in foldFPToIntToFP()
17710 if (N->getOpcode() == ISD::UINT_TO_FP && N0.getOpcode() == ISD::FP_TO_UINT && in foldFPToIntToFP()
17718 SDValue N0 = N->getOperand(0); in visitSINT_TO_FP()
17719 EVT VT = N->getValueType(0); in visitSINT_TO_FP()
17726 // fold (sint_to_fp c1) -> c1fp in visitSINT_TO_FP()
17728 // ...but only if the target supports immediate floating-point values in visitSINT_TO_FP()
17743 // fold (sint_to_fp (setcc x, y, cc)) -> (select (setcc x, y, cc), -1.0, 0.0) in visitSINT_TO_FP()
17748 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(-1.0, DL, VT), in visitSINT_TO_FP()
17752 // fold (sint_to_fp (zext (setcc x, y, cc))) -> in visitSINT_TO_FP()
17770 SDValue N0 = N->getOperand(0); in visitUINT_TO_FP()
17771 EVT VT = N->getValueType(0); in visitUINT_TO_FP()
17778 // fold (uint_to_fp c1) -> c1fp in visitUINT_TO_FP()
17780 // ...but only if the target supports immediate floating-point values in visitUINT_TO_FP()
17794 // fold (uint_to_fp (setcc x, y, cc)) -> (select (setcc x, y, cc), 1.0, 0.0) in visitUINT_TO_FP()
17808 // Fold (fp_to_{s/u}int ({s/u}int_to_fpx)) -> zext x, sext x, trunc x, or x
17810 SDValue N0 = N->getOperand(0); in FoldIntToFPToInt()
17811 EVT VT = N->getValueType(0); in FoldIntToFPToInt()
17819 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT; in FoldIntToFPToInt()
17830 unsigned InputSize = (int)SrcVT.getScalarSizeInBits() - IsInputSigned; in FoldIntToFPToInt()
17851 SDValue N0 = N->getOperand(0); in visitFP_TO_SINT()
17852 EVT VT = N->getValueType(0); in visitFP_TO_SINT()
17854 // fold (fp_to_sint undef) -> undef in visitFP_TO_SINT()
17858 // fold (fp_to_sint c1fp) -> c1 in visitFP_TO_SINT()
17866 SDValue N0 = N->getOperand(0); in visitFP_TO_UINT()
17867 EVT VT = N->getValueType(0); in visitFP_TO_UINT()
17869 // fold (fp_to_uint undef) -> undef in visitFP_TO_UINT()
17873 // fold (fp_to_uint c1fp) -> c1 in visitFP_TO_UINT()
17881 SDValue N0 = N->getOperand(0); in visitXRINT()
17882 EVT VT = N->getValueType(0); in visitXRINT()
17884 // fold (lrint|llrint undef) -> undef in visitXRINT()
17888 // fold (lrint|llrint c1fp) -> c1 in visitXRINT()
17890 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0); in visitXRINT()
17896 SDValue N0 = N->getOperand(0); in visitFP_ROUND()
17897 SDValue N1 = N->getOperand(1); in visitFP_ROUND()
17898 EVT VT = N->getValueType(0); in visitFP_ROUND()
17900 // fold (fp_round c1fp) -> c1fp in visitFP_ROUND()
17905 // fold (fp_round (fp_extend x)) -> x in visitFP_ROUND()
17909 // fold (fp_round (fp_round x)) -> (fp_round x) in visitFP_ROUND()
17911 const bool NIsTrunc = N->getConstantOperandVal(1) == 1; in visitFP_ROUND()
17914 // Avoid folding legal fp_rounds into non-legal ones. in visitFP_ROUND()
17922 // instructions from f32 or f64. Moreover, the first (value-preserving) in visitFP_ROUND()
17930 // single-step fp_round we want to fold to. in visitFP_ROUND()
17941 // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y) in visitFP_ROUND()
17946 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitFP_ROUND()
17963 SDValue N0 = N->getOperand(0); in visitFP_EXTEND()
17964 EVT VT = N->getValueType(0); in visitFP_EXTEND()
17971 if (N->hasOneUse() && in visitFP_EXTEND()
17972 N->use_begin()->getOpcode() == ISD::FP_ROUND) in visitFP_EXTEND()
17975 // fold (fp_extend c1fp) -> c1fp in visitFP_EXTEND()
17979 // fold (fp_extend (fp16_to_fp op)) -> (fp16_to_fp op) in visitFP_EXTEND()
17984 // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the in visitFP_EXTEND()
17996 // fold (fpext (load x)) -> (fpext (fptrunc (extload x))) in visitFP_EXTEND()
18001 LN0->getChain(), in visitFP_EXTEND()
18002 LN0->getBasePtr(), N0.getValueType(), in visitFP_EXTEND()
18003 LN0->getMemOperand()); in visitFP_EXTEND()
18020 SDValue N0 = N->getOperand(0); in visitFCEIL()
18021 EVT VT = N->getValueType(0); in visitFCEIL()
18023 // fold (fceil c1) -> fceil(c1) in visitFCEIL()
18031 SDValue N0 = N->getOperand(0); in visitFTRUNC()
18032 EVT VT = N->getValueType(0); in visitFTRUNC()
18034 // fold (ftrunc c1) -> ftrunc(c1) in visitFTRUNC()
18038 // fold ftrunc (known rounded int x) -> x in visitFTRUNC()
18056 SDValue N0 = N->getOperand(0); in visitFFREXP()
18058 // fold (ffrexp c1) -> ffrexp(c1) in visitFFREXP()
18060 return DAG.getNode(ISD::FFREXP, SDLoc(N), N->getVTList(), N0); in visitFFREXP()
18065 SDValue N0 = N->getOperand(0); in visitFFLOOR()
18066 EVT VT = N->getValueType(0); in visitFFLOOR()
18068 // fold (ffloor c1) -> ffloor(c1) in visitFFLOOR()
18076 SDValue N0 = N->getOperand(0); in visitFNEG()
18077 EVT VT = N->getValueType(0); in visitFNEG()
18088 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 in visitFNEG()
18094 N->getFlags().hasNoSignedZeros()) && N0.hasOneUse()) { in visitFNEG()
18106 SDValue N0 = N->getOperand(0); in visitFMinMax()
18107 SDValue N1 = N->getOperand(1); in visitFMinMax()
18108 EVT VT = N->getValueType(0); in visitFMinMax()
18109 const SDNodeFlags Flags = N->getFlags(); in visitFMinMax()
18110 unsigned Opc = N->getOpcode(); in visitFMinMax()
18122 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitFMinMax()
18125 const APFloat &AF = N1CFP->getValueAPF(); in visitFMinMax()
18127 // minnum(X, nan) -> X in visitFMinMax()
18128 // maxnum(X, nan) -> X in visitFMinMax()
18129 // minimum(X, nan) -> nan in visitFMinMax()
18130 // maximum(X, nan) -> nan in visitFMinMax()
18132 return PropagatesNaN ? N->getOperand(1) : N->getOperand(0); in visitFMinMax()
18137 // minnum(X, -inf) -> -inf in visitFMinMax()
18138 // maxnum(X, +inf) -> +inf in visitFMinMax()
18139 // minimum(X, -inf) -> -inf if nnan in visitFMinMax()
18140 // maximum(X, +inf) -> +inf if nnan in visitFMinMax()
18142 return N->getOperand(1); in visitFMinMax()
18144 // minnum(X, +inf) -> X if nnan in visitFMinMax()
18145 // maxnum(X, -inf) -> X if nnan in visitFMinMax()
18146 // minimum(X, +inf) -> X in visitFMinMax()
18147 // maximum(X, -inf) -> X in visitFMinMax()
18149 return N->getOperand(0); in visitFMinMax()
18164 SDValue N0 = N->getOperand(0); in visitFABS()
18165 EVT VT = N->getValueType(0); in visitFABS()
18167 // fold (fabs c1) -> fabs(c1) in visitFABS()
18171 // fold (fabs (fabs x)) -> (fabs x) in visitFABS()
18173 return N->getOperand(0); in visitFABS()
18175 // fold (fabs (fneg x)) -> (fabs x) in visitFABS()
18176 // fold (fabs (fcopysign x, y)) -> (fabs x) in visitFABS()
18187 SDValue Chain = N->getOperand(0); in visitBRCOND()
18188 SDValue N1 = N->getOperand(1); in visitBRCOND()
18189 SDValue N2 = N->getOperand(2); in visitBRCOND()
18193 if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) { in visitBRCOND()
18195 N1->getOperand(0), N2); in visitBRCOND()
18206 // For example, SETCC(FREEZE(X), -128, SETULT) cannot be folded to in visitBRCOND()
18207 // FREEZE(SETCC(X, -128, SETULT)) because X can be poison. in visitBRCOND()
18208 if (N1->getOpcode() == ISD::SETCC && N1.hasOneUse()) { in visitBRCOND()
18209 SDValue S0 = N1->getOperand(0), S1 = N1->getOperand(1); in visitBRCOND()
18210 ISD::CondCode Cond = cast<CondCodeSDNode>(N1->getOperand(2))->get(); in visitBRCOND()
18217 bool False = (Cond == ISD::SETULT && C->isZero()) || in visitBRCOND()
18218 (Cond == ISD::SETLT && C->isMinSignedValue()) || in visitBRCOND()
18219 (Cond == ISD::SETUGT && C->isAllOnes()) || in visitBRCOND()
18220 (Cond == ISD::SETGT && C->isMaxSignedValue()); in visitBRCOND()
18221 bool True = (Cond == ISD::SETULE && C->isAllOnes()) || in visitBRCOND()
18222 (Cond == ISD::SETLE && C->isMaxSignedValue()) || in visitBRCOND()
18223 (Cond == ISD::SETUGE && C->isZero()) || in visitBRCOND()
18224 (Cond == ISD::SETGE && C->isMinSignedValue()); in visitBRCOND()
18228 if (S0->getOpcode() == ISD::FREEZE && S0.hasOneUse() && S1C) { in visitBRCOND()
18230 S0 = S0->getOperand(0); in visitBRCOND()
18234 if (S1->getOpcode() == ISD::FREEZE && S1.hasOneUse() && S0C) { in visitBRCOND()
18236 S1 = S1->getOperand(0); in visitBRCOND()
18244 DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2); in visitBRCOND()
18299 // SRL constant is equal to the log2 of the AND constant. The back-end is in rebuildSetCC()
18308 const APInt &AndConst = AndOp1->getAsAPIntVal(); in rebuildSetCC()
18311 Op1->getAsAPIntVal() == AndConst.logBase2()) { in rebuildSetCC()
18321 // Transform (brcond (xor x, y)) -> (brcond (setcc, x, y, ne)) in rebuildSetCC()
18322 // Transform (brcond (xor (xor x, y), -1)) -> (brcond (setcc, x, y, eq)) in rebuildSetCC()
18335 // Returning N is form in-visit replacement that may invalidated in rebuildSetCC()
18346 SDValue Op0 = N->getOperand(0); in rebuildSetCC()
18347 SDValue Op1 = N->getOperand(1); in rebuildSetCC()
18351 // (brcond (xor (xor x, y), -1)) -> (brcond (setcc x, y, eq)) in rebuildSetCC()
18355 Op0 = N->getOperand(0); in rebuildSetCC()
18356 Op1 = N->getOperand(1); in rebuildSetCC()
18375 CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1)); in visitBR_CC()
18376 SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3); in visitBR_CC()
18386 CondLHS, CondRHS, CC->get(), SDLoc(N), in visitBR_CC()
18393 N->getOperand(0), Simp.getOperand(2), in visitBR_CC()
18395 N->getOperand(4)); in visitBR_CC()
18404 if (LD->isIndexed()) in getCombineLoadStoreParts()
18406 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts()
18409 Ptr = LD->getBasePtr(); in getCombineLoadStoreParts()
18411 if (ST->isIndexed()) in getCombineLoadStoreParts()
18413 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts()
18416 Ptr = ST->getBasePtr(); in getCombineLoadStoreParts()
18419 if (LD->isIndexed()) in getCombineLoadStoreParts()
18421 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts()
18425 Ptr = LD->getBasePtr(); in getCombineLoadStoreParts()
18428 if (ST->isIndexed()) in getCombineLoadStoreParts()
18430 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts()
18434 Ptr = ST->getBasePtr(); in getCombineLoadStoreParts()
18443 /// Try turning a load/store into a pre-indexed load/store when the base
18462 Ptr->hasOneUse()) in CombineToPreIndexedLoadStore()
18472 // Backends without true r+i pre-indexed forms may need to pass a in CombineToPreIndexedLoadStore()
18485 // Try turning it into a pre-indexed load / store except when: in CombineToPreIndexedLoadStore()
18500 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(N)->getValue() in CombineToPreIndexedLoadStore()
18501 : cast<StoreSDNode>(N)->getValue(); in CombineToPreIndexedLoadStore()
18508 if (Val == Ptr || Ptr->isPredecessorOf(Val.getNode())) in CombineToPreIndexedLoadStore()
18523 for (SDNode::use_iterator UI = BasePtr->use_begin(), in CombineToPreIndexedLoadStore()
18524 UE = BasePtr->use_end(); in CombineToPreIndexedLoadStore()
18536 if (Use.getUser()->getOpcode() != ISD::ADD && in CombineToPreIndexedLoadStore()
18537 Use.getUser()->getOpcode() != ISD::SUB) { in CombineToPreIndexedLoadStore()
18542 SDValue Op1 = Use.getUser()->getOperand((UI.getOperandNo() + 1) & 1); in CombineToPreIndexedLoadStore()
18563 for (SDNode *Use : Ptr->uses()) { in CombineToPreIndexedLoadStore()
18595 LLVM_DEBUG(dbgs() << "\nReplacing.4 "; N->dump(&DAG); dbgs() << "\nWith: "; in CombineToPreIndexedLoadStore()
18614 if (OtherUses[i]->getOperand(OffsetIdx).getNode() == BasePtr.getNode()) in CombineToPreIndexedLoadStore()
18616 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() == in CombineToPreIndexedLoadStore()
18624 // where x0, x1, y0 and y1 in {-1, 1} are given by the types of the in CombineToPreIndexedLoadStore()
18625 // indexed load/store and the expression that needs to be re-written. in CombineToPreIndexedLoadStore()
18628 // t0 = (x0 * offset0 - x1 * y0 * y1 *offset1) + (y0 * y1) * t1 in CombineToPreIndexedLoadStore()
18630 auto *CN = cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx)); in CombineToPreIndexedLoadStore()
18631 const APInt &Offset0 = CN->getAPIntValue(); in CombineToPreIndexedLoadStore()
18632 const APInt &Offset1 = Offset->getAsAPIntVal(); in CombineToPreIndexedLoadStore()
18633 int X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1; in CombineToPreIndexedLoadStore()
18634 int Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1; in CombineToPreIndexedLoadStore()
18635 int X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
18636 int Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
18641 if (X0 < 0) CNV = -CNV; in CombineToPreIndexedLoadStore()
18643 else CNV = CNV - Offset1; in CombineToPreIndexedLoadStore()
18648 SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0)); in CombineToPreIndexedLoadStore()
18653 OtherUses[i]->getValueType(0), NewOp1, NewOp2); in CombineToPreIndexedLoadStore()
18672 (PtrUse->getOpcode() != ISD::ADD && PtrUse->getOpcode() != ISD::SUB)) in shouldCombineToPostInc()
18686 for (SDNode *Use : BasePtr->uses()) { in shouldCombineToPostInc()
18706 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB) { in shouldCombineToPostInc()
18707 for (SDNode *UseUse : Use->uses()) in shouldCombineToPostInc()
18723 Ptr->hasOneUse()) in getPostIndexedLoadStoreOp()
18726 // Try turning it into a post-indexed load / store except when in getPostIndexedLoadStoreOp()
18732 for (SDNode *Op : Ptr->uses()) { in getPostIndexedLoadStoreOp()
18753 /// post-indexed load/store. The transformation folded the add/subtract into the
18784 LLVM_DEBUG(dbgs() << "\nReplacing.5 "; N->dump(&DAG); dbgs() << "\nWith: "; in CombineToPostIndexedLoadStore()
18804 /// Return the base-pointer arithmetic from an indexed \p LD.
18806 ISD::MemIndexedMode AM = LD->getAddressingMode(); in SplitIndexingFromLoad()
18808 SDValue BP = LD->getOperand(1); in SplitIndexingFromLoad()
18809 SDValue Inc = LD->getOperand(2); in SplitIndexingFromLoad()
18815 !cast<ConstantSDNode>(Inc)->isOpaque()) && in SplitIndexingFromLoad()
18819 Inc = DAG.getConstant(*ConstInc->getConstantIntValue(), SDLoc(Inc), in SplitIndexingFromLoad()
18820 ConstInc->getValueType(0)); in SplitIndexingFromLoad()
18834 EVT STMemType = ST->getMemoryVT(); in getTruncatedStoreValue()
18857 EVT LDMemType = LD->getMemoryVT(); in extendLoadedValueToExtension()
18858 EVT LDType = LD->getValueType(0); in extendLoadedValueToExtension()
18860 "Attempting to extend value of non-matching type"); in extendLoadedValueToExtension()
18864 switch (LD->getExtensionType()) { in extendLoadedValueToExtension()
18884 SDValue Chain = LD->getOperand(0); in getUniqueStoreFeeding()
18888 Chain = Chain->getOperand(0); in getUniqueStoreFeeding()
18894 for (SDValue Op : Chain->ops()) { in getUniqueStoreFeeding()
18923 if (OptLevel == CodeGenOptLevel::None || !LD->isSimple()) in ForwardStoreValueToDirectLoad()
18925 SDValue Chain = LD->getOperand(0); in ForwardStoreValueToDirectLoad()
18930 if (!ST || !ST->isSimple() || ST->getAddressSpace() != LD->getAddressSpace()) in ForwardStoreValueToDirectLoad()
18933 EVT LDType = LD->getValueType(0); in ForwardStoreValueToDirectLoad()
18934 EVT LDMemType = LD->getMemoryVT(); in ForwardStoreValueToDirectLoad()
18935 EVT STMemType = ST->getMemoryVT(); in ForwardStoreValueToDirectLoad()
18936 EVT STType = ST->getValue().getValueType(); in ForwardStoreValueToDirectLoad()
18944 // no cost-benefit analysis to prove it's worth it. in ForwardStoreValueToDirectLoad()
18952 // analysis on big-endian platforms it seems better to bail out for now. in ForwardStoreValueToDirectLoad()
18962 Offset = ((int64_t)STMemType.getStoreSizeInBits().getFixedValue() - in ForwardStoreValueToDirectLoad()
18964 8 - in ForwardStoreValueToDirectLoad()
18978 auto ReplaceLd = [&](LoadSDNode *LD, SDValue Val, SDValue Chain) -> SDValue { in ForwardStoreValueToDirectLoad()
18979 if (LD->isIndexed()) { in ForwardStoreValueToDirectLoad()
18996 // Simple case: Direct non-truncating forwarding in ForwardStoreValueToDirectLoad()
18998 return ReplaceLd(LD, ST->getValue(), Chain); in ForwardStoreValueToDirectLoad()
19001 !LDMemType.isVector() && LD->getExtensionType() != ISD::SEXTLOAD) { in ForwardStoreValueToDirectLoad()
19007 auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask); in ForwardStoreValueToDirectLoad()
19012 // Handle some cases for big-endian that would be Offset 0 and handled for in ForwardStoreValueToDirectLoad()
19013 // little-endian. in ForwardStoreValueToDirectLoad()
19014 SDValue Val = ST->getValue(); in ForwardStoreValueToDirectLoad()
19026 if (LD->getBasePtr().isUndef() || Offset != 0) in ForwardStoreValueToDirectLoad()
19049 if (Val->use_empty()) in ForwardStoreValueToDirectLoad()
19056 SDValue Chain = LD->getChain(); in visitLOAD()
19057 SDValue Ptr = LD->getBasePtr(); in visitLOAD()
19063 if (LD->isSimple()) { in visitLOAD()
19064 if (N->getValueType(1) == MVT::Other) { in visitLOAD()
19066 if (!N->hasAnyUseOfValue(0)) { in visitLOAD()
19073 LLVM_DEBUG(dbgs() << "\nReplacing.6 "; N->dump(&DAG); in visitLOAD()
19079 if (N->use_empty()) in visitLOAD()
19086 assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?"); in visitLOAD()
19094 if (!N->hasAnyUseOfValue(0) && (CanSplitIdx || !N->hasAnyUseOfValue(1))) { in visitLOAD()
19095 SDValue Undef = DAG.getUNDEF(N->getValueType(0)); in visitLOAD()
19097 if (N->hasAnyUseOfValue(1) && CanSplitIdx) { in visitLOAD()
19103 Index = DAG.getUNDEF(N->getValueType(1)); in visitLOAD()
19104 LLVM_DEBUG(dbgs() << "\nReplacing.7 "; N->dump(&DAG); in visitLOAD()
19123 if (OptLevel != CodeGenOptLevel::None && LD->isUnindexed() && in visitLOAD()
19124 !LD->isAtomic()) { in visitLOAD()
19126 if (*Alignment > LD->getAlign() && in visitLOAD()
19127 isAligned(*Alignment, LD->getSrcValueOffset())) { in visitLOAD()
19129 LD->getExtensionType(), SDLoc(N), LD->getValueType(0), Chain, Ptr, in visitLOAD()
19130 LD->getPointerInfo(), LD->getMemoryVT(), *Alignment, in visitLOAD()
19131 LD->getMemOperand()->getFlags(), LD->getAAInfo()); in visitLOAD()
19139 if (LD->isUnindexed()) { in visitLOAD()
19140 // Walk up chain skipping non-aliasing memory nodes. in visitLOAD()
19148 if (LD->getExtensionType() == ISD::NON_EXTLOAD) { in visitLOAD()
19149 ReplLoad = DAG.getLoad(N->getValueType(0), SDLoc(LD), in visitLOAD()
19150 BetterChain, Ptr, LD->getMemOperand()); in visitLOAD()
19152 ReplLoad = DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), in visitLOAD()
19153 LD->getValueType(0), in visitLOAD()
19154 BetterChain, Ptr, LD->getMemoryVT(), in visitLOAD()
19155 LD->getMemOperand()); in visitLOAD()
19211 EVT TruncType = LS.Inst->getValueType(0); in Cost()
19214 !LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType)) in Cost()
19224 const TargetLowering &TLI = LS.DAG->getTargetLoweringInfo(); in addSliceGain()
19225 if (!TLI.isTruncateFree(LS.Inst->getOperand(0), LS.Inst->getValueType(0))) in addSliceGain()
19295 // - Start from the truncated value. in getUsedBits()
19296 // - Zero extend to the desired bit width. in getUsedBits()
19297 // - Shift left. in getUsedBits()
19299 unsigned BitWidth = Origin->getValueSizeInBits(0); in getUsedBits()
19301 assert(Inst->getValueSizeInBits(0) <= BitWidth && in getUsedBits()
19303 APInt UsedBits(Inst->getValueSizeInBits(0), 0); in getUsedBits()
19321 LLVMContext &Ctxt = *DAG->getContext(); in getLoadedType()
19327 Align Alignment = Origin->getAlign(); in getAlign()
19341 if (!Origin->getOffset().isUndef()) in isLegal()
19344 const TargetLowering &TLI = DAG->getTargetLoweringInfo(); in isLegal()
19357 EVT PtrType = Origin->getBasePtr().getValueType(); in isLegal()
19370 EVT TruncateType = Inst->getValueType(0); in isLegal()
19383 bool IsBigEndian = DAG->getDataLayout().isBigEndian(); in getOffsetFromBase()
19386 unsigned TySizeInBytes = Origin->getValueSizeInBits(0) / 8; in getOffsetFromBase()
19387 assert(!(Origin->getValueSizeInBits(0) & 0x7) && in getOffsetFromBase()
19395 Offset = TySizeInBytes - Offset - getLoadedSize(); in getOffsetFromBase()
19402 /// \pre this->Inst && this->Origin are valid Instructions and this
19406 assert(Inst && Origin && "Unable to replace a non-existing slice."); in loadSlice()
19407 const SDValue &OldBaseAddr = Origin->getBasePtr(); in loadSlice()
19416 BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr, in loadSlice()
19417 DAG->getConstant(Offset, DL, ArithType)); in loadSlice()
19425 DAG->getLoad(SliceType, SDLoc(Origin), Origin->getChain(), BaseAddr, in loadSlice()
19426 Origin->getPointerInfo().getWithOffset(Offset), getAlign(), in loadSlice()
19427 Origin->getMemOperand()->getFlags()); in loadSlice()
19430 EVT FinalType = Inst->getValueType(0); in loadSlice()
19433 DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst); in loadSlice()
19442 if (!Inst || !Inst->hasOneUse()) in canMergeExpensiveCrossRegisterBankCopy()
19444 SDNode *Use = *Inst->use_begin(); in canMergeExpensiveCrossRegisterBankCopy()
19445 if (Use->getOpcode() != ISD::BITCAST) in canMergeExpensiveCrossRegisterBankCopy()
19448 const TargetLowering &TLI = DAG->getTargetLoweringInfo(); in canMergeExpensiveCrossRegisterBankCopy()
19449 EVT ResVT = Use->getValueType(0); in canMergeExpensiveCrossRegisterBankCopy()
19451 TLI.getRegClassFor(ResVT.getSimpleVT(), Use->isDivergent()); in canMergeExpensiveCrossRegisterBankCopy()
19453 TLI.getRegClassFor(Use->getOperand(0).getValueType().getSimpleVT(), in canMergeExpensiveCrossRegisterBankCopy()
19454 Use->getOperand(0)->isDivergent()); in canMergeExpensiveCrossRegisterBankCopy()
19458 // At this point, we know that we perform a cross-register-bank copy. in canMergeExpensiveCrossRegisterBankCopy()
19460 const TargetRegisterInfo *TRI = DAG->getSubtarget().getRegisterInfo(); in canMergeExpensiveCrossRegisterBankCopy()
19463 if (!TRI || TRI->getCommonSubClass(ArgRC, ResRC)) in canMergeExpensiveCrossRegisterBankCopy()
19469 if (!TLI.allowsMemoryAccess(*DAG->getContext(), DAG->getDataLayout(), ResVT, in canMergeExpensiveCrossRegisterBankCopy()
19470 Origin->getAddressSpace(), getAlign(), in canMergeExpensiveCrossRegisterBankCopy()
19471 Origin->getMemOperand()->getFlags(), &IsFast) || in canMergeExpensiveCrossRegisterBankCopy()
19480 if (Inst->getValueType(0) != getLoadedType()) in canMergeExpensiveCrossRegisterBankCopy()
19536 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo(); in adjustCostForPairing()
19551 EVT LoadedType = First->getLoadedType(); in adjustCostForPairing()
19554 if (LoadedType != Second->getLoadedType()) in adjustCostForPairing()
19565 if (First->getAlign() < RequiredAlignment) in adjustCostForPairing()
19573 --GlobalLSCost.Loads; in adjustCostForPairing()
19634 if (!LD->isSimple() || !ISD::isNormalLoad(LD) || in SliceUpLoad()
19635 !LD->getValueType(0).isInteger()) in SliceUpLoad()
19641 if (LD->getValueType(0).isScalableVector()) in SliceUpLoad()
19646 APInt UsedBits(LD->getValueSizeInBits(0), 0); in SliceUpLoad()
19653 for (SDNode::use_iterator UI = LD->use_begin(), UIEnd = LD->use_end(); in SliceUpLoad()
19663 if (User->getOpcode() == ISD::SRL && User->hasOneUse() && in SliceUpLoad()
19664 isa<ConstantSDNode>(User->getOperand(1))) { in SliceUpLoad()
19665 Shift = User->getConstantOperandVal(1); in SliceUpLoad()
19666 User = *User->use_begin(); in SliceUpLoad()
19671 if (User->getOpcode() != ISD::TRUNCATE) in SliceUpLoad()
19674 // The width of the type must be a power of 2 and greater than 8-bits. in SliceUpLoad()
19676 // Moreover, if we shifted with a non-8-bits multiple, the slice in SliceUpLoad()
19678 unsigned Width = User->getValueSizeInBits(0); in SliceUpLoad()
19716 assert(SliceInst->getOpcode() == ISD::LOAD && in SliceUpLoad()
19736 if (V->getOpcode() != ISD::AND || in CheckForMaskedLoad()
19737 !isa<ConstantSDNode>(V->getOperand(1)) || in CheckForMaskedLoad()
19738 !ISD::isNormalLoad(V->getOperand(0).getNode())) in CheckForMaskedLoad()
19742 LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0)); in CheckForMaskedLoad()
19743 if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer. in CheckForMaskedLoad()
19754 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue(); in CheckForMaskedLoad()
19767 NotMaskLZ -= 64-V.getValueSizeInBits(); in CheckForMaskedLoad()
19769 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8; in CheckForMaskedLoad()
19774 default: return Result; // All one mask, or 5-byte mask. in CheckForMaskedLoad()
19785 else if (Chain->getOpcode() == ISD::TokenFactor && in CheckForMaskedLoad()
19788 if (!LD->isOperandOf(Chain.getNode())) in CheckForMaskedLoad()
19807 SelectionDAG &DAG = DC->getDAG(); in ShrinkLoadReplaceStoreWithStore()
19822 if (DC->isTypeLegal(VT)) in ShrinkLoadReplaceStoreWithStore()
19831 if (St->isIndexed()) in ShrinkLoadReplaceStoreWithStore()
19835 if (St->getMemOperand() && in ShrinkLoadReplaceStoreWithStore()
19837 *St->getMemOperand())) in ShrinkLoadReplaceStoreWithStore()
19854 StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes; in ShrinkLoadReplaceStoreWithStore()
19856 SDValue Ptr = St->getBasePtr(); in ShrinkLoadReplaceStoreWithStore()
19864 return DAG.getTruncStore(St->getChain(), SDLoc(St), IVal, Ptr, in ShrinkLoadReplaceStoreWithStore()
19865 St->getPointerInfo().getWithOffset(StOffset), in ShrinkLoadReplaceStoreWithStore()
19866 VT, St->getOriginalAlign()); in ShrinkLoadReplaceStoreWithStore()
19872 .getStore(St->getChain(), SDLoc(St), IVal, Ptr, in ShrinkLoadReplaceStoreWithStore()
19873 St->getPointerInfo().getWithOffset(StOffset), in ShrinkLoadReplaceStoreWithStore()
19874 St->getOriginalAlign()); in ShrinkLoadReplaceStoreWithStore()
19883 if (!ST->isSimple()) in ReduceLoadOpStoreWidth()
19886 SDValue Chain = ST->getChain(); in ReduceLoadOpStoreWidth()
19887 SDValue Value = ST->getValue(); in ReduceLoadOpStoreWidth()
19888 SDValue Ptr = ST->getBasePtr(); in ReduceLoadOpStoreWidth()
19891 if (ST->isTruncatingStore() || VT.isVector()) in ReduceLoadOpStoreWidth()
19931 if (LD->getBasePtr() != Ptr || in ReduceLoadOpStoreWidth()
19932 LD->getPointerInfo().getAddrSpace() != in ReduceLoadOpStoreWidth()
19933 ST->getPointerInfo().getAddrSpace()) in ReduceLoadOpStoreWidth()
19939 APInt Imm = N1->getAsAPIntVal(); in ReduceLoadOpStoreWidth()
19945 unsigned MSB = BitWidth - Imm.countl_zero() - 1; in ReduceLoadOpStoreWidth()
19946 unsigned NewBW = NextPowerOf2(MSB - ShAmt); in ReduceLoadOpStoreWidth()
19963 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW; in ReduceLoadOpStoreWidth()
19974 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff; in ReduceLoadOpStoreWidth()
19977 Align NewAlign = commonAlignment(LD->getAlign(), PtrOff); in ReduceLoadOpStoreWidth()
19979 LD->getAddressSpace(), NewAlign, in ReduceLoadOpStoreWidth()
19980 LD->getMemOperand()->getFlags(), &IsFast) || in ReduceLoadOpStoreWidth()
19987 DAG.getLoad(NewVT, SDLoc(N0), LD->getChain(), NewPtr, in ReduceLoadOpStoreWidth()
19988 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign, in ReduceLoadOpStoreWidth()
19989 LD->getMemOperand()->getFlags(), LD->getAAInfo()); in ReduceLoadOpStoreWidth()
19995 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign); in ReduceLoadOpStoreWidth()
20015 SDValue Value = ST->getValue(); in TransformFPLoadStorePair()
20019 EVT VT = LD->getMemoryVT(); in TransformFPLoadStorePair()
20021 VT != ST->getMemoryVT() || in TransformFPLoadStorePair()
20022 LD->isNonTemporal() || in TransformFPLoadStorePair()
20023 ST->isNonTemporal() || in TransformFPLoadStorePair()
20024 LD->getPointerInfo().getAddrSpace() != 0 || in TransformFPLoadStorePair()
20025 ST->getPointerInfo().getAddrSpace() != 0) in TransformFPLoadStorePair()
20042 *LD->getMemOperand(), &FastLD) || in TransformFPLoadStorePair()
20044 *ST->getMemOperand(), &FastST) || in TransformFPLoadStorePair()
20049 DAG.getLoad(IntVT, SDLoc(Value), LD->getChain(), LD->getBasePtr(), in TransformFPLoadStorePair()
20050 LD->getPointerInfo(), LD->getAlign()); in TransformFPLoadStorePair()
20053 DAG.getStore(ST->getChain(), SDLoc(N), NewLD, ST->getBasePtr(), in TransformFPLoadStorePair()
20054 ST->getPointerInfo(), ST->getAlign()); in TransformFPLoadStorePair()
20068 // of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
20086 if (AddNode->hasOneUse() && in isMulAddWithConstProfitable()
20091 for (SDNode *Use : ConstNode->uses()) { in isMulAddWithConstProfitable()
20095 if (Use->getOpcode() == ISD::MUL) { // We have another multiply use. in isMulAddWithConstProfitable()
20100 if (Use->getOperand(0) == ConstNode) in isMulAddWithConstProfitable()
20101 OtherOp = Use->getOperand(1).getNode(); in isMulAddWithConstProfitable()
20103 OtherOp = Use->getOperand(0).getNode(); in isMulAddWithConstProfitable()
20108 // Use = ConstNode * A <-- visiting Use. OtherOp is A. in isMulAddWithConstProfitable()
20110 // AddNode = (A + c1) <-- MulVar is A. in isMulAddWithConstProfitable()
20111 // = AddNode * ConstNode <-- current visiting instruction. in isMulAddWithConstProfitable()
20123 // ... = AddNode * ConstNode <-- current visiting instruction. in isMulAddWithConstProfitable()
20126 // Use = OtherOp * ConstNode <-- visiting Use. in isMulAddWithConstProfitable()
20131 if (OtherOp->getOpcode() == ISD::ADD && in isMulAddWithConstProfitable()
20132 DAG.isConstantIntBuildVectorOrConstantInt(OtherOp->getOperand(1)) && in isMulAddWithConstProfitable()
20133 OtherOp->getOperand(0).getNode() == MulVar) in isMulAddWithConstProfitable()
20154 if (Visited.insert(StoreNodes[i].MemNode->getChain().getNode()).second) in getMergeStoreChains()
20155 Chains.push_back(StoreNodes[i].MemNode->getChain()); in getMergeStoreChains()
20165 const MachineMemOperand *MMO = MemOp.MemNode->getMemOperand(); in hasSameUnderlyingObj()
20168 if (MMO->getPseudoValue()) in hasSameUnderlyingObj()
20171 if (!MMO->getValue()) in hasSameUnderlyingObj()
20174 const Value *Obj = getUnderlyingObject(MMO->getValue()); in hasSameUnderlyingObj()
20208 Flags = St->getMemOperand()->getFlags(); in mergeStoresOfConstantsOrVecElts()
20209 AAInfo = St->getAAInfo(); in mergeStoresOfConstantsOrVecElts()
20213 if (Flags != St->getMemOperand()->getFlags()) in mergeStoresOfConstantsOrVecElts()
20216 AAInfo = AAInfo.concat(St->getAAInfo()); in mergeStoresOfConstantsOrVecElts()
20233 SDValue Val = St->getValue(); in mergeStoresOfConstantsOrVecElts()
20248 Val = DAG.getConstant(C->getAPIntValue() in mergeStoresOfConstantsOrVecElts()
20265 SDValue Val = peekThroughBitcasts(St->getValue()); in mergeStoresOfConstantsOrVecElts()
20309 unsigned Idx = IsLE ? (NumStores - 1 - i) : i; in mergeStoresOfConstantsOrVecElts()
20312 SDValue Val = St->getValue(); in mergeStoresOfConstantsOrVecElts()
20316 StoreInt |= C->getAPIntValue() in mergeStoresOfConstantsOrVecElts()
20320 StoreInt |= C->getValueAPF() in mergeStoresOfConstantsOrVecElts()
20352 NewChain, DL, StoredVal, FirstInChain->getBasePtr(), in mergeStoresOfConstantsOrVecElts()
20354 ? FirstInChain->getPointerInfo() in mergeStoresOfConstantsOrVecElts()
20355 : MachinePointerInfo(FirstInChain->getPointerInfo().getAddrSpace()), in mergeStoresOfConstantsOrVecElts()
20356 FirstInChain->getAlign(), *Flags, AAInfo); in mergeStoresOfConstantsOrVecElts()
20363 DAG.getConstant(C->getAPIntValue().zextOrTrunc(LegalizedStoreSize), DL, in mergeStoresOfConstantsOrVecElts()
20366 NewChain, DL, ExtendedStoreVal, FirstInChain->getBasePtr(), in mergeStoresOfConstantsOrVecElts()
20368 ? FirstInChain->getPointerInfo() in mergeStoresOfConstantsOrVecElts()
20369 : MachinePointerInfo(FirstInChain->getPointerInfo().getAddrSpace()), in mergeStoresOfConstantsOrVecElts()
20370 StoredVal.getValueType() /*TVT*/, FirstInChain->getAlign(), *Flags, in mergeStoresOfConstantsOrVecElts()
20392 SDValue Val = peekThroughBitcasts(St->getValue()); in getStoreMergeCandidates()
20397 EVT MemVT = St->getMemoryVT(); in getStoreMergeCandidates()
20403 LoadVT = Ld->getMemoryVT(); in getStoreMergeCandidates()
20408 if (!Ld->hasNUsesOfValue(1, 0)) in getStoreMergeCandidates()
20412 if (!Ld->isSimple() || Ld->isIndexed()) in getStoreMergeCandidates()
20416 int64_t &Offset) -> bool { in getStoreMergeCandidates()
20419 if (!Other->isSimple() || Other->isIndexed()) in getStoreMergeCandidates()
20421 // Don't mix temporal stores with non-temporal stores. in getStoreMergeCandidates()
20422 if (St->isNonTemporal() != Other->isNonTemporal()) in getStoreMergeCandidates()
20426 SDValue OtherBC = peekThroughBitcasts(Other->getValue()); in getStoreMergeCandidates()
20428 bool NoTypeMatch = (MemVT.isInteger()) ? !MemVT.bitsEq(Other->getMemoryVT()) in getStoreMergeCandidates()
20429 : Other->getMemoryVT() != MemVT; in getStoreMergeCandidates()
20439 if (LoadVT != OtherLd->getMemoryVT()) in getStoreMergeCandidates()
20442 if (!OtherLd->hasNUsesOfValue(1, 0)) in getStoreMergeCandidates()
20446 if (!OtherLd->isSimple() || OtherLd->isIndexed()) in getStoreMergeCandidates()
20448 // Don't mix temporal loads with non-temporal loads. in getStoreMergeCandidates()
20449 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal()) in getStoreMergeCandidates()
20466 if (Other->isTruncatingStore()) in getStoreMergeCandidates()
20489 // |-------|-------| in getStoreMergeCandidates()
20497 SDNode *RootNode = St->getChain().getNode(); in getStoreMergeCandidates()
20505 SDNode *RootNode) -> bool { in getStoreMergeCandidates()
20508 RootCount->second.first == RootNode && in getStoreMergeCandidates()
20509 RootCount->second.second > StoreMergeDependenceLimit; in getStoreMergeCandidates()
20528 RootNode = Ldn->getChain().getNode(); in getStoreMergeCandidates()
20532 for (auto I = RootNode->use_begin(), E = RootNode->use_end(); in getStoreMergeCandidates()
20535 for (auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2) in getStoreMergeCandidates()
20544 for (auto I = RootNode->use_begin(), E = RootNode->use_end(); in getStoreMergeCandidates()
20575 if (N->getOpcode() == ISD::TokenFactor) { in checkMergeStoreCandidatesForDependencies()
20576 for (SDValue Op : N->ops()) in checkMergeStoreCandidatesForDependencies()
20587 // * Chain (Op 0) -> We have already considered these in checkMergeStoreCandidatesForDependencies()
20590 // dependency to a load, that has a non-chain dep to in checkMergeStoreCandidatesForDependencies()
20593 // of chain and non-chain deps, and we need to include in checkMergeStoreCandidatesForDependencies()
20595 // * Value (Op 1) -> Cycles may happen (e.g. through load chains) in checkMergeStoreCandidatesForDependencies()
20596 // * Address (Op 2) -> Merged addresses may only vary by a fixed constant, in checkMergeStoreCandidatesForDependencies()
20599 // * (Op 3) -> Represents the pre or post-indexing offset (or undef for in checkMergeStoreCandidatesForDependencies()
20600 // non-indexed stores). Not constant on all targets (e.g. ARM) in checkMergeStoreCandidatesForDependencies()
20602 for (const SDValue &Op : N->op_values()) in checkMergeStoreCandidatesForDependencies()
20644 // non-consecutive store memory address. in getConsecutiveStores()
20651 if (CurrAddress - StartAddress != (ElementSizeBytes * i)) in getConsecutiveStores()
20676 unsigned FirstStoreAS = FirstInChain->getAddressSpace(); in tryStoreMergeOfConstants()
20677 Align FirstStoreAlign = FirstInChain->getAlign(); in tryStoreMergeOfConstants()
20685 SDValue StoredVal = ST->getValue(); in tryStoreMergeOfConstants()
20688 IsElementZero = C->isZero(); in tryStoreMergeOfConstants()
20690 IsElementZero = C->getConstantFPValue()->isNullValue(); in tryStoreMergeOfConstants()
20712 *FirstInChain->getMemOperand(), &IsFast) && in tryStoreMergeOfConstants()
20725 *FirstInChain->getMemOperand(), &IsFast) && in tryStoreMergeOfConstants()
20742 *FirstInChain->getMemOperand(), &IsFast) && in tryStoreMergeOfConstants()
20760 // improved or we've dropped a non-zero value. Drop as many in tryStoreMergeOfConstants()
20765 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign)) in tryStoreMergeOfConstants()
20769 NumConsecutiveStores -= NumSkip; in tryStoreMergeOfConstants()
20777 NumConsecutiveStores -= NumElem; in tryStoreMergeOfConstants()
20787 NumConsecutiveStores -= NumElem; in tryStoreMergeOfConstants()
20803 unsigned FirstStoreAS = FirstInChain->getAddressSpace(); in tryStoreMergeOfExtracts()
20804 Align FirstStoreAlign = FirstInChain->getAlign(); in tryStoreMergeOfExtracts()
20819 *FirstInChain->getMemOperand(), &IsFast) && in tryStoreMergeOfExtracts()
20835 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign)) in tryStoreMergeOfExtracts()
20839 NumConsecutiveStores -= NumSkip; in tryStoreMergeOfExtracts()
20848 NumConsecutiveStores -= NumStoresToMerge; in tryStoreMergeOfExtracts()
20857 NumConsecutiveStores -= NumStoresToMerge; in tryStoreMergeOfExtracts()
20882 SDValue Val = peekThroughBitcasts(St->getValue()); in tryStoreMergeOfLoads()
20908 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) { in tryStoreMergeOfLoads()
20917 if (Offset0 - Offset1 == ElementSizeBytes && in tryStoreMergeOfLoads()
20925 unsigned FirstStoreAS = FirstInChain->getAddressSpace(); in tryStoreMergeOfLoads()
20926 Align FirstStoreAlign = FirstInChain->getAlign(); in tryStoreMergeOfLoads()
20930 // non-consecutive load memory address. These variables hold the index in in tryStoreMergeOfLoads()
20941 SDValue LoadChain = FirstLoad->getChain(); in tryStoreMergeOfLoads()
20944 if (LoadNodes[i].MemNode->getChain() != LoadChain) in tryStoreMergeOfLoads()
20948 if (CurrAddress - StartAddress != (ElementSizeBytes * i)) in tryStoreMergeOfLoads()
20952 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable()) in tryStoreMergeOfLoads()
20973 *FirstInChain->getMemOperand(), &IsFastSt) && in tryStoreMergeOfLoads()
20976 *FirstLoad->getMemOperand(), &IsFastLd) && in tryStoreMergeOfLoads()
20988 *FirstInChain->getMemOperand(), &IsFastSt) && in tryStoreMergeOfLoads()
20991 *FirstLoad->getMemOperand(), &IsFastLd) && in tryStoreMergeOfLoads()
21006 *FirstInChain->getMemOperand(), &IsFastSt) && in tryStoreMergeOfLoads()
21009 *FirstLoad->getMemOperand(), &IsFastLd) && in tryStoreMergeOfLoads()
21028 Align FirstLoadAlign = FirstLoad->getAlign(); in tryStoreMergeOfLoads()
21040 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) && in tryStoreMergeOfLoads()
21041 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign)) in tryStoreMergeOfLoads()
21045 NumConsecutiveStores -= NumSkip; in tryStoreMergeOfLoads()
21054 NumConsecutiveStores -= NumElem; in tryStoreMergeOfLoads()
21097 JointMemOpVT, LoadDL, FirstLoad->getChain(), FirstLoad->getBasePtr(), in tryStoreMergeOfLoads()
21098 FirstLoad->getPointerInfo(), FirstLoadAlign, LdMMOFlags); in tryStoreMergeOfLoads()
21103 "Unexpected type for rotate-able load pair"); in tryStoreMergeOfLoads()
21110 NewStoreChain, StoreDL, StoreOp, FirstInChain->getBasePtr(), in tryStoreMergeOfLoads()
21111 CanReusePtrInfo ? FirstInChain->getPointerInfo() in tryStoreMergeOfLoads()
21118 FirstLoad->getChain(), FirstLoad->getBasePtr(), in tryStoreMergeOfLoads()
21119 FirstLoad->getPointerInfo(), JointMemOpVT, in tryStoreMergeOfLoads()
21122 NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(), in tryStoreMergeOfLoads()
21123 CanReusePtrInfo ? FirstInChain->getPointerInfo() in tryStoreMergeOfLoads()
21125 JointMemOpVT, FirstInChain->getAlign(), in tryStoreMergeOfLoads()
21126 FirstInChain->getMemOperand()->getFlags()); in tryStoreMergeOfLoads()
21139 SDValue Val = StoreNodes[i].MemNode->getOperand(1); in tryStoreMergeOfLoads()
21141 if (Val->use_empty()) in tryStoreMergeOfLoads()
21148 NumConsecutiveStores -= NumElem; in tryStoreMergeOfLoads()
21161 EVT MemVT = St->getMemoryVT(); in mergeConsecutiveStores()
21167 // This function cannot currently deal with non-byte-sized memory sizes. in mergeConsecutiveStores()
21174 SDValue StoredVal = peekThroughBitcasts(St->getValue()); in mergeConsecutiveStores()
21180 // Find potential store merge candidates by searching through chain sub-DAG in mergeConsecutiveStores()
21195 bool IsNonTemporalStore = St->isNonTemporal(); in mergeConsecutiveStores()
21197 cast<LoadSDNode>(StoredVal)->isNonTemporal(); in mergeConsecutiveStores()
21202 // case that a non-mergeable store is found first, e.g., {p[-2], in mergeConsecutiveStores()
21250 if (ST->isTruncatingStore()) { in replaceStoreChain()
21251 ReplStore = DAG.getTruncStore(BetterChain, SL, ST->getValue(), in replaceStoreChain()
21252 ST->getBasePtr(), ST->getMemoryVT(), in replaceStoreChain()
21253 ST->getMemOperand()); in replaceStoreChain()
21255 ReplStore = DAG.getStore(BetterChain, SL, ST->getValue(), ST->getBasePtr(), in replaceStoreChain()
21256 ST->getMemOperand()); in replaceStoreChain()
21261 MVT::Other, ST->getChain(), ReplStore); in replaceStoreChain()
21271 SDValue Value = ST->getValue(); in replaceStoreOfFPConstant()
21280 SDValue Chain = ST->getChain(); in replaceStoreOfFPConstant()
21281 SDValue Ptr = ST->getBasePtr(); in replaceStoreOfFPConstant()
21286 // the number of stores. For example, on x86-32 an f64 can be stored in one in replaceStoreOfFPConstant()
21291 switch (CFP->getSimpleValueType(0).SimpleTy) { in replaceStoreOfFPConstant()
21301 if ((isTypeLegal(MVT::i32) && !LegalOperations && ST->isSimple()) || in replaceStoreOfFPConstant()
21303 Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF(). in replaceStoreOfFPConstant()
21306 return DAG.getStore(Chain, DL, Tmp, Ptr, ST->getMemOperand()); in replaceStoreOfFPConstant()
21312 ST->isSimple()) || in replaceStoreOfFPConstant()
21314 Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). in replaceStoreOfFPConstant()
21317 Ptr, ST->getMemOperand()); in replaceStoreOfFPConstant()
21320 if (ST->isSimple() && TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32) && in replaceStoreOfFPConstant()
21321 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { in replaceStoreOfFPConstant()
21324 // 64-bit integer store into two 32-bit stores. in replaceStoreOfFPConstant()
21325 uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); in replaceStoreOfFPConstant()
21331 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); in replaceStoreOfFPConstant()
21332 AAMDNodes AAInfo = ST->getAAInfo(); in replaceStoreOfFPConstant()
21334 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(), in replaceStoreOfFPConstant()
21335 ST->getOriginalAlign(), MMOFlags, AAInfo); in replaceStoreOfFPConstant()
21338 ST->getPointerInfo().getWithOffset(4), in replaceStoreOfFPConstant()
21339 ST->getOriginalAlign(), MMOFlags, AAInfo); in replaceStoreOfFPConstant()
21348 // (store (insert_vector_elt (load p), x, i), p) -> (store x, p+offset)
21355 SDValue Value = ST->getValue(); in replaceStoreOfInsertLoad()
21356 SDValue Ptr = ST->getBasePtr(); in replaceStoreOfInsertLoad()
21357 SDValue Chain = ST->getChain(); in replaceStoreOfInsertLoad()
21372 if (!Ld || Ld->getBasePtr() != Ptr || in replaceStoreOfInsertLoad()
21373 ST->getMemoryVT() != Ld->getMemoryVT() || !ST->isSimple() || in replaceStoreOfInsertLoad()
21375 Ld->getAddressSpace() != ST->getAddressSpace() || in replaceStoreOfInsertLoad()
21381 Elt.getValueType(), ST->getAddressSpace(), in replaceStoreOfInsertLoad()
21382 ST->getAlign(), ST->getMemOperand()->getFlags(), in replaceStoreOfInsertLoad()
21387 MachinePointerInfo PointerInfo(ST->getAddressSpace()); in replaceStoreOfInsertLoad()
21393 unsigned COffset = CIdx->getSExtValue() * EltVT.getSizeInBits() / 8; in replaceStoreOfInsertLoad()
21395 PointerInfo = ST->getPointerInfo().getWithOffset(COffset); in replaceStoreOfInsertLoad()
21400 return DAG.getStore(Chain, DL, Elt, NewPtr, PointerInfo, ST->getAlign(), in replaceStoreOfInsertLoad()
21401 ST->getMemOperand()->getFlags()); in replaceStoreOfInsertLoad()
21406 SDValue Val = ST->getVal(); in visitATOMIC_STORE()
21408 EVT MemVT = ST->getMemoryVT(); in visitATOMIC_STORE()
21424 SDValue Chain = ST->getChain(); in visitSTORE()
21425 SDValue Value = ST->getValue(); in visitSTORE()
21426 SDValue Ptr = ST->getBasePtr(); in visitSTORE()
21430 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() && in visitSTORE()
21431 ST->isUnindexed()) { in visitSTORE()
21439 if (((!LegalOperations && ST->isSimple()) || in visitSTORE()
21442 DAG, *ST->getMemOperand())) { in visitSTORE()
21444 ST->getMemOperand()); in visitSTORE()
21448 // Turn 'store undef, Ptr' -> nothing. in visitSTORE()
21449 if (Value.isUndef() && ST->isUnindexed()) in visitSTORE()
21453 if (OptLevel != CodeGenOptLevel::None && ST->isUnindexed() && in visitSTORE()
21454 !ST->isAtomic()) { in visitSTORE()
21456 if (*Alignment > ST->getAlign() && in visitSTORE()
21457 isAligned(*Alignment, ST->getSrcValueOffset())) { in visitSTORE()
21459 DAG.getTruncStore(Chain, SDLoc(N), Value, Ptr, ST->getPointerInfo(), in visitSTORE()
21460 ST->getMemoryVT(), *Alignment, in visitSTORE()
21461 ST->getMemOperand()->getFlags(), ST->getAAInfo()); in visitSTORE()
21478 if (ST->isUnindexed()) { in visitSTORE()
21479 // Walk up chain skipping non-aliasing memory nodes, on this store and any in visitSTORE()
21486 Chain = ST->getChain(); in visitSTORE()
21490 if (ST->isTruncatingStore() && ST->isUnindexed() && in visitSTORE()
21493 !cast<ConstantSDNode>(Value)->isOpaque())) { in visitSTORE()
21498 Value.getOperand(0).getValueType() == ST->getMemoryVT() && in visitSTORE()
21499 TLI.isOperationLegalOrCustom(ISD::STORE, ST->getMemoryVT())) in visitSTORE()
21501 ST->getMemOperand()); in visitSTORE()
21505 ST->getMemoryVT().getScalarSizeInBits()); in visitSTORE()
21511 // Re-visit the store if anything changed and the store hasn't been merged in visitSTORE()
21513 // node back to the worklist if necessary, but we also need to re-visit in visitSTORE()
21515 if (N->getOpcode() != ISD::DELETED_NODE) in visitSTORE()
21522 // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8" in visitSTORE()
21525 return DAG.getTruncStore(Chain, SDLoc(N), Shorter, Ptr, ST->getMemoryVT(), in visitSTORE()
21526 ST->getMemOperand()); in visitSTORE()
21531 if (!Cst->isOpaque()) { in visitSTORE()
21532 const APInt &CValue = Cst->getAPIntValue(); in visitSTORE()
21538 ST->getMemoryVT(), ST->getMemOperand()); in visitSTORE()
21545 // TODO: Add big-endian truncate support with test coverage. in visitSTORE()
21551 if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() && in visitSTORE()
21552 ST->isUnindexed() && ST->isSimple() && in visitSTORE()
21553 Ld->getAddressSpace() == ST->getAddressSpace() && in visitSTORE()
21568 if (ST->isUnindexed() && ST->isSimple() && in visitSTORE()
21569 ST1->isUnindexed() && ST1->isSimple()) { in visitSTORE()
21570 if (OptLevel != CodeGenOptLevel::None && ST1->getBasePtr() == Ptr && in visitSTORE()
21571 ST1->getValue() == Value && ST->getMemoryVT() == ST1->getMemoryVT() && in visitSTORE()
21572 ST->getAddressSpace() == ST1->getAddressSpace()) { in visitSTORE()
21578 if (OptLevel != CodeGenOptLevel::None && ST1->hasOneUse() && in visitSTORE()
21579 !ST1->getBasePtr().isUndef() && in visitSTORE()
21580 ST->getAddressSpace() == ST1->getAddressSpace()) { in visitSTORE()
21585 if (ST->getMemoryVT().isScalableVector() || in visitSTORE()
21586 ST1->getMemoryVT().isScalableVector()) { in visitSTORE()
21587 if (ST1->getBasePtr() == Ptr && in visitSTORE()
21588 TypeSize::isKnownLE(ST1->getMemoryVT().getStoreSize(), in visitSTORE()
21589 ST->getMemoryVT().getStoreSize())) { in visitSTORE()
21590 CombineTo(ST1, ST1->getChain()); in visitSTORE()
21600 if (STBase.contains(DAG, ST->getMemoryVT().getFixedSizeInBits(), in visitSTORE()
21602 ST1->getMemoryVT().getFixedSizeInBits())) { in visitSTORE()
21603 CombineTo(ST1, ST1->getChain()); in visitSTORE()
21615 Value->hasOneUse() && ST->isUnindexed() && in visitSTORE()
21617 ST->getMemoryVT(), LegalOperations)) { in visitSTORE()
21619 Ptr, ST->getMemoryVT(), ST->getMemOperand()); in visitSTORE()
21625 if (!LegalTypes || (TLI.mergeStoresAfterLegalization(ST->getMemoryVT()))) { in visitSTORE()
21634 if (N->getOpcode() == ISD::DELETED_NODE || !isa<StoreSDNode>(N)) in visitSTORE()
21643 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' in visitSTORE()
21648 if (isa<ConstantFPSDNode>(ST->getValue())) { in visitSTORE()
21661 if (!LifetimeEnd->hasOffset()) in visitLIFETIME_END()
21664 const BaseIndexOffset LifetimeEndBase(N->getOperand(1), SDValue(), in visitLIFETIME_END()
21665 LifetimeEnd->getOffset(), false); in visitLIFETIME_END()
21668 SmallVector<SDValue, 8> Chains = {N->getOperand(0)}; in visitLIFETIME_END()
21676 Chains.push_back(Chain.getOperand(--Nops)); in visitLIFETIME_END()
21688 if (!ST->isSimple() || ST->isIndexed()) in visitLIFETIME_END()
21690 const TypeSize StoreSize = ST->getMemoryVT().getStoreSize(); in visitLIFETIME_END()
21698 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase, in visitLIFETIME_END()
21703 CombineTo(ST, ST->getChain()); in visitLIFETIME_END()
21718 /// (shl (zext I to i64), 32)), addr) -->
21722 /// For pair of {i32, i32}, i64 store --> two i32 stores.
21723 /// For pair of {i32, i16}, i64 store --> two i32 stores.
21724 /// For pair of {i16, i16}, i32 store --> two i16 stores.
21725 /// For pair of {i16, i8}, i32 store --> two i16 stores.
21726 /// For pair of {i8, i8}, i16 store --> two i8 stores.
21746 if (!ST->isSimple()) in splitMergedValStore()
21749 SDValue Val = ST->getValue(); in splitMergedValStore()
21773 if (!ShAmt || ShAmt->getAPIntValue() != HalfValBitSize) in splitMergedValStore()
21776 // Lo and Hi are zero-extended from int with size less equal than 32 in splitMergedValStore()
21798 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); in splitMergedValStore()
21799 AAMDNodes AAInfo = ST->getAAInfo(); in splitMergedValStore()
21806 SDValue Chain = ST->getChain(); in splitMergedValStore()
21807 SDValue Ptr = ST->getBasePtr(); in splitMergedValStore()
21809 SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(), in splitMergedValStore()
21810 ST->getOriginalAlign(), MMOFlags, AAInfo); in splitMergedValStore()
21815 St0, DL, Hi, Ptr, ST->getPointerInfo().getWithOffset(HalfValBitSize / 8), in splitMergedValStore()
21816 ST->getOriginalAlign(), MMOFlags, AAInfo); in splitMergedValStore()
21823 // --> (vector_shuffle X, Y, NewMask)
21832 // Vec's operand 0 is using indices from 0 to N-1 and in mergeEltWithShuffle()
21833 // operand 1 from N to 2N - 1, where N is the number of in mergeEltWithShuffle()
21836 int ElementOffset = -1; in mergeEltWithShuffle()
21860 for (SDValue Op : reverse(ArgVal->ops())) { in mergeEltWithShuffle()
21861 CurrentArgOffset -= Step; in mergeEltWithShuffle()
21873 if (ElementOffset == -1) { in mergeEltWithShuffle()
21890 // --> (vector_shuffle X, Y) and variations where shuffle operands may be
21893 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in mergeInsertEltWithShuffle()
21895 SDValue InsertVal = N->getOperand(1); in mergeInsertEltWithShuffle()
21896 SDValue Vec = N->getOperand(0); in mergeInsertEltWithShuffle()
21902 ArrayRef<int> Mask = SVN->getMask(); in mergeInsertEltWithShuffle()
21918 // insert_vector_elt V, (bitcast X from vector type), IdxC -->
21923 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in combineInsertEltToShuffle()
21925 SDValue InsertVal = N->getOperand(1); in combineInsertEltToShuffle()
21932 SDValue DestVec = N->getOperand(0); in combineInsertEltToShuffle()
21947 // insert v4i32 V, (v2i16 X), 2 --> shuffle v8i16 V', X', {0,1,2,3,8,9,6,7} in combineInsertEltToShuffle()
21982 EVT VT = N->getValueType(0); in combineInsertEltToLoad()
21986 (InsIndex != 0 && InsIndex != VT.getVectorNumElements() - 1)) in combineInsertEltToLoad()
21991 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0)); in combineInsertEltToLoad()
21992 SDValue Scalar = N->getOperand(1); in combineInsertEltToLoad()
21993 if (!Shuffle || !all_of(enumerate(Shuffle->getMask()), [&](auto P) { in combineInsertEltToLoad()
21995 (InsIndex == 0 && P.value() == (int)P.index() - 1) || in combineInsertEltToLoad()
21996 (InsIndex == VT.getVectorNumElements() - 1 && in combineInsertEltToLoad()
22015 SDValue Vec = Shuffle->getOperand(0); in combineInsertEltToLoad()
22025 int EltSize = ScalarLoad->getValueType(0).getScalarSizeInBits(); in combineInsertEltToLoad()
22026 if (EltSize == 0 || EltSize % 8 != 0 || !ScalarLoad->isSimple() || in combineInsertEltToLoad()
22027 !VecLoad->isSimple() || VecLoad->getExtensionType() != ISD::NON_EXTLOAD || in combineInsertEltToLoad()
22028 ScalarLoad->getExtensionType() != ISD::NON_EXTLOAD || in combineInsertEltToLoad()
22029 ScalarLoad->getAddressSpace() != VecLoad->getAddressSpace()) in combineInsertEltToLoad()
22036 -1)) in combineInsertEltToLoad()
22040 VecLoad, ScalarLoad, VT.getVectorNumElements() * EltSize / 8, -1)) in combineInsertEltToLoad()
22046 Align NewAlign = commonAlignment(VecLoad->getAlign(), EltSize / 8); in combineInsertEltToLoad()
22048 Vec.getValueType(), VecLoad->getAddressSpace(), in combineInsertEltToLoad()
22049 NewAlign, VecLoad->getMemOperand()->getFlags(), in combineInsertEltToLoad()
22056 SDValue Ptr = ScalarLoad->getBasePtr(); in combineInsertEltToLoad()
22058 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), VecLoad->getBasePtr(), in combineInsertEltToLoad()
22061 InsIndex == 0 ? ScalarLoad->getPointerInfo() in combineInsertEltToLoad()
22062 : VecLoad->getPointerInfo().getWithOffset(EltSize / 8); in combineInsertEltToLoad()
22064 SDValue Load = DAG.getLoad(VecLoad->getValueType(0), DL, in combineInsertEltToLoad()
22065 ScalarLoad->getChain(), Ptr, PtrInfo, NewAlign); in combineInsertEltToLoad()
22072 SDValue InVec = N->getOperand(0); in visitINSERT_VECTOR_ELT()
22073 SDValue InVal = N->getOperand(1); in visitINSERT_VECTOR_ELT()
22074 SDValue EltNo = N->getOperand(2); in visitINSERT_VECTOR_ELT()
22080 // Insert into out-of-bounds element is undefined. in visitINSERT_VECTOR_ELT()
22082 IndexC->getZExtValue() >= VT.getVectorNumElements()) in visitINSERT_VECTOR_ELT()
22086 // (insert_vector_elt x (extract_vector_elt x idx) idx) -> x in visitINSERT_VECTOR_ELT()
22093 // inselt undef, InVal, EltNo --> build_vector < InVal, InVal, ... > in visitINSERT_VECTOR_ELT()
22105 unsigned Elt = IndexC->getZExtValue(); in visitINSERT_VECTOR_ELT()
22109 // insert_vector_elt(x, extract_vector_elt(y, 0), 0) -> y in visitINSERT_VECTOR_ELT()
22119 // -> (insert_vector_elt (insert_vector_elt A, Idx1), Idx0) in visitINSERT_VECTOR_ELT()
22147 // vXi1 vector - we don't need to recurse. in visitINSERT_VECTOR_ELT()
22182 // UNDEF - build new BUILD_VECTOR from already inserted operands. in visitINSERT_VECTOR_ELT()
22186 // BUILD_VECTOR - insert unused operands and build new BUILD_VECTOR. in visitINSERT_VECTOR_ELT()
22193 // SCALAR_TO_VECTOR - insert unused scalar and build new BUILD_VECTOR. in visitINSERT_VECTOR_ELT()
22199 // INSERT_VECTOR_ELT - insert operand and continue up the chain. in visitINSERT_VECTOR_ELT()
22202 if (CurIdx->getAPIntValue().ult(NumElts)) { in visitINSERT_VECTOR_ELT()
22203 unsigned Idx = CurIdx->getZExtValue(); in visitINSERT_VECTOR_ELT()
22210 CurVec = CurVec->getOperand(0); in visitINSERT_VECTOR_ELT()
22214 // VECTOR_SHUFFLE - if all the operands match the shuffle's sources, in visitINSERT_VECTOR_ELT()
22219 SDValue LHS = SVN->getOperand(0); in visitINSERT_VECTOR_ELT()
22220 SDValue RHS = SVN->getOperand(1); in visitINSERT_VECTOR_ELT()
22221 SmallVector<int, 16> Mask(SVN->getMask()); in visitINSERT_VECTOR_ELT()
22241 // TODO: Do this for -1 with OR mask? in visitINSERT_VECTOR_ELT()
22254 // Failed to find a match in the chain - bail. in visitINSERT_VECTOR_ELT()
22282 assert(OriginalLoad->isSimple()); in scalarizeExtractedVectorLoad()
22284 EVT ResultVT = EVE->getValueType(0); in scalarizeExtractedVectorLoad()
22299 Align Alignment = OriginalLoad->getAlign(); in scalarizeExtractedVectorLoad()
22303 int Elt = ConstEltNo->getZExtValue(); in scalarizeExtractedVectorLoad()
22305 MPI = OriginalLoad->getPointerInfo().getWithOffset(PtrOff); in scalarizeExtractedVectorLoad()
22310 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace()); in scalarizeExtractedVectorLoad()
22316 OriginalLoad->getAddressSpace(), Alignment, in scalarizeExtractedVectorLoad()
22317 OriginalLoad->getMemOperand()->getFlags(), in scalarizeExtractedVectorLoad()
22322 SDValue NewPtr = TLI.getVectorElementPointer(DAG, OriginalLoad->getBasePtr(), in scalarizeExtractedVectorLoad()
22334 Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(), in scalarizeExtractedVectorLoad()
22336 OriginalLoad->getMemOperand()->getFlags(), in scalarizeExtractedVectorLoad()
22337 OriginalLoad->getAAInfo()); in scalarizeExtractedVectorLoad()
22341 Load = DAG.getLoad(VecEltVT, DL, OriginalLoad->getChain(), NewPtr, MPI, in scalarizeExtractedVectorLoad()
22342 Alignment, OriginalLoad->getMemOperand()->getFlags(), in scalarizeExtractedVectorLoad()
22343 OriginalLoad->getAAInfo()); in scalarizeExtractedVectorLoad()
22359 SDValue Vec = ExtElt->getOperand(0); in scalarizeExtractedBinop()
22360 SDValue Index = ExtElt->getOperand(1); in scalarizeExtractedBinop()
22363 Vec->getNumValues() != 1) in scalarizeExtractedBinop()
22370 // Extracting an element of a vector constant is constant-folded, so this in scalarizeExtractedBinop()
22380 // extractelt (binop X, C), IndexC --> binop (extractelt X, IndexC), C' in scalarizeExtractedBinop()
22381 // extractelt (binop C, X), IndexC --> binop C', (extractelt X, IndexC) in scalarizeExtractedBinop()
22382 EVT VT = ExtElt->getValueType(0); in scalarizeExtractedBinop()
22402 // We perform this optimization post type-legalization because in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22403 // the type-legalizer often scalarizes integer-promoted vectors. in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22408 // TODO: Add support for big-endian. in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22412 SDValue VecOp = N->getOperand(0); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22417 auto *IndexC = dyn_cast<ConstantSDNode>(N->getOperand(1)); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22421 assert(IndexC->getZExtValue() < VecVT.getVectorNumElements() && in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22426 EVT ScalarVT = N->getValueType(0); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22430 // TODO: deal with the cases other than everything being integer-typed. in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22455 Worklist.emplace_back(N, /*BitPos=*/VecEltBitWidth * IndexC->getZExtValue(), in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22467 for (SDNode *User : E.Producer->uses()) { in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22468 switch (User->getOpcode()) { in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22476 /*NumBits=*/User->getValueSizeInBits(0)); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22482 if (auto *ShAmtC = dyn_cast<ConstantSDNode>(User->getOperand(1)); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22483 User->getOperand(0).getNode() == E.Producer && ShAmtC) { in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22484 // Logical right-shift means that we start extraction later, in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22486 unsigned ShAmt = ShAmtC->getZExtValue(); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22487 Worklist.emplace_back(User, E.BitPos + ShAmt, E.NumBits - ShAmt); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22497 if (User->getOpcode() != ISD::BUILD_VECTOR) in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22521 E.Producer->getValueSizeInBits(0) == NewVecEltBitWidth && in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22544 "Creating out-of-bounds ISD::EXTRACT_VECTOR_ELT?"); in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22554 SDValue VecOp = N->getOperand(0); in visitEXTRACT_VECTOR_ELT()
22555 SDValue Index = N->getOperand(1); in visitEXTRACT_VECTOR_ELT()
22556 EVT ScalarVT = N->getValueType(0); in visitEXTRACT_VECTOR_ELT()
22561 // extract_vector_elt (insert_vector_elt vec, val, idx), idx) -> val in visitEXTRACT_VECTOR_ELT()
22563 // This only really matters if the index is non-constant since other combines in visitEXTRACT_VECTOR_ELT()
22572 // (vextract (scalar_to_vector val, 0) -> val in visitEXTRACT_VECTOR_ELT()
22591 // extract_vector_elt of out-of-bounds element -> UNDEF in visitEXTRACT_VECTOR_ELT()
22594 IndexC->getAPIntValue().uge(VecVT.getVectorNumElements())) in visitEXTRACT_VECTOR_ELT()
22597 // extract_vector_elt (build_vector x, y), 1 -> y in visitEXTRACT_VECTOR_ELT()
22605 VecOp.getOpcode() == ISD::BUILD_VECTOR ? IndexC->getZExtValue() : 0; in visitEXTRACT_VECTOR_ELT()
22635 APInt EltMask = APInt::getOneBitSet(NumElts, IndexC->getZExtValue()); in visitEXTRACT_VECTOR_ELT()
22650 // The vector index of the LSBs of the source depend on the endian-ness. in visitEXTRACT_VECTOR_ELT()
22652 unsigned ExtractIndex = IndexC->getZExtValue(); in visitEXTRACT_VECTOR_ELT()
22653 // extract_elt (v2i32 (bitcast i64:x)), BCTruncElt -> i32 (trunc i64:x) in visitEXTRACT_VECTOR_ELT()
22654 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1; in visitEXTRACT_VECTOR_ELT()
22661 // ext_elt (bitcast (scalar_to_vec i64 X to v2i64) to v4i32), TruncElt --> in visitEXTRACT_VECTOR_ELT()
22668 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1; in visitEXTRACT_VECTOR_ELT()
22681 // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT. in visitEXTRACT_VECTOR_ELT()
22690 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue()); in visitEXTRACT_VECTOR_ELT()
22693 if (OrigElt == -1) in visitEXTRACT_VECTOR_ELT()
22702 OrigElt -= NumElts; in visitEXTRACT_VECTOR_ELT()
22729 if (llvm::all_of(VecOp->uses(), [&](SDNode *Use) { in visitEXTRACT_VECTOR_ELT()
22730 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
22731 Use->getOperand(0) == VecOp && in visitEXTRACT_VECTOR_ELT()
22732 isa<ConstantSDNode>(Use->getOperand(1)); in visitEXTRACT_VECTOR_ELT()
22735 for (SDNode *Use : VecOp->uses()) { in visitEXTRACT_VECTOR_ELT()
22736 auto *CstElt = cast<ConstantSDNode>(Use->getOperand(1)); in visitEXTRACT_VECTOR_ELT()
22737 if (CstElt->getAPIntValue().ult(NumElts)) in visitEXTRACT_VECTOR_ELT()
22738 DemandedElts.setBit(CstElt->getZExtValue()); in visitEXTRACT_VECTOR_ELT()
22743 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
22751 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
22783 // extract (vector load $addr), i --> load $addr + i * size in visitEXTRACT_VECTOR_ELT()
22786 !Index->hasPredecessor(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
22788 if (VecLoad && VecLoad->isSimple()) in visitEXTRACT_VECTOR_ELT()
22797 // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size) in visitEXTRACT_VECTOR_ELT()
22798 // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size) in visitEXTRACT_VECTOR_ELT()
22799 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr) in visitEXTRACT_VECTOR_ELT()
22800 int Elt = IndexC->getZExtValue(); in visitEXTRACT_VECTOR_ELT()
22828 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt); in visitEXTRACT_VECTOR_ELT()
22840 Elt = (Idx < (int)NumElts) ? Idx : Idx - (int)NumElts; in visitEXTRACT_VECTOR_ELT()
22849 // -> extract_vector_elt a, 0 in visitEXTRACT_VECTOR_ELT()
22851 // -> extract_vector_elt a, 1 in visitEXTRACT_VECTOR_ELT()
22853 // -> extract_vector_elt b, 0 in visitEXTRACT_VECTOR_ELT()
22855 // -> extract_vector_elt b, 1 in visitEXTRACT_VECTOR_ELT()
22868 // Make sure we found a non-volatile load and the extractelement is in visitEXTRACT_VECTOR_ELT()
22870 if (!LN0 || !LN0->hasNUsesOfValue(1,0) || !LN0->isSimple()) in visitEXTRACT_VECTOR_ELT()
22873 // If Idx was -1 above, Elt is going to be -1, so just return undef. in visitEXTRACT_VECTOR_ELT()
22874 if (Elt == -1) in visitEXTRACT_VECTOR_ELT()
22882 // We perform this optimization post type-legalization because in reduceBuildVecExtToExtBuildVec()
22883 // the type-legalizer often scalarizes integer-promoted vectors. in reduceBuildVecExtToExtBuildVec()
22884 // Performing this optimization before may create bit-casts which in reduceBuildVecExtToExtBuildVec()
22885 // will be type-legalized to complex code sequences. in reduceBuildVecExtToExtBuildVec()
22891 unsigned NumInScalars = N->getNumOperands(); in reduceBuildVecExtToExtBuildVec()
22893 EVT VT = N->getValueType(0); in reduceBuildVecExtToExtBuildVec()
22897 // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR in reduceBuildVecExtToExtBuildVec()
22898 // optimizations. We do not handle sign-extend because we can't fill the sign in reduceBuildVecExtToExtBuildVec()
22904 SDValue In = N->getOperand(i); in reduceBuildVecExtToExtBuildVec()
22963 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { in reduceBuildVecExtToExtBuildVec()
22964 SDValue Cast = N->getOperand(i); in reduceBuildVecExtToExtBuildVec()
22972 In = Cast->getOperand(0); in reduceBuildVecExtToExtBuildVec()
22974 (i * ElemRatio + (ElemRatio - 1)); in reduceBuildVecExtToExtBuildVec()
23000 // (trunc (srl $1 half-width))
23001 // (trunc (srl $1 (2 * half-width))))
23004 assert(N->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecTruncToBitCast()
23006 EVT VT = N->getValueType(0); in reduceBuildVecTruncToBitCast()
23025 unsigned NumInScalars = N->getNumOperands(); in reduceBuildVecTruncToBitCast()
23037 SDValue In = PeekThroughBitcast(N->getOperand(i)); in reduceBuildVecTruncToBitCast()
23089 EVT VT = N->getValueType(0); in createBuildVecShuffle()
23105 "Inputs must be sorted to be in non-increasing vector size order."); in createBuildVecShuffle()
23145 // Legalizing INSERT_SUBVECTOR is tricky - you basically have to in createBuildVecShuffle()
23193 SmallVector<int, 8> Mask(ShuffleNumElems, -1); in createBuildVecShuffle()
23196 // total number of elements in the shuffle - if we are shuffling a wider in createBuildVecShuffle()
23202 unsigned ExtIndex = N->getOperand(i).getConstantOperandVal(1); in createBuildVecShuffle()
23226 assert(BV->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecToShuffleWithZero()
23230 int NumBVOps = BV->getNumOperands(); in reduceBuildVecToShuffleWithZero()
23231 int ZextElt = -1; in reduceBuildVecToShuffleWithZero()
23233 SDValue Op = BV->getOperand(i); in reduceBuildVecToShuffleWithZero()
23236 if (ZextElt == -1) in reduceBuildVecToShuffleWithZero()
23241 // Bail out if there's no non-undef element. in reduceBuildVecToShuffleWithZero()
23242 if (ZextElt == -1) in reduceBuildVecToShuffleWithZero()
23246 // one other element. That other element must be a zero-extended scalar in reduceBuildVecToShuffleWithZero()
23251 EVT VT = BV->getValueType(0); in reduceBuildVecToShuffleWithZero()
23252 SDValue Zext = BV->getOperand(ZextElt); in reduceBuildVecToShuffleWithZero()
23259 // The zero-extend must be a multiple of the source size, and we must be in reduceBuildVecToShuffleWithZero()
23272 SmallVector<int, 32> ShufMask(NumMaskElts, -1); in reduceBuildVecToShuffleWithZero()
23278 // that vector (mask value is number-of-elements) for the high bits. in reduceBuildVecToShuffleWithZero()
23279 int Low = DAG.getDataLayout().isBigEndian() ? (ZextRatio - 1) : 0; in reduceBuildVecToShuffleWithZero()
23285 // the shuffle mask with -1. in reduceBuildVecToShuffleWithZero()
23288 // buildvec undef, ..., (zext (extractelt V, IndexC)), undef... --> in reduceBuildVecToShuffleWithZero()
23306 return static_cast<decltype(std::distance(Range.begin(), I))>(-1); in getFirstIndexOf()
23315 EVT VT = N->getValueType(0); in reduceBuildVecToShuffle()
23317 // Only type-legal BUILD_VECTOR nodes are converted to shuffle nodes. in reduceBuildVecToShuffle()
23329 unsigned NumElems = N->getNumOperands(); in reduceBuildVecToShuffle()
23332 // that element comes from. -1 stands for undef, 0 for the zero vector, in reduceBuildVecToShuffle()
23337 SmallVector<int, 8> VectorMask(NumElems, -1); in reduceBuildVecToShuffle()
23342 SDValue Op = N->getOperand(i); in reduceBuildVecToShuffle()
23357 // EXTRACT_VECTOR_ELT with an in-range constant index, bail out. in reduceBuildVecToShuffle()
23379 if (Idx == -1) { // A new source vector? in reduceBuildVecToShuffle()
23406 unsigned Index = N->getOperand(i).getConstantOperandVal(1); in reduceBuildVecToShuffle()
23439 // while preserving the relative order of equally-sized vectors. in reduceBuildVecToShuffle()
23440 // Note that we keep the first "implicit zero vector as-is. in reduceBuildVecToShuffle()
23500 // Update the vector mask to point to the post-shuffle vectors. in reduceBuildVecToShuffle()
23503 Vec = Shuffles.size() - 1; in reduceBuildVecToShuffle()
23505 Vec = (Vec - 1) / 2; in reduceBuildVecToShuffle()
23530 SmallVector<int, 8> Mask(NumElems, -1); in reduceBuildVecToShuffle()
23537 LMask = cast<ShuffleVectorSDNode>(L.getNode())->getMask(); in reduceBuildVecToShuffle()
23546 RMask = cast<ShuffleVectorSDNode>(R.getNode())->getMask(); in reduceBuildVecToShuffle()
23577 EVT VT = N->getValueType(0); in convertBuildVecZextToZext()
23580 SDValue Op0 = N->getOperand(0); in convertBuildVecZextToZext()
23581 auto checkElem = [&](SDValue Op) -> int64_t { in convertBuildVecZextToZext()
23588 return C->getZExtValue(); in convertBuildVecZextToZext()
23589 return -1; in convertBuildVecZextToZext()
23595 // known-minimum vector length of the result type. in convertBuildVecZextToZext()
23600 unsigned NumElems = N->getNumOperands(); in convertBuildVecZextToZext()
23611 if ((Offset + i) != checkElem(N->getOperand(i))) in convertBuildVecZextToZext()
23628 // non-constant-zero op, UNDEF's, and to be KnownBits-based,
23634 // FIXME: support big-endian. in convertBuildVecZextToBuildVecWithZeros()
23638 EVT VT = N->getValueType(0); in convertBuildVecZextToBuildVecWithZeros()
23639 EVT OpVT = N->getOperand(0).getValueType(); in convertBuildVecZextToBuildVecWithZeros()
23655 for (auto I : enumerate(N->ops())) { in convertBuildVecZextToBuildVecWithZeros()
23660 Cst->getAPIntValue().trunc(EltBitwidth).getActiveBits(); in convertBuildVecZextToBuildVecWithZeros()
23665 // Profitability check: don't allow non-zero constant operands. in convertBuildVecZextToBuildVecWithZeros()
23668 // Profitability check: there must only be a single non-zero operand, in convertBuildVecZextToBuildVecWithZeros()
23672 // The operand must be a zero-extension itself. in convertBuildVecZextToBuildVecWithZeros()
23678 assert(!ActiveBits && "Already encountered non-constant-zero operand?"); in convertBuildVecZextToBuildVecWithZeros()
23685 // This BUILD_VECTOR must have at least one non-constant-zero operand. in convertBuildVecZextToBuildVecWithZeros()
23699 for (unsigned Scale = EltBitwidth / ActiveBits; Scale >= 2; --Scale) { in convertBuildVecZextToBuildVecWithZeros() local
23700 if (EltBitwidth % Scale != 0) in convertBuildVecZextToBuildVecWithZeros()
23702 unsigned ChunkBitwidth = EltBitwidth / Scale; in convertBuildVecZextToBuildVecWithZeros()
23706 Scale * N->getNumOperands()); in convertBuildVecZextToBuildVecWithZeros()
23712 Factor = Scale; in convertBuildVecZextToBuildVecWithZeros()
23724 for (auto I : enumerate(N->ops())) { in convertBuildVecZextToBuildVecWithZeros()
23735 NewOps.append(*Factor - 1, ZeroOp); in convertBuildVecZextToBuildVecWithZeros()
23744 EVT VT = N->getValueType(0); in visitBUILD_VECTOR()
23753 // (build_vector (i64 (bitcast (v2i32 X))), (i64 (bitcast (v2i32 X)))) -> in visitBUILD_VECTOR()
23757 // TODO: Maybe this is useful for non-splat too? in visitBUILD_VECTOR()
23759 SDValue Splat = cast<BuildVectorSDNode>(N)->getSplatValue(); in visitBUILD_VECTOR()
23766 unsigned NumElts = N->getNumOperands() * SrcVT.getVectorNumElements(); in visitBUILD_VECTOR()
23770 SmallVector<SDValue, 8> Ops(N->getNumOperands(), Splat); in visitBUILD_VECTOR()
23780 if (!LegalTypes && (N->getNumOperands() > 1)) { in visitBUILD_VECTOR()
23781 SDValue Op0 = N->getOperand(0); in visitBUILD_VECTOR()
23782 auto checkElem = [&](SDValue Op) -> uint64_t { in visitBUILD_VECTOR()
23786 return CNode->getZExtValue(); in visitBUILD_VECTOR()
23787 return -1; in visitBUILD_VECTOR()
23791 for (unsigned i = 0; i < N->getNumOperands(); ++i) { in visitBUILD_VECTOR()
23792 if (Offset + i != checkElem(N->getOperand(i))) { in visitBUILD_VECTOR()
23793 Offset = -1; in visitBUILD_VECTOR()
23799 (Op0.getOperand(0).getValueType() == N->getValueType(0))) in visitBUILD_VECTOR()
23801 if ((Offset != -1) && in visitBUILD_VECTOR()
23802 ((Offset % N->getValueType(0).getVectorNumElements()) == in visitBUILD_VECTOR()
23804 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), N->getValueType(0), in visitBUILD_VECTOR()
23826 if (SDValue V = cast<BuildVectorSDNode>(N)->getSplatValue()) { in visitBUILD_VECTOR()
23836 EVT OpVT = N->getOperand(0).getValueType(); in combineConcatVectorOfScalars()
23843 EVT VT = N->getValueType(0); in combineConcatVectorOfScalars()
23850 for (const SDValue &Op : N->ops()) { in combineConcatVectorOfScalars()
23894 // --> concat_vectors(x,y,z,w,u,u,u,u,u,u,u,u,a,b,c,d)
23897 EVT VT = N->getValueType(0); in combineConcatVectorOfConcatVectors()
23902 for (const SDValue &Op : N->ops()) { in combineConcatVectorOfConcatVectors()
23917 assert(FirstConcat && "Concat of all-undefs found"); in combineConcatVectorOfConcatVectors()
23920 for (const SDValue &Op : N->ops()) { in combineConcatVectorOfConcatVectors()
23922 ConcatOps.append(FirstConcat->getNumOperands(), DAG.getUNDEF(SubVT)); in combineConcatVectorOfConcatVectors()
23925 ConcatOps.append(Op->op_begin(), Op->op_end()); in combineConcatVectorOfConcatVectors()
23935 EVT VT = N->getValueType(0); in combineConcatVectorOfExtracts()
23936 EVT OpVT = N->getOperand(0).getValueType(); in combineConcatVectorOfExtracts()
23948 for (SDValue Op : N->ops()) { in combineConcatVectorOfExtracts()
23953 Mask.append((unsigned)NumOpElts, -1); in combineConcatVectorOfExtracts()
23964 // We want the EVT of the original extraction to correctly scale the in combineConcatVectorOfExtracts()
23971 Mask.append((unsigned)NumOpElts, -1); in combineConcatVectorOfExtracts()
23980 // Scale the subvector index to account for any bitcast. in combineConcatVectorOfExtracts()
24009 unsigned CastOpcode = N->getOperand(0).getOpcode(); in combineConcatVectorOfCasts()
24026 EVT SrcVT = N->getOperand(0).getOperand(0).getValueType(); in combineConcatVectorOfCasts()
24033 for (SDValue Op : N->ops()) { in combineConcatVectorOfCasts()
24043 EVT VT = N->getValueType(0); in combineConcatVectorOfCasts()
24045 ElementCount NumElts = SrcVT.getVectorElementCount() * N->getNumOperands(); in combineConcatVectorOfCasts()
24065 // concat (cast X), (cast Y)... -> cast (concat X, Y...) in combineConcatVectorOfCasts()
24077 EVT VT = N->getValueType(0); in combineConcatVectorOfShuffleAndItsOperands()
24078 EVT OpVT = N->getOperand(0).getValueType(); in combineConcatVectorOfShuffleAndItsOperands()
24082 // For now, only allow simple 2-operand concatenations. in combineConcatVectorOfShuffleAndItsOperands()
24083 if (N->getNumOperands() != 2) in combineConcatVectorOfShuffleAndItsOperands()
24098 for (SDValue Op : N->ops()) { in combineConcatVectorOfShuffleAndItsOperands()
24100 CurSVN && CurSVN->getOperand(1).isUndef() && N->isOnlyUserOf(CurSVN) && in combineConcatVectorOfShuffleAndItsOperands()
24101 all_of(N->ops(), [CurSVN](SDValue Op) { in combineConcatVectorOfShuffleAndItsOperands()
24104 (Op.getNode() == CurSVN || is_contained(CurSVN->ops(), Op)); in combineConcatVectorOfShuffleAndItsOperands()
24116 AdjustedMask.reserve(SVN->getMask().size()); in combineConcatVectorOfShuffleAndItsOperands()
24117 assert(SVN->getOperand(1).isUndef() && "Expected unary shuffle!"); in combineConcatVectorOfShuffleAndItsOperands()
24118 append_range(AdjustedMask, SVN->getMask()); in combineConcatVectorOfShuffleAndItsOperands()
24134 for (SDValue Op : N->ops()) { in combineConcatVectorOfShuffleAndItsOperands()
24140 if (Op == SVN->getOperand(0)) { in combineConcatVectorOfShuffleAndItsOperands()
24144 if (Op == SVN->getOperand(1)) { in combineConcatVectorOfShuffleAndItsOperands()
24158 for (auto I : zip(SVN->ops(), ShufOps)) { in combineConcatVectorOfShuffleAndItsOperands()
24164 SmallVector<SDValue, 2> ShufOpParts(N->getNumOperands(), in combineConcatVectorOfShuffleAndItsOperands()
24176 if (N->getNumOperands() == 1) in visitCONCAT_VECTORS()
24177 return N->getOperand(0); in visitCONCAT_VECTORS()
24180 EVT VT = N->getValueType(0); in visitCONCAT_VECTORS()
24185 if (all_of(drop_begin(N->ops()), in visitCONCAT_VECTORS()
24187 SDValue In = N->getOperand(0); in visitCONCAT_VECTORS()
24199 unsigned NumOps = N->getNumOperands() * In.getNumOperands(); in visitCONCAT_VECTORS()
24200 SmallVector<SDValue, 4> Ops(In->op_begin(), In->op_end()); in visitCONCAT_VECTORS()
24207 // concat_vectors(scalar_to_vector(scalar), undef) -> in visitCONCAT_VECTORS()
24216 // concat_vectors(scalar, undef) -> scalar_to_vector(scalar) in visitCONCAT_VECTORS()
24220 // concat_vectors(trunc(scalar), undef) -> scalar_to_vector(scalar) in visitCONCAT_VECTORS()
24221 if (Scalar->getOpcode() == ISD::TRUNCATE && in visitCONCAT_VECTORS()
24223 TLI.isTypeLegal(Scalar->getOperand(0).getValueType())) in visitCONCAT_VECTORS()
24224 Scalar = Scalar->getOperand(0); in visitCONCAT_VECTORS()
24251 // -> (BUILD_VECTOR A, B, ..., C, D, ...) in visitCONCAT_VECTORS()
24255 if (llvm::all_of(N->ops(), IsBuildVectorOrUndef)) { in visitCONCAT_VECTORS()
24264 for (const SDValue &Op : N->ops()) in visitCONCAT_VECTORS()
24273 for (const SDValue &Op : N->ops()) { in visitCONCAT_VECTORS()
24283 Opnds.append(Op->op_begin(), Op->op_begin() + NumElts); in visitCONCAT_VECTORS()
24327 N->getOperand(0).getValueType().getVectorMinNumElements(); in visitCONCAT_VECTORS()
24329 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { in visitCONCAT_VECTORS()
24330 SDValue Op = N->getOperand(i); in visitCONCAT_VECTORS()
24349 if (SingleSource.getValueType() != N->getValueType(0)) in visitCONCAT_VECTORS()
24375 (IndexC->getZExtValue() % SubVT.getVectorMinNumElements()) == 0) { in getSubVectorSrc()
24376 uint64_t SubIdx = IndexC->getZExtValue() / SubVT.getVectorMinNumElements(); in getSubVectorSrc()
24386 SDValue BinOp = Extract->getOperand(0); in narrowInsertExtractVectorBinOp()
24388 if (!TLI.isBinOp(BinOpcode) || BinOp->getNumValues() != 1) in narrowInsertExtractVectorBinOp()
24396 SDValue Index = Extract->getOperand(1); in narrowInsertExtractVectorBinOp()
24397 EVT SubVT = Extract->getValueType(0); in narrowInsertExtractVectorBinOp()
24412 // ext (binop (ins ?, X, Index), (ins ?, Y, Index)), Index --> binop X, Y in narrowInsertExtractVectorBinOp()
24414 BinOp->getFlags()); in narrowInsertExtractVectorBinOp()
24428 auto *ExtractIndexC = dyn_cast<ConstantSDNode>(Extract->getOperand(1)); in narrowExtractedVectorBinOp()
24435 SDValue BinOp = peekThroughBitcasts(Extract->getOperand(0)); in narrowExtractedVectorBinOp()
24437 if (!TLI.isBinOp(BOpcode) || BinOp->getNumValues() != 1) in narrowExtractedVectorBinOp()
24440 // Exclude the fake form of fneg (fsub -0.0, x) because that is likely to be in narrowExtractedVectorBinOp()
24442 // with fneg in a target-specific way. in narrowExtractedVectorBinOp()
24445 if (C && C->getValueAPF().isNegZero()) in narrowExtractedVectorBinOp()
24457 EVT VT = Extract->getValueType(0); in narrowExtractedVectorBinOp()
24458 unsigned ExtractIndex = ExtractIndexC->getZExtValue(); in narrowExtractedVectorBinOp()
24489 BinOp.hasOneUse() && Extract->getOperand(0)->hasOneUse()) { in narrowExtractedVectorBinOp()
24490 // extract (binop B0, B1), N --> binop (extract B0, N), (extract B1, N) in narrowExtractedVectorBinOp()
24498 DAG.getNode(BOpcode, DL, NarrowBVT, X, Y, BinOp->getFlags()); in narrowExtractedVectorBinOp()
24508 // target has temptingly almost legal versions of bitwise logic ops in 256-bit in narrowExtractedVectorBinOp()
24509 // flavors, but no other 256-bit integer support. This could be extended to in narrowExtractedVectorBinOp()
24517 auto GetSubVector = [ConcatOpNum](SDValue V) -> SDValue { in narrowExtractedVectorBinOp()
24527 // half-sized operand for our new narrow binop: in narrowExtractedVectorBinOp()
24528 // extract (binop (concat X1, X2), (concat Y1, Y2)), N --> binop XN, YN in narrowExtractedVectorBinOp()
24529 // extract (binop (concat X1, X2), Y), N --> binop XN, (extract Y, IndexC) in narrowExtractedVectorBinOp()
24530 // extract (binop X, (concat Y1, Y2)), N --> binop (extract X, IndexC), YN in narrowExtractedVectorBinOp()
24550 /// (extract_subvector (load wide vector)) --> (load narrow vector)
24552 // TODO: Add support for big-endian. The offset calculation must be adjusted. in narrowExtractedVectorLoad()
24556 auto *Ld = dyn_cast<LoadSDNode>(Extract->getOperand(0)); in narrowExtractedVectorLoad()
24557 if (!Ld || Ld->getExtensionType() || !Ld->isSimple()) in narrowExtractedVectorLoad()
24560 // Allow targets to opt-out. in narrowExtractedVectorLoad()
24561 EVT VT = Extract->getValueType(0); in narrowExtractedVectorLoad()
24567 unsigned Index = Extract->getConstantOperandVal(1); in narrowExtractedVectorLoad()
24571 if (Index == 0 && NumElts >= Ld->getValueType(0).getVectorMinNumElements()) in narrowExtractedVectorLoad()
24583 if (!TLI.shouldReduceLoadWidth(Ld, Ld->getExtensionType(), VT)) in narrowExtractedVectorLoad()
24587 // we are extracting from something besides index 0 (little-endian). in narrowExtractedVectorLoad()
24591 SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL); in narrowExtractedVectorLoad()
24598 MachinePointerInfo(Ld->getPointerInfo().getAddrSpace()); in narrowExtractedVectorLoad()
24599 MMO = MF.getMachineMemOperand(Ld->getMemOperand(), MPI, StoreSize); in narrowExtractedVectorLoad()
24601 MMO = MF.getMachineMemOperand(Ld->getMemOperand(), Offset.getFixedValue(), in narrowExtractedVectorLoad()
24604 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO); in narrowExtractedVectorLoad()
24620 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && in foldExtractSubvectorFromShuffleVector()
24623 SDValue N0 = N->getOperand(0); in foldExtractSubvectorFromShuffleVector()
24625 // Only deal with non-scalable vectors. in foldExtractSubvectorFromShuffleVector()
24626 EVT NarrowVT = N->getValueType(0); in foldExtractSubvectorFromShuffleVector()
24637 if (!WideShuffleVector->hasOneUse()) in foldExtractSubvectorFromShuffleVector()
24645 uint64_t FirstExtractedEltIdx = N->getConstantOperandVal(1); in foldExtractSubvectorFromShuffleVector()
24658 for (int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx, in foldExtractSubvectorFromShuffleVector()
24660 assert((M >= -1) && (M < (2 * WideNumElts)) && in foldExtractSubvectorFromShuffleVector()
24661 "Out-of-bounds shuffle mask?"); in foldExtractSubvectorFromShuffleVector()
24677 // And which NumEltsExtracted-sized subvector of that operand is that? in foldExtractSubvectorFromShuffleVector()
24689 SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx); in foldExtractSubvectorFromShuffleVector()
24693 NewMask.emplace_back(-1); in foldExtractSubvectorFromShuffleVector()
24738 if (TLI.isShuffleMaskLegal(WideShuffleVector->getMask(), WideVT) && in foldExtractSubvectorFromShuffleVector()
24764 EVT NVT = N->getValueType(0); in visitEXTRACT_SUBVECTOR()
24765 SDValue V = N->getOperand(0); in visitEXTRACT_SUBVECTOR()
24766 uint64_t ExtIdx = N->getConstantOperandVal(1); in visitEXTRACT_SUBVECTOR()
24778 // ext (ext X, C), 0 --> ext X, C in visitEXTRACT_SUBVECTOR()
24788 // ty1 extract_vector(ty2 splat(V))) -> ty1 splat(V) in visitEXTRACT_SUBVECTOR()
24795 // --> extract_subvector(y,c2-c1) in visitEXTRACT_SUBVECTOR()
24804 TLI.isExtractSubvectorCheap(NVT, InsSubVT, ExtIdx - InsIdx) && in visitEXTRACT_SUBVECTOR()
24808 DAG.getVectorIdxConstant(ExtIdx - InsIdx, DL)); in visitEXTRACT_SUBVECTOR()
24812 // extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index') in visitEXTRACT_SUBVECTOR()
24875 // extract_subvec (concat V1, V2, ...), i --> Vi in visitEXTRACT_SUBVECTOR()
24882 // v2i8 extract_subvec (v16i8 concat (v8i8 X), (v8i8 Y), 14 --> in visitEXTRACT_SUBVECTOR()
24886 unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts; in visitEXTRACT_SUBVECTOR()
24922 SDValue Src = V->getOperand(IdxVal); in visitEXTRACT_SUBVECTOR()
24930 DAG.getBuildVector(ExtractVT, DL, V->ops().slice(IdxVal, NumElems)); in visitEXTRACT_SUBVECTOR()
24958 DAG.getBitcast(N->getOperand(0).getValueType(), V.getOperand(0)), in visitEXTRACT_SUBVECTOR()
24959 N->getOperand(1)); in visitEXTRACT_SUBVECTOR()
24977 SDValue N0 = Shuf->getOperand(0), N1 = Shuf->getOperand(1); in foldShuffleOfConcatUndefs()
24985 ArrayRef<int> Mask = Shuf->getMask(); in foldShuffleOfConcatUndefs()
24986 EVT VT = Shuf->getValueType(0); in foldShuffleOfConcatUndefs()
24989 SmallVector<int, 16> Mask0(HalfNumElts, -1); in foldShuffleOfConcatUndefs()
24990 SmallVector<int, 16> Mask1(HalfNumElts, -1); in foldShuffleOfConcatUndefs()
24992 if (Mask[i] == -1) in foldShuffleOfConcatUndefs()
24997 int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (int)HalfNumElts; in foldShuffleOfConcatUndefs()
25001 Mask1[i - HalfNumElts] = M; in foldShuffleOfConcatUndefs()
25012 // shuffle (concat X, undef), (concat Y, undef), Mask --> in foldShuffleOfConcatUndefs()
25024 EVT VT = N->getValueType(0); in partitionShuffleOfConcats()
25027 SDValue N0 = N->getOperand(0); in partitionShuffleOfConcats()
25028 SDValue N1 = N->getOperand(1); in partitionShuffleOfConcats()
25030 ArrayRef<int> Mask = SVN->getMask(); in partitionShuffleOfConcats()
25037 auto IsUndefMaskElt = [](int i) { return i == -1; }; in partitionShuffleOfConcats()
25053 // subvector-sized copies from a concatenated vector in partitionShuffleOfConcats()
25064 int OpIdx = -1; in partitionShuffleOfConcats()
25080 Ops.push_back(N1.getOperand(OpIdx - N0.getNumOperands())); in partitionShuffleOfConcats()
25086 // Attempt to combine a shuffle of 2 inputs of 'scalar sources' -
25089 // SHUFFLE(BUILD_VECTOR(), BUILD_VECTOR()) -> BUILD_VECTOR() is always
25099 // We don't fold shuffles where one side is a non-zero constant, and we don't
25100 // fold shuffles if the resulting (non-splat) BUILD_VECTOR would have duplicate
25101 // non-constant operands. This seems to work out reasonably well in practice.
25105 EVT VT = SVN->getValueType(0); in combineShuffleOfScalars()
25107 SDValue N0 = SVN->getOperand(0); in combineShuffleOfScalars()
25108 SDValue N1 = SVN->getOperand(1); in combineShuffleOfScalars()
25110 if (!N0->hasOneUse()) in combineShuffleOfScalars()
25116 if (!N1->hasOneUse()) in combineShuffleOfScalars()
25133 if (SDValue Splat0 = BV0->getSplatValue()) in combineShuffleOfScalars()
25134 IsSplat = (Splat0 == BV1->getSplatValue()); in combineShuffleOfScalars()
25138 for (int M : SVN->getMask()) { in combineShuffleOfScalars()
25141 int Idx = M < (int)NumElts ? M : M - NumElts; in combineShuffleOfScalars()
25149 // Operand can't be combined - bail out. in combineShuffleOfScalars()
25154 // Don't duplicate a non-constant BUILD_VECTOR operand unless we're in combineShuffleOfScalars()
25156 // generate low-quality code if the target can't reconstruct an appropriate in combineShuffleOfScalars()
25182 // e.g. v4i32 <0,u,1,u> -> (v2i64 any_vector_extend_in_reg(v4i32 src)),
25191 // TODO Add support for big-endian when we have a test case. in canCombineShuffleToExtendVectorInreg()
25199 // power-of-2 extensions as they are the most likely. in canCombineShuffleToExtendVectorInreg()
25200 // FIXME: should try Scale == NumElts case too, in canCombineShuffleToExtendVectorInreg()
25201 for (unsigned Scale = 2; Scale < NumElts; Scale *= 2) { in canCombineShuffleToExtendVectorInreg() local
25202 // The vector width must be a multiple of Scale. in canCombineShuffleToExtendVectorInreg()
25203 if (NumElts % Scale != 0) in canCombineShuffleToExtendVectorInreg()
25206 EVT OutSVT = EVT::getIntegerVT(*DAG.getContext(), EltSizeInBits * Scale); in canCombineShuffleToExtendVectorInreg()
25207 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), OutSVT, NumElts / Scale); in canCombineShuffleToExtendVectorInreg()
25213 if (Match(Scale)) in canCombineShuffleToExtendVectorInreg()
25222 // e.g. v4i32 <0,u,1,u> -> (v2i64 any_vector_extend_in_reg(v4i32 src))
25227 EVT VT = SVN->getValueType(0); in combineShuffleToAnyExtendVectorInreg()
25230 // TODO Add support for big-endian when we have a test case. in combineShuffleToAnyExtendVectorInreg()
25234 // shuffle<0,-1,1,-1> == (v2i64 anyextend_vector_inreg(v4i32)) in combineShuffleToAnyExtendVectorInreg()
25236 Mask = SVN->getMask()](unsigned Scale) { in combineShuffleToAnyExtendVectorInreg() argument
25240 if ((i % Scale) == 0 && Mask[i] == (int)(i / Scale)) in combineShuffleToAnyExtendVectorInreg()
25248 SDValue N0 = SVN->getOperand(0); in combineShuffleToAnyExtendVectorInreg()
25250 // are pre-legalization. in combineShuffleToAnyExtendVectorInreg()
25260 // e.g. v4i32 <0,z,1,u> -> (v2i64 zero_extend_vector_inreg(v4i32 src))
25266 EVT VT = SVN->getValueType(0); in combineShuffleToZeroExtendVectorInReg()
25271 // TODO: add support for big-endian when we have a test case. in combineShuffleToZeroExtendVectorInReg()
25276 SmallVector<int, 16> Mask(SVN->getMask().begin(), SVN->getMask().end()); in combineShuffleToZeroExtendVectorInReg()
25282 int OpEltIdx = (unsigned)Indice < NumElts ? Indice : Indice - NumElts; in combineShuffleToZeroExtendVectorInReg()
25296 // Element-wise(!), which of these demanded elements are know to be zero? in combineShuffleToZeroExtendVectorInReg()
25298 for (auto I : zip(SVN->ops(), OpsDemandedElts, OpsKnownZeroElts)) in combineShuffleToZeroExtendVectorInReg()
25310 Indice = -2; // Zeroable element. in combineShuffleToZeroExtendVectorInReg()
25322 // The shuffle may be more fine-grained than we want. Widen elements first. in combineShuffleToZeroExtendVectorInReg()
25341 // shuffle<0,z,1,-1> == (v2i64 zero_extend_vector_inreg(v4i32)) in combineShuffleToZeroExtendVectorInReg()
25342 // But not shuffle<z,z,1,-1> and not shuffle<0,z,z,-1> ! (for same types) in combineShuffleToZeroExtendVectorInReg()
25343 auto isZeroExtend = [NumElts, &ScaledMask](unsigned Scale) { in combineShuffleToZeroExtendVectorInReg() argument
25344 assert(Scale >= 2 && Scale <= NumElts && NumElts % Scale == 0 && in combineShuffleToZeroExtendVectorInReg()
25347 for (unsigned SrcElt = 0, NumSrcElts = NumElts / Scale; in combineShuffleToZeroExtendVectorInReg()
25349 // Analyze the shuffle mask in Scale-sized chunks. in combineShuffleToZeroExtendVectorInReg()
25350 ArrayRef<int> MaskChunk = Mask.take_front(Scale); in combineShuffleToZeroExtendVectorInReg()
25351 assert(MaskChunk.size() == Scale && "Unexpected mask size."); in combineShuffleToZeroExtendVectorInReg()
25354 // FIXME: undef should be fine, but that results in more-defined result. in combineShuffleToZeroExtendVectorInReg()
25358 // FIXME: undef should be fine, but that results in more-defined result. in combineShuffleToZeroExtendVectorInReg()
25360 [](int Indice) { return Indice == -2; })) in combineShuffleToZeroExtendVectorInReg()
25369 SDValue Op = SVN->getOperand(!Commuted ? 0 : 1); in combineShuffleToZeroExtendVectorInReg()
25389 EVT VT = SVN->getValueType(0); in combineTruncationShuffle()
25392 // TODO Add support for big-endian when we have a test case. in combineTruncationShuffle()
25396 SDValue N0 = peekThroughBitcasts(SVN->getOperand(0)); in combineTruncationShuffle()
25403 ArrayRef<int> Mask = SVN->getMask(); in combineTruncationShuffle()
25413 // (v4i32 truncate_vector_inreg(v2i64)) == shuffle<0,2-1,-1> in combineTruncationShuffle()
25414 // (v8i16 truncate_vector_inreg(v4i32)) == shuffle<0,2,4,6,-1,-1,-1,-1> in combineTruncationShuffle()
25415 // (v8i16 truncate_vector_inreg(v2i64)) == shuffle<0,4,-1,-1,-1,-1,-1,-1> in combineTruncationShuffle()
25416 auto isTruncate = [&Mask, &NumElts](unsigned Scale) { in combineTruncationShuffle() argument
25420 if ((i * Scale) < NumElts && Mask[i] == (int)(i * Scale)) in combineTruncationShuffle()
25434 // the same scale as the extension. in combineTruncationShuffle()
25441 // Combine shuffles of splat-shuffles of the form:
25442 // shuffle (shuffle V, undef, splat-mask), undef, M
25443 // If splat-mask contains undef elements, we need to be careful about
25448 EVT VT = Shuf->getValueType(0); in combineShuffleOfSplatVal()
25451 if (!Shuf->getOperand(1).isUndef()) in combineShuffleOfSplatVal()
25454 // See if this unary non-splat shuffle actually *is* a splat shuffle, in combineShuffleOfSplatVal()
25456 // FIXME: this can be done per-operand. in combineShuffleOfSplatVal()
25457 if (!Shuf->isSplat()) { in combineShuffleOfSplatVal()
25459 for (int Idx : Shuf->getMask()) { in combineShuffleOfSplatVal()
25462 assert((unsigned)Idx < NumElts && "Out-of-bounds shuffle indice?"); in combineShuffleOfSplatVal()
25467 if (DAG.isSplatValue(Shuf->getOperand(0), DemandedElts, UndefElts)) { in combineShuffleOfSplatVal()
25469 // Which lowest demanded element is *not* known-undef? in combineShuffleOfSplatVal()
25471 for (int Idx : Shuf->getMask()) { in combineShuffleOfSplatVal()
25477 return DAG.getUNDEF(VT); // All undef - result is undef. in combineShuffleOfSplatVal()
25479 SmallVector<int, 8> SplatMask(Shuf->getMask().begin(), in combineShuffleOfSplatVal()
25480 Shuf->getMask().end()); in combineShuffleOfSplatVal()
25484 // Otherwise, just pick the lowest demanded non-undef element. in combineShuffleOfSplatVal()
25485 // Or sentinel undef, if we know we'd pick a known-undef element. in combineShuffleOfSplatVal()
25486 Idx = UndefElts[Idx] ? -1 : *MinNonUndefIdx; in combineShuffleOfSplatVal()
25488 assert(SplatMask != Shuf->getMask() && "Expected mask to change!"); in combineShuffleOfSplatVal()
25489 return DAG.getVectorShuffle(VT, SDLoc(Shuf), Shuf->getOperand(0), in combineShuffleOfSplatVal()
25490 Shuf->getOperand(1), SplatMask); in combineShuffleOfSplatVal()
25497 if (DAG.isSplatValue(Shuf->getOperand(0), /*AllowUndefs*/ false)) in combineShuffleOfSplatVal()
25498 return Shuf->getOperand(0); in combineShuffleOfSplatVal()
25500 auto *Splat = dyn_cast<ShuffleVectorSDNode>(Shuf->getOperand(0)); in combineShuffleOfSplatVal()
25501 if (!Splat || !Splat->isSplat()) in combineShuffleOfSplatVal()
25504 ArrayRef<int> ShufMask = Shuf->getMask(); in combineShuffleOfSplatVal()
25505 ArrayRef<int> SplatMask = Splat->getMask(); in combineShuffleOfSplatVal()
25508 // Prefer simplifying to the splat-shuffle, if possible. This is legal if in combineShuffleOfSplatVal()
25509 // every undef mask element in the splat-shuffle has a corresponding undef in combineShuffleOfSplatVal()
25510 // element in the user-shuffle's mask or if the composition of mask elements in combineShuffleOfSplatVal()
25513 // * UserMask=[0,2,u,u], SplatMask=[2,u,2,u] -> [2,2,u,u] in combineShuffleOfSplatVal()
25514 // In this case it is not legal to simplify to the splat-shuffle because we in combineShuffleOfSplatVal()
25517 // * UserMask=[0,u,2,u], SplatMask=[2,u,2,u] -> [2,u,2,u] in combineShuffleOfSplatVal()
25519 // simplify to the splat-shuffle. in combineShuffleOfSplatVal()
25520 // * UserMask=[3,u,2,u], SplatMask=[2,u,2,u] -> [u,u,2,u] in combineShuffleOfSplatVal()
25523 // the splat-shuffle. in combineShuffleOfSplatVal()
25527 if (UserMask[i] != -1 && SplatMask[i] == -1 && in combineShuffleOfSplatVal()
25528 SplatMask[UserMask[i]] != -1) in combineShuffleOfSplatVal()
25533 return Shuf->getOperand(0); in combineShuffleOfSplatVal()
25539 NewMask.push_back(Idx == -1 ? -1 : SplatMask[Idx]); in combineShuffleOfSplatVal()
25541 return DAG.getVectorShuffle(Splat->getValueType(0), SDLoc(Splat), in combineShuffleOfSplatVal()
25542 Splat->getOperand(0), Splat->getOperand(1), in combineShuffleOfSplatVal()
25552 SDValue Op0 = SVN->getOperand(0); in combineShuffleOfBitcast()
25553 SDValue Op1 = SVN->getOperand(1); in combineShuffleOfBitcast()
25554 EVT VT = SVN->getValueType(0); in combineShuffleOfBitcast()
25576 ArrayRef<int> Mask = SVN->getMask(); in combineShuffleOfBitcast()
25594 /// shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X
25597 if (!OuterShuf->getOperand(1).isUndef()) in formSplatFromShuffles()
25599 auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(OuterShuf->getOperand(0)); in formSplatFromShuffles()
25600 if (!InnerShuf || !InnerShuf->getOperand(1).isUndef()) in formSplatFromShuffles()
25603 ArrayRef<int> OuterMask = OuterShuf->getMask(); in formSplatFromShuffles()
25604 ArrayRef<int> InnerMask = InnerShuf->getMask(); in formSplatFromShuffles()
25607 SmallVector<int, 32> CombinedMask(NumElts, -1); in formSplatFromShuffles()
25608 int SplatIndex = -1; in formSplatFromShuffles()
25612 if (OuterMaskElt == -1) in formSplatFromShuffles()
25617 if (InnerMaskElt == -1) in formSplatFromShuffles()
25621 if (SplatIndex == -1) in formSplatFromShuffles()
25624 // Non-matching index - this is not a splat. in formSplatFromShuffles()
25630 assert((all_of(CombinedMask, [](int M) { return M == -1; }) || in formSplatFromShuffles()
25631 getSplatIndex(CombinedMask) != -1) && in formSplatFromShuffles()
25635 EVT VT = OuterShuf->getValueType(0); in formSplatFromShuffles()
25636 assert(VT == InnerShuf->getValueType(0) && "Expected matching shuffle types"); in formSplatFromShuffles()
25640 return DAG.getVectorShuffle(VT, SDLoc(OuterShuf), InnerShuf->getOperand(0), in formSplatFromShuffles()
25641 InnerShuf->getOperand(1), CombinedMask); in formSplatFromShuffles()
25647 /// from the first operand. Otherwise, return -1.
25650 int EltFromOp0 = -1; in getShuffleMaskIndexOfOneElementFromOp0IntoOp1()
25652 // Should we ignore undefs in the shuffle mask instead? The trade-off is in getShuffleMaskIndexOfOneElementFromOp0IntoOp1()
25658 if (EltFromOp0 != -1) in getShuffleMaskIndexOfOneElementFromOp0IntoOp1()
25659 return -1; in getShuffleMaskIndexOfOneElementFromOp0IntoOp1()
25663 return -1; in getShuffleMaskIndexOfOneElementFromOp0IntoOp1()
25676 ArrayRef<int> Mask = Shuf->getMask(); in replaceShuffleOfInsert()
25678 SDValue Op0 = Shuf->getOperand(0); in replaceShuffleOfInsert()
25679 SDValue Op1 = Shuf->getOperand(1); in replaceShuffleOfInsert()
25681 if (ShufOp0Index == -1) { in replaceShuffleOfInsert()
25685 if (ShufOp0Index == -1) in replaceShuffleOfInsert()
25703 if (!InsIndexC || InsIndexC->getSExtValue() != Mask[ShufOp0Index]) in replaceShuffleOfInsert()
25710 // to a scalar-to-vector plus shuffle. in replaceShuffleOfInsert()
25715 // shuffle (insertelt v1, x, C), v2, mask --> insertelt v2, x, C' in replaceShuffleOfInsert()
25727 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Shuf->getOperand(0)); in simplifyShuffleOfShuffle()
25728 if (!Shuf0 || !Shuf->getOperand(1).isUndef()) in simplifyShuffleOfShuffle()
25731 ArrayRef<int> Mask = Shuf->getMask(); in simplifyShuffleOfShuffle()
25732 ArrayRef<int> Mask0 = Shuf0->getMask(); in simplifyShuffleOfShuffle()
25735 if (Mask[i] == -1) in simplifyShuffleOfShuffle()
25746 return Shuf->getOperand(0); in simplifyShuffleOfShuffle()
25750 EVT VT = N->getValueType(0); in visitVECTOR_SHUFFLE()
25753 SDValue N0 = N->getOperand(0); in visitVECTOR_SHUFFLE()
25754 SDValue N1 = N->getOperand(1); in visitVECTOR_SHUFFLE()
25758 // Canonicalize shuffle undef, undef -> undef in visitVECTOR_SHUFFLE()
25764 // Canonicalize shuffle v, v -> v, undef in visitVECTOR_SHUFFLE()
25767 createUnaryMask(SVN->getMask(), NumElts)); in visitVECTOR_SHUFFLE()
25769 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. in visitVECTOR_SHUFFLE()
25778 int Idx = SVN->getMaskElt(i); in visitVECTOR_SHUFFLE()
25780 Idx = -1; in visitVECTOR_SHUFFLE()
25801 if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) { in visitVECTOR_SHUFFLE()
25802 int SplatIndex = SVN->getSplatIndex(); in visitVECTOR_SHUFFLE()
25804 TLI.isBinOp(N0.getOpcode()) && N0->getNumValues() == 1) { in visitVECTOR_SHUFFLE()
25805 // splat (vector_bo L, R), Index --> in visitVECTOR_SHUFFLE()
25814 DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR, N0->getFlags()); in visitVECTOR_SHUFFLE()
25820 // splat(scalar_to_vector(x), 0) -> build_vector(x,...,x) in visitVECTOR_SHUFFLE()
25821 // splat(insert_vector_elt(v, x, c), c) -> build_vector(x,...,x) in visitVECTOR_SHUFFLE()
25829 if (Idx->getAPIntValue() == SplatIndex) in visitVECTOR_SHUFFLE()
25854 if (V->getOpcode() == ISD::BITCAST) { in visitVECTOR_SHUFFLE()
25855 SDValue ConvInput = V->getOperand(0); in visitVECTOR_SHUFFLE()
25861 if (V->getOpcode() == ISD::BUILD_VECTOR) { in visitVECTOR_SHUFFLE()
25862 assert(V->getNumOperands() == NumElts && in visitVECTOR_SHUFFLE()
25867 if (!V->getOperand(i).isUndef()) { in visitVECTOR_SHUFFLE()
25868 Base = V->getOperand(i); in visitVECTOR_SHUFFLE()
25876 if (V->getOperand(i) != Base) { in visitVECTOR_SHUFFLE()
25886 SDValue Splatted = V->getOperand(SplatIndex); in visitVECTOR_SHUFFLE()
25888 SDValue NewBV = DAG.getBuildVector(V->getValueType(0), SDLoc(N), Ops); in visitVECTOR_SHUFFLE()
25892 if (V->getValueType(0) != VT) in visitVECTOR_SHUFFLE()
25926 // only low-half elements of a concat with undef: in visitVECTOR_SHUFFLE()
25927 // shuf (concat X, X), undef, Mask --> shuf (concat X, undef), undef, Mask' in visitVECTOR_SHUFFLE()
25934 int Idx = SVN->getMaskElt(i); in visitVECTOR_SHUFFLE()
25937 Idx -= HalfNumElts; in visitVECTOR_SHUFFLE()
25952 // --> insert_subvector(lhs,rhs1,4). in visitVECTOR_SHUFFLE()
26001 ArrayRef<int> Mask = SVN->getMask(); in visitVECTOR_SHUFFLE()
26014 // shuffle into a AND node, with all the out-of-lane elements are known zero. in visitVECTOR_SHUFFLE()
26017 ArrayRef<int> Mask = SVN->getMask(); in visitVECTOR_SHUFFLE()
26018 SmallVector<int, 16> ClearMask(NumElts, -1); in visitVECTOR_SHUFFLE()
26041 // original type, incase the value is split into two (eg i64->i32). in visitVECTOR_SHUFFLE()
26067 // Attempt to combine a shuffle of 2 inputs of 'scalar sources' - in visitVECTOR_SHUFFLE()
26083 EVT InnerVT = BC0->getValueType(0); in visitVECTOR_SHUFFLE()
26096 // Scale the shuffle masks to the smaller scalar type. in visitVECTOR_SHUFFLE()
26100 narrowShuffleMaskElts(InnerScale, InnerSVN->getMask(), InnerMask); in visitVECTOR_SHUFFLE()
26101 narrowShuffleMaskElts(OuterScale, SVN->getMask(), OuterMask); in visitVECTOR_SHUFFLE()
26106 NewMask.push_back(M < 0 ? -1 : InnerMask[M]); in visitVECTOR_SHUFFLE()
26109 SDValue SV0 = BC0->getOperand(0); in visitVECTOR_SHUFFLE()
26110 SDValue SV1 = BC0->getOperand(1); in visitVECTOR_SHUFFLE()
26135 // i.e. Merge SVN(OtherSVN, N1) -> shuffle(SV0, SV1, Mask) iff Commute = false in visitVECTOR_SHUFFLE()
26136 // Merge SVN(N1, OtherSVN) -> shuffle(SV0, SV1, Mask') iff Commute = true in visitVECTOR_SHUFFLE()
26141 SmallVectorImpl<int> &Mask) -> bool { in visitVECTOR_SHUFFLE()
26144 if (OtherSVN->isSplat()) in visitVECTOR_SHUFFLE()
26151 int Idx = SVN->getMaskElt(i); in visitVECTOR_SHUFFLE()
26159 Idx = (Idx < (int)NumElts) ? (Idx + NumElts) : (Idx - NumElts); in visitVECTOR_SHUFFLE()
26165 Idx = OtherSVN->getMaskElt(Idx); in visitVECTOR_SHUFFLE()
26171 CurrentVec = (Idx < (int)NumElts) ? OtherSVN->getOperand(0) in visitVECTOR_SHUFFLE()
26172 : OtherSVN->getOperand(1); in visitVECTOR_SHUFFLE()
26180 Mask.push_back(-1); in visitVECTOR_SHUFFLE()
26202 // Last chance - see if the vector is another shuffle and if it in visitVECTOR_SHUFFLE()
26205 int InnerIdx = CurrentSVN->getMaskElt(Idx); in visitVECTOR_SHUFFLE()
26207 Mask.push_back(-1); in visitVECTOR_SHUFFLE()
26211 ? CurrentSVN->getOperand(0) in visitVECTOR_SHUFFLE()
26212 : CurrentSVN->getOperand(1); in visitVECTOR_SHUFFLE()
26214 Mask.push_back(-1); in visitVECTOR_SHUFFLE()
26236 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, B, M2) in visitVECTOR_SHUFFLE()
26237 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, C, M2) in visitVECTOR_SHUFFLE()
26238 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(B, C, M2) in visitVECTOR_SHUFFLE()
26239 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(B, A, M2) in visitVECTOR_SHUFFLE()
26240 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(C, A, M2) in visitVECTOR_SHUFFLE()
26241 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(C, B, M2) in visitVECTOR_SHUFFLE()
26252 // shuffle(A, shuffle(A, B)) -> shuffle(shuffle(A,B), A) in visitVECTOR_SHUFFLE()
26253 // shuffle(B, shuffle(A, B)) -> shuffle(shuffle(A,B), B) in visitVECTOR_SHUFFLE()
26254 // shuffle(B, shuffle(A, Undef)) -> shuffle(shuffle(A, Undef), B) in visitVECTOR_SHUFFLE()
26259 assert(N1->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
26262 SDValue SV0 = N1->getOperand(0); in visitVECTOR_SHUFFLE()
26263 SDValue SV1 = N1->getOperand(1); in visitVECTOR_SHUFFLE()
26272 // shuffle(splat(A,u), shuffle(C,D)) -> shuffle'(shuffle(C,D), splat(A,u)) in visitVECTOR_SHUFFLE()
26275 cast<ShuffleVectorSDNode>(N0)->isSplat() && in visitVECTOR_SHUFFLE()
26276 !cast<ShuffleVectorSDNode>(N1)->isSplat()) { in visitVECTOR_SHUFFLE()
26281 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, B, M2) in visitVECTOR_SHUFFLE()
26282 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, C, M2) in visitVECTOR_SHUFFLE()
26283 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(B, C, M2) in visitVECTOR_SHUFFLE()
26288 if (N->getOperand(i).getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
26289 N->isOnlyUserOf(N->getOperand(i).getNode())) { in visitVECTOR_SHUFFLE()
26292 auto *OtherSV = cast<ShuffleVectorSDNode>(N->getOperand(i)); in visitVECTOR_SHUFFLE()
26293 assert(OtherSV->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
26298 if (MergeInnerShuffle(i != 0, SVN, OtherSV, N->getOperand(1 - i), TLI, in visitVECTOR_SHUFFLE()
26316 if (TLI.isBinOp(SrcOpcode) && N->isOnlyUserOf(N0.getNode()) && in visitVECTOR_SHUFFLE()
26318 (SrcOpcode == N1.getOpcode() && N->isOnlyUserOf(N1.getNode())))) { in visitVECTOR_SHUFFLE()
26344 return SVN0 && InnerN->isOnlyUserOf(SVN0) && in visitVECTOR_SHUFFLE()
26347 (llvm::any_of(SVN0->getMask(), [](int M) { return M < 0; }) || in visitVECTOR_SHUFFLE()
26351 // Ensure we don't increase the number of shuffles - we must merge a in visitVECTOR_SHUFFLE()
26360 LeftMask.assign(SVN->getMask().begin(), SVN->getMask().end()); in visitVECTOR_SHUFFLE()
26371 RightMask.assign(SVN->getMask().begin(), SVN->getMask().end()); in visitVECTOR_SHUFFLE()
26404 EVT VT = N->getValueType(0); in visitSCALAR_TO_VECTOR()
26413 SDValue Scalar = N->getOperand(0); in visitSCALAR_TO_VECTOR()
26416 if (Scalar.hasOneUse() && Scalar->getNumValues() == 1 && in visitSCALAR_TO_VECTOR()
26420 Scalar->isOnlyUserOf(Scalar.getOperand(0).getNode()) && in visitSCALAR_TO_VECTOR()
26421 Scalar->isOnlyUserOf(Scalar.getOperand(1).getNode()) && in visitSCALAR_TO_VECTOR()
26424 SmallVector<int, 8> ShufMask(VT.getVectorNumElements(), -1); in visitSCALAR_TO_VECTOR()
26427 // s2v (bo (extelt V, Idx), C) --> shuffle (bo V, C'), {Idx, -1, -1...} in visitSCALAR_TO_VECTOR()
26428 // s2v (bo C, (extelt V, Idx)) --> shuffle (bo C', V), {Idx, -1, -1...} in visitSCALAR_TO_VECTOR()
26440 DAG.getConstant(C->getAPIntValue(), DL, VT)}; in visitSCALAR_TO_VECTOR()
26441 SDValue VecBO = DAG.getNode(Opcode, DL, VT, V[i], V[1 - i]); in visitSCALAR_TO_VECTOR()
26471 // Create a shuffle equivalent for scalar-to-vector: {ExtIndex, -1, -1, ...} in visitSCALAR_TO_VECTOR()
26472 SmallVector<int, 8> Mask(SrcNumElts, -1); in visitSCALAR_TO_VECTOR()
26473 Mask[0] = ExtIndexC->getZExtValue(); in visitSCALAR_TO_VECTOR()
26497 EVT VT = N->getValueType(0); in visitINSERT_SUBVECTOR()
26498 SDValue N0 = N->getOperand(0); in visitINSERT_SUBVECTOR()
26499 SDValue N1 = N->getOperand(1); in visitINSERT_SUBVECTOR()
26500 SDValue N2 = N->getOperand(2); in visitINSERT_SUBVECTOR()
26501 uint64_t InsIdx = N->getConstantOperandVal(2); in visitINSERT_SUBVECTOR()
26531 // insert_subvector(N0, extract_subvector(N0, N2), N2) --> N0 in visitINSERT_SUBVECTOR()
26537 // insert_subvector undef, (splat X), N2 -> splat X in visitINSERT_SUBVECTOR()
26544 // i.e. INSERT_SUBVECTOR UNDEF (BITCAST N1) N2 -> in visitINSERT_SUBVECTOR()
26558 // i.e. INSERT_SUBVECTOR (BITCAST N0) (BITCAST N1) N2 -> in visitINSERT_SUBVECTOR()
26576 // --> INSERT_SUBVECTOR( Vec, SubNew, Idx ) in visitINSERT_SUBVECTOR()
26584 // insert_subvector undef, (insert_subvector undef, X, 0), 0 --> in visitINSERT_SUBVECTOR()
26594 // -> bitcast(insert_subvector(v, s, c2)) in visitINSERT_SUBVECTOR()
26610 unsigned Scale = EltSizeInBits / N1SrcSVT.getSizeInBits(); in visitINSERT_SUBVECTOR() local
26611 NewVT = EVT::getVectorVT(Ctx, N1SrcSVT, NumElts * Scale); in visitINSERT_SUBVECTOR()
26612 NewIdx = DAG.getVectorIdxConstant(InsIdx * Scale, DL); in visitINSERT_SUBVECTOR()
26614 unsigned Scale = N1SrcSVT.getSizeInBits() / EltSizeInBits; in visitINSERT_SUBVECTOR() local
26615 if (NumElts.isKnownMultipleOf(Scale) && (InsIdx % Scale) == 0) { in visitINSERT_SUBVECTOR()
26617 NumElts.divideCoefficientBy(Scale)); in visitINSERT_SUBVECTOR()
26618 NewIdx = DAG.getVectorIdxConstant(InsIdx / Scale, DL); in visitINSERT_SUBVECTOR()
26632 // -> (insert_subvector (insert_subvector A, Idx1), Idx0) in visitINSERT_SUBVECTOR()
26653 SmallVector<SDValue, 8> Ops(N0->op_begin(), N0->op_end()); in visitINSERT_SUBVECTOR()
26666 SDValue N0 = N->getOperand(0); in visitFP_TO_FP16()
26668 // fold (fp_to_fp16 (fp16_to_fp op)) -> op in visitFP_TO_FP16()
26669 if (N0->getOpcode() == ISD::FP16_TO_FP) in visitFP_TO_FP16()
26670 return N0->getOperand(0); in visitFP_TO_FP16()
26676 auto Op = N->getOpcode(); in visitFP16_TO_FP()
26679 SDValue N0 = N->getOperand(0); in visitFP16_TO_FP()
26681 // fold fp16_to_fp(op & 0xffff) -> fp16_to_fp(op) or in visitFP16_TO_FP()
26682 // fold bf16_to_fp(op & 0xffff) -> bf16_to_fp(op) in visitFP16_TO_FP()
26683 if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) { in visitFP16_TO_FP()
26685 if (AndConst && AndConst->getAPIntValue() == 0xffff) { in visitFP16_TO_FP()
26686 return DAG.getNode(Op, SDLoc(N), N->getValueType(0), N0.getOperand(0)); in visitFP16_TO_FP()
26693 SDValue Folded = DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), in visitFP16_TO_FP()
26694 N->getValueType(0), {N0}); in visitFP16_TO_FP()
26699 SDValue N0 = N->getOperand(0); in visitFP_TO_BF16()
26701 // fold (fp_to_bf16 (bf16_to_fp op)) -> op in visitFP_TO_BF16()
26702 if (N0->getOpcode() == ISD::BF16_TO_FP) in visitFP_TO_BF16()
26703 return N0->getOperand(0); in visitFP_TO_BF16()
26709 // fold bf16_to_fp(op & 0xffff) -> bf16_to_fp(op) in visitBF16_TO_FP()
26714 SDValue N0 = N->getOperand(0); in visitVECREDUCE()
26716 unsigned Opcode = N->getOpcode(); in visitVECREDUCE()
26718 // VECREDUCE over 1-element vector is just an extract. in visitVECREDUCE()
26724 if (Res.getValueType() != N->getValueType(0)) in visitVECREDUCE()
26725 Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res); in visitVECREDUCE()
26737 return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), N0); in visitVECREDUCE()
26740 // vecreduce_or(insert_subvector(zero or undef, val)) -> vecreduce_or(val) in visitVECREDUCE()
26741 // vecreduce_and(insert_subvector(ones or undef, val)) -> vecreduce_and(val) in visitVECREDUCE()
26750 return DAG.getNode(Opcode, SDLoc(N), N->getValueType(0), Subvec); in visitVECREDUCE()
26759 // FSUB -> FMA combines: in visitVP_FSUB()
26769 if (N->getOpcode() == ISD::VP_GATHER) in visitVPOp()
26773 if (N->getOpcode() == ISD::VP_SCATTER) in visitVPOp()
26777 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD) in visitVPOp()
26781 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE) in visitVPOp()
26785 // VP operations in which all vector elements are disabled - either by in visitVPOp()
26786 // determining that the mask is all false or that the EVL is 0 - can be in visitVPOp()
26789 if (auto EVLIdx = ISD::getVPExplicitVectorLengthIdx(N->getOpcode())) in visitVPOp()
26790 AreAllEltsDisabled |= isNullConstant(N->getOperand(*EVLIdx)); in visitVPOp()
26791 if (auto MaskIdx = ISD::getVPMaskIdx(N->getOpcode())) in visitVPOp()
26793 ISD::isConstantSplatVectorAllZeros(N->getOperand(*MaskIdx).getNode()); in visitVPOp()
26797 switch (N->getOpcode()) { in visitVPOp()
26815 if (ISD::isVPBinaryOp(N->getOpcode())) in visitVPOp()
26816 return DAG.getUNDEF(N->getValueType(0)); in visitVPOp()
26821 if (MemSD->writeMem()) in visitVPOp()
26822 return MemSD->getChain(); in visitVPOp()
26823 return CombineTo(N, DAG.getUNDEF(N->getValueType(0)), MemSD->getChain()); in visitVPOp()
26827 if (ISD::isVPReduction(N->getOpcode())) in visitVPOp()
26828 return N->getOperand(0); in visitVPOp()
26834 SDValue Chain = N->getOperand(0); in visitGET_FPENV_MEM()
26835 SDValue Ptr = N->getOperand(1); in visitGET_FPENV_MEM()
26836 EVT MemVT = cast<FPStateAccessSDNode>(N)->getMemoryVT(); in visitGET_FPENV_MEM()
26841 for (auto *U : Ptr->uses()) { in visitGET_FPENV_MEM()
26852 if (!LdNode || !LdNode->isSimple() || LdNode->isIndexed() || in visitGET_FPENV_MEM()
26853 !LdNode->getOffset().isUndef() || LdNode->getMemoryVT() != MemVT || in visitGET_FPENV_MEM()
26854 !LdNode->getChain().reachesChainWithoutSideEffects(SDValue(N, 0))) in visitGET_FPENV_MEM()
26859 for (auto I = LdNode->use_begin(), E = LdNode->use_end(); I != E; ++I) { in visitGET_FPENV_MEM()
26871 if (!StNode || !StNode->isSimple() || StNode->isIndexed() || in visitGET_FPENV_MEM()
26872 !StNode->getOffset().isUndef() || StNode->getMemoryVT() != MemVT || in visitGET_FPENV_MEM()
26873 !StNode->getChain().reachesChainWithoutSideEffects(SDValue(LdNode, 1))) in visitGET_FPENV_MEM()
26878 SDValue Res = DAG.getGetFPEnv(Chain, SDLoc(N), StNode->getBasePtr(), MemVT, in visitGET_FPENV_MEM()
26879 StNode->getMemOperand()); in visitGET_FPENV_MEM()
26885 SDValue Chain = N->getOperand(0); in visitSET_FPENV_MEM()
26886 SDValue Ptr = N->getOperand(1); in visitSET_FPENV_MEM()
26887 EVT MemVT = cast<FPStateAccessSDNode>(N)->getMemoryVT(); in visitSET_FPENV_MEM()
26891 for (auto *U : Ptr->uses()) { in visitSET_FPENV_MEM()
26902 if (!StNode || !StNode->isSimple() || StNode->isIndexed() || in visitSET_FPENV_MEM()
26903 !StNode->getOffset().isUndef() || StNode->getMemoryVT() != MemVT || in visitSET_FPENV_MEM()
26909 SDValue StValue = StNode->getValue(); in visitSET_FPENV_MEM()
26911 if (!LdNode || !LdNode->isSimple() || LdNode->isIndexed() || in visitSET_FPENV_MEM()
26912 !LdNode->getOffset().isUndef() || LdNode->getMemoryVT() != MemVT || in visitSET_FPENV_MEM()
26913 !StNode->getChain().reachesChainWithoutSideEffects(SDValue(LdNode, 1))) in visitSET_FPENV_MEM()
26919 DAG.getSetFPEnv(LdNode->getChain(), SDLoc(N), LdNode->getBasePtr(), MemVT, in visitSET_FPENV_MEM()
26920 LdNode->getMemOperand()); in visitSET_FPENV_MEM()
26929 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!"); in XformToShuffleWithZero()
26931 EVT VT = N->getValueType(0); in XformToShuffleWithZero()
26932 SDValue LHS = N->getOperand(0); in XformToShuffleWithZero()
26933 SDValue RHS = peekThroughBitcasts(N->getOperand(1)); in XformToShuffleWithZero()
26949 // all zeros or all ones - suitable for shuffle masking. in XformToShuffleWithZero()
26959 // X & undef --> 0 (not undef). So this lane must be converted to choose in XformToShuffleWithZero()
26960 // from the zero constant vector (same as if the element had all 0-bits). in XformToShuffleWithZero()
26968 Bits = Cst->getAPIntValue(); in XformToShuffleWithZero()
26970 Bits = CstFP->getValueAPF().bitcastToAPInt(); in XformToShuffleWithZero()
26976 Bits = Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits); in XformToShuffleWithZero()
27017 SDValue N0 = N->getOperand(0); in scalarizeBinOpOfSplats()
27018 SDValue N1 = N->getOperand(1); in scalarizeBinOpOfSplats()
27019 unsigned Opcode = N->getOpcode(); in scalarizeBinOpOfSplats()
27020 EVT VT = N->getValueType(0); in scalarizeBinOpOfSplats()
27044 SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, X, Y, N->getFlags()); in scalarizeBinOpOfSplats()
27049 count_if(N0->ops(), [](SDValue V) { return !V.isUndef(); }) == 1 && in scalarizeBinOpOfSplats()
27050 count_if(N1->ops(), [](SDValue V) { return !V.isUndef(); }) == 1) { in scalarizeBinOpOfSplats()
27051 // bo (build_vec ..undef, X, undef...), (build_vec ..undef, Y, undef...) --> in scalarizeBinOpOfSplats()
27058 // bo (splat X, Index), (splat Y, Index) --> splat (bo X, Y), Index in scalarizeBinOpOfSplats()
27064 EVT VT = N->getValueType(0); in SimplifyVCastOp()
27067 unsigned Opcode = N->getOpcode(); in SimplifyVCastOp()
27069 SDValue N0 = N->getOperand(0); in SimplifyVCastOp()
27085 SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, Elt, N->getFlags()); in SimplifyVCastOp()
27097 EVT VT = N->getValueType(0); in SimplifyVBinOp()
27100 SDValue LHS = N->getOperand(0); in SimplifyVBinOp()
27101 SDValue RHS = N->getOperand(1); in SimplifyVBinOp()
27102 unsigned Opcode = N->getOpcode(); in SimplifyVBinOp()
27103 SDNodeFlags Flags = N->getFlags(); in SimplifyVBinOp()
27107 // --> shuffle (VBinOp A, B), Undef, Mask in SimplifyVBinOp()
27110 // restrict ops like integer div that have immediate UB (eg, div-by-zero) in SimplifyVBinOp()
27115 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) && in SimplifyVBinOp()
27121 return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); in SimplifyVBinOp()
27126 // undefined elements because that could be poison-unsafe or inhibit in SimplifyVBinOp()
27129 // load-folding or other target-specific behaviors. in SimplifyVBinOp()
27130 if (isConstOrConstSplat(RHS) && Shuf0 && all_equal(Shuf0->getMask()) && in SimplifyVBinOp()
27131 Shuf0->hasOneUse() && Shuf0->getOperand(1).isUndef() && in SimplifyVBinOp()
27132 Shuf0->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
27133 // binop (splat X), (splat C) --> splat (binop X, C) in SimplifyVBinOp()
27134 SDValue X = Shuf0->getOperand(0); in SimplifyVBinOp()
27137 Shuf0->getMask()); in SimplifyVBinOp()
27139 if (isConstOrConstSplat(LHS) && Shuf1 && all_equal(Shuf1->getMask()) && in SimplifyVBinOp()
27140 Shuf1->hasOneUse() && Shuf1->getOperand(1).isUndef() && in SimplifyVBinOp()
27141 Shuf1->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
27142 // binop (splat C), (splat X) --> splat (binop C, X) in SimplifyVBinOp()
27143 SDValue X = Shuf1->getOperand(0); in SimplifyVBinOp()
27146 Shuf1->getMask()); in SimplifyVBinOp()
27153 // VBinOp (ins undef, X, Z), (ins undef, Y, Z) --> ins VecC, (VBinOp X, Y), Z in SimplifyVBinOp()
27176 all_of(drop_begin(Concat->ops()), [](const SDValue &Op) { in SimplifyVBinOp()
27185 // VBinOp (concat X, undef/constant), (concat Y, undef/constant) --> in SimplifyVBinOp()
27216 cast<CondCodeSDNode>(N0.getOperand(2))->get()); in SimplifySelect()
27225 const SDNodeFlags Flags = N0->getFlags(); in SimplifySelect()
27233 SelectNode->setFlags(Flags); in SimplifySelect()
27245 /// should return the appropriate thing (e.g. the node) back to the top-level of
27249 // fold (select (setcc x, [+-]0.0, *lt), NaN, (fsqrt x)) in SimplifySelectOps()
27252 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) { in SimplifySelectOps()
27259 if (TheSelect->getOpcode() == ISD::SELECT_CC) { in SimplifySelectOps()
27260 CC = cast<CondCodeSDNode>(TheSelect->getOperand(4))->get(); in SimplifySelectOps()
27261 CmpLHS = TheSelect->getOperand(0); in SimplifySelectOps()
27262 Zero = isConstOrConstSplatFP(TheSelect->getOperand(1)); in SimplifySelectOps()
27265 SDValue Cmp = TheSelect->getOperand(0); in SimplifySelectOps()
27267 CC = cast<CondCodeSDNode>(Cmp.getOperand(2))->get(); in SimplifySelectOps()
27272 if (Zero && Zero->isZero() && in SimplifySelectOps()
27275 // We have: (select (setcc x, [+-]0.0, *lt), NaN, (fsqrt x)) in SimplifySelectOps()
27282 if (TheSelect->getOperand(0).getValueType().isVector()) return false; in SimplifySelectOps()
27303 !LLD->isSimple() || !RLD->isSimple() || in SimplifySelectOps()
27306 LLD->isIndexed() || RLD->isIndexed() || in SimplifySelectOps()
27308 LLD->getMemoryVT() != RLD->getMemoryVT() || in SimplifySelectOps()
27310 (LLD->getExtensionType() != RLD->getExtensionType() && in SimplifySelectOps()
27312 LLD->getExtensionType() != ISD::EXTLOAD && in SimplifySelectOps()
27313 RLD->getExtensionType() != ISD::EXTLOAD) || in SimplifySelectOps()
27315 // over-conservative. It would be beneficial to be able to remember in SimplifySelectOps()
27319 LLD->getPointerInfo().getAddrSpace() != 0 || in SimplifySelectOps()
27320 RLD->getPointerInfo().getAddrSpace() != 0 || in SimplifySelectOps()
27323 LLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
27324 RLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
27325 !TLI.isOperationLegalOrCustom(TheSelect->getOpcode(), in SimplifySelectOps()
27326 LLD->getBasePtr().getValueType())) in SimplifySelectOps()
27330 if (LLD->isPredecessorOf(RLD) || RLD->isPredecessorOf(LLD)) in SimplifySelectOps()
27352 if (TheSelect->getOpcode() == ISD::SELECT) { in SimplifySelectOps()
27358 SDNode *CondNode = TheSelect->getOperand(0).getNode(); in SimplifySelectOps()
27361 if ((LLD->hasAnyUseOfValue(1) && in SimplifySelectOps()
27363 (RLD->hasAnyUseOfValue(1) && in SimplifySelectOps()
27368 LLD->getBasePtr().getValueType(), in SimplifySelectOps()
27369 TheSelect->getOperand(0), LLD->getBasePtr(), in SimplifySelectOps()
27370 RLD->getBasePtr()); in SimplifySelectOps()
27378 SDNode *CondLHS = TheSelect->getOperand(0).getNode(); in SimplifySelectOps()
27379 SDNode *CondRHS = TheSelect->getOperand(1).getNode(); in SimplifySelectOps()
27383 if ((LLD->hasAnyUseOfValue(1) && in SimplifySelectOps()
27385 (RLD->hasAnyUseOfValue(1) && in SimplifySelectOps()
27390 LLD->getBasePtr().getValueType(), in SimplifySelectOps()
27391 TheSelect->getOperand(0), in SimplifySelectOps()
27392 TheSelect->getOperand(1), in SimplifySelectOps()
27393 LLD->getBasePtr(), RLD->getBasePtr(), in SimplifySelectOps()
27394 TheSelect->getOperand(4)); in SimplifySelectOps()
27401 Align Alignment = std::min(LLD->getAlign(), RLD->getAlign()); in SimplifySelectOps()
27402 MachineMemOperand::Flags MMOFlags = LLD->getMemOperand()->getFlags(); in SimplifySelectOps()
27403 if (!RLD->isInvariant()) in SimplifySelectOps()
27405 if (!RLD->isDereferenceable()) in SimplifySelectOps()
27407 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { in SimplifySelectOps()
27409 Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect), in SimplifySelectOps()
27410 LLD->getChain(), Addr, MachinePointerInfo(), Alignment, in SimplifySelectOps()
27415 LLD->getExtensionType() == ISD::EXTLOAD ? RLD->getExtensionType() in SimplifySelectOps()
27416 : LLD->getExtensionType(), in SimplifySelectOps()
27417 SDLoc(TheSelect), TheSelect->getValueType(0), LLD->getChain(), Addr, in SimplifySelectOps()
27418 MachinePointerInfo(), LLD->getMemoryVT(), Alignment, MMOFlags); in SimplifySelectOps()
27425 // old-load value is dead now. in SimplifySelectOps()
27441 // select_cc setlt X, 0, A, 0 -> and (sra X, size(X)-1), A in foldSelectCCToShiftAnd()
27442 // select_cc setgt X, 0, A, 0 -> and (not (sra X, size(X)-1)), A in foldSelectCCToShiftAnd()
27452 // (X > -1) ? A : 0 in foldSelectCCToShiftAnd()
27453 // (X > 0) ? X : 0 <-- This is canonical signed max. in foldSelectCCToShiftAnd()
27458 // (X < 1) ? X : 0 <-- This is un-canonicalized signed min. in foldSelectCCToShiftAnd()
27465 // and (sra X, size(X)-1), A -> "and (srl X, C2), A" iff A is a single-bit in foldSelectCCToShiftAnd()
27468 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) { in foldSelectCCToShiftAnd()
27469 unsigned ShCt = XType.getSizeInBits() - N2C->getAPIntValue().logBase2() - 1; in foldSelectCCToShiftAnd()
27487 unsigned ShCt = XType.getSizeInBits() - 1; in foldSelectCCToShiftAnd()
27506 // Fold select(cc, binop(), binop()) -> binop(select(), select()) etc.
27508 SDValue N0 = N->getOperand(0); in foldSelectOfBinops()
27509 SDValue N1 = N->getOperand(1); in foldSelectOfBinops()
27510 SDValue N2 = N->getOperand(2); in foldSelectOfBinops()
27522 if (!N0->hasOneUse() || !N1->hasOneUse() || !N2->hasOneUse()) in foldSelectOfBinops()
27527 SDVTList OpVTs = N1->getVTList(); in foldSelectOfBinops()
27530 // --> binop(select(cond, x, z), y) in foldSelectOfBinops()
27536 NewBinOp->setFlags(N1->getFlags()); in foldSelectOfBinops()
27537 NewBinOp->intersectFlagsWith(N2->getFlags()); in foldSelectOfBinops()
27542 // --> binop(x, select(cond, y, z)) in foldSelectOfBinops()
27551 NewBinOp->setFlags(N1->getFlags()); in foldSelectOfBinops()
27552 NewBinOp->intersectFlagsWith(N2->getFlags()); in foldSelectOfBinops()
27563 SDValue N0 = N->getOperand(0); in foldSignChangeInBitcast()
27564 EVT VT = N->getValueType(0); in foldSignChangeInBitcast()
27565 bool IsFabs = N->getOpcode() == ISD::FABS; in foldSignChangeInBitcast()
27578 // (fneg (bitconvert x)) -> (bitconvert (xor x sign)) in foldSignChangeInBitcast()
27579 // (fabs (bitconvert x)) -> (bitconvert (and x ~sign)) in foldSignChangeInBitcast()
27621 TLI.isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) || in convertSelectOfFPConstantsToLoadOffset()
27622 TLI.isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize)) in convertSelectOfFPConstantsToLoadOffset()
27627 if (!TV->hasOneUse() && !FV->hasOneUse()) in convertSelectOfFPConstantsToLoadOffset()
27630 Constant *Elts[] = { const_cast<ConstantFP*>(FV->getConstantFPValue()), in convertSelectOfFPConstantsToLoadOffset()
27631 const_cast<ConstantFP*>(TV->getConstantFPValue()) }; in convertSelectOfFPConstantsToLoadOffset()
27632 Type *FPTy = Elts[0]->getType(); in convertSelectOfFPConstantsToLoadOffset()
27639 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign(); in convertSelectOfFPConstantsToLoadOffset()
27644 unsigned EltSize = (unsigned)TD.getTypeAllocSize(Elts[0]->getType()); in convertSelectOfFPConstantsToLoadOffset()
27653 return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx, in convertSelectOfFPConstantsToLoadOffset()
27663 // (x ? y : y) -> y. in SimplifySelectCC()
27677 // fold select_cc true, x, y -> x in SimplifySelectCC()
27678 // fold select_cc false, x, y -> y in SimplifySelectCC()
27679 return !(SCCC->isZero()) ? N2 : N3; in SimplifySelectCC()
27690 // fold (select_cc seteq (and x, y), 0, 0, A) -> (and (sra (shl x)) A) in SimplifySelectCC()
27693 // when the condition can be materialized as an all-ones register. Any in SimplifySelectCC()
27694 // single bit-test can be materialized as an all-ones register with in SimplifySelectCC()
27695 // shift-left and shift-right-arith. in SimplifySelectCC()
27696 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && in SimplifySelectCC()
27697 N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) { in SimplifySelectCC()
27698 SDValue AndLHS = N0->getOperand(0); in SimplifySelectCC()
27699 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1)); in SimplifySelectCC()
27700 if (ConstAndRHS && ConstAndRHS->getAPIntValue().popcount() == 1) { in SimplifySelectCC()
27702 const APInt &AndMask = ConstAndRHS->getAPIntValue(); in SimplifySelectCC()
27704 unsigned ShCt = AndMask.getBitWidth() - 1; in SimplifySelectCC()
27710 // either all-ones, or zero. in SimplifySelectCC()
27719 // fold select C, 16, 0 -> shl C, 4 in SimplifySelectCC()
27720 bool Fold = N2C && isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2(); in SimplifySelectCC()
27721 bool Swap = N3C && isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2(); in SimplifySelectCC()
27735 if (NotExtCompare && N2C->isOne()) in SimplifySelectCC()
27751 if (N2C->isOne()) in SimplifySelectCC()
27754 unsigned ShCt = N2C->getAPIntValue().logBase2(); in SimplifySelectCC()
27764 // select_cc seteq X, 0, sizeof(X), ctlz(X) -> ctlz(X) in SimplifySelectCC()
27765 // select_cc seteq X, 0, sizeof(X), ctlz_zero_undef(X) -> ctlz(X) in SimplifySelectCC()
27766 // select_cc seteq X, 0, sizeof(X), cttz(X) -> cttz(X) in SimplifySelectCC()
27767 // select_cc seteq X, 0, sizeof(X), cttz_zero_undef(X) -> cttz(X) in SimplifySelectCC()
27768 // select_cc setne X, 0, ctlz(X), sizeof(X) -> ctlz(X) in SimplifySelectCC()
27769 // select_cc setne X, 0, ctlz_zero_undef(X), sizeof(X) -> ctlz(X) in SimplifySelectCC()
27770 // select_cc setne X, 0, cttz(X), sizeof(X) -> cttz(X) in SimplifySelectCC()
27771 // select_cc setne X, 0, cttz_zero_undef(X), sizeof(X) -> cttz(X) in SimplifySelectCC()
27772 if (N1C && N1C->isZero() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { in SimplifySelectCC()
27780 if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) { in SimplifySelectCC()
27799 // Fold select_cc setgt X, -1, C, ~C -> xor (ashr X, BW-1), C in SimplifySelectCC()
27800 // Fold select_cc setlt X, 0, C, ~C -> xor (ashr X, BW-1), ~C in SimplifySelectCC()
27802 N2C->getAPIntValue() == ~N3C->getAPIntValue() && in SimplifySelectCC()
27803 ((N1C->isAllOnes() && CC == ISD::SETGT) || in SimplifySelectCC()
27804 (N1C->isZero() && CC == ISD::SETLT)) && in SimplifySelectCC()
27805 !TLI.shouldAvoidTransformToShift(VT, CmpOpVT.getScalarSizeInBits() - 1)) { in SimplifySelectCC()
27808 DAG.getConstant(CmpOpVT.getScalarSizeInBits() - 1, DL, CmpOpVT)); in SimplifySelectCC()
27853 ConstantSDNode *C = isConstOrConstSplat(N->getOperand(1)); in BuildSDIVPow2()
27858 if (C->isZero()) in BuildSDIVPow2()
27862 if (SDValue S = TLI.BuildSDIVPow2(N, C->getAPIntValue(), DAG, Built)) { in BuildSDIVPow2()
27894 ConstantSDNode *C = isConstOrConstSplat(N->getOperand(1)); in BuildSREMPow2()
27899 if (C->isZero()) in BuildSREMPow2()
27903 if (SDValue S = TLI.BuildSREMPow2(N, C->getAPIntValue(), DAG, Built)) { in BuildSREMPow2()
27919 // This will only return `Log2(Op)` if we can prove `Op` is non-zero. Set
27921 // `Op` is non-zero).
27945 // Helper for determining whether a value is a power-2 constant scalar or a in takeInexpensiveLog2()
27949 if (C->isZero() || C->isOpaque()) in takeInexpensiveLog2()
27952 if (C->getAPIntValue().isPowerOf2()) { in takeInexpensiveLog2()
27953 Pow2Constants.emplace_back(C->getAPIntValue()); in takeInexpensiveLog2()
27989 // log2(X << Y) -> log2(X) + Y in takeInexpensiveLog2()
27991 // 1 << Y and X nuw/nsw << Y are all non-zero. in takeInexpensiveLog2()
27992 if (AssumeNonZero || Op->getFlags().hasNoUnsignedWrap() || in takeInexpensiveLog2()
27993 Op->getFlags().hasNoSignedWrap() || isOneConstant(Op.getOperand(0))) in takeInexpensiveLog2()
28000 // c ? X : Y -> c ? Log2(X) : Log2(Y) in takeInexpensiveLog2()
28010 // log2(umin(X, Y)) -> umin(log2(X), log2(Y)) in takeInexpensiveLog2()
28011 // log2(umax(X, Y)) -> umax(log2(X), log2(Y)) in takeInexpensiveLog2()
28028 /// Determines the LogBase2 value for a non-null input value using the
28029 /// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
28040 SDValue Base = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in BuildLogBase2()
28045 /// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
28047 /// F(X) = 1/X - A [which has a zero at X = 1/A]
28049 /// X_{i+1} = X_i (2 - A X_i) = X_i + X_i (1 - A X_i) [this second form
28052 /// Result = N X_i + X_i (N - N A X_i)
28080 // Newton iterations: Est = Est + Est (N - Arg * Est) in BuildDivEstimate()
28085 if (i == Iterations - 1) { in BuildDivEstimate()
28094 (i == Iterations - 1 ? N : FPOne), NewEst, Flags); in BuildDivEstimate()
28115 /// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
28117 /// F(X) = 1/X^2 - A [which has a zero at X = 1/sqrt(A)]
28119 /// X_{i+1} = X_i (1.5 - A X_i^2 / 2)
28128 // We now need 0.5 * Arg which we can write as (1.5 * Arg - Arg) so that in buildSqrtNROneConst()
28133 // Newton iterations: Est = Est * (1.5 - HalfArg * Est * Est) in buildSqrtNROneConst()
28141 // If non-reciprocal square root is requested, multiply the result by Arg. in buildSqrtNROneConst()
28148 /// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
28150 /// F(X) = 1/X^2 - A [which has a zero at X = 1/sqrt(A)]
28152 /// X_{i+1} = (-0.5 * X_i) * (A * X_i * X_i + (-3.0))
28158 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT); in buildSqrtNRTwoConst()
28159 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT); in buildSqrtNRTwoConst()
28166 // E = (E * -0.5) * ((A * E) * E + -3.0) in buildSqrtNRTwoConst()
28173 // S = ((A * E) * -0.5) * ((A * E) * E + -3.0) in buildSqrtNRTwoConst()
28177 // RSQRT: LHS = (E * -0.5) in buildSqrtNRTwoConst()
28180 // SQRT: LHS = (A * E) * -0.5 in buildSqrtNRTwoConst()
28262 auto getCharacteristics = [](SDNode *N) -> MemUseCharacteristics { in mayAlias()
28265 if (auto *C = dyn_cast<ConstantSDNode>(LSN->getOffset())) in mayAlias()
28266 Offset = (LSN->getAddressingMode() == ISD::PRE_INC) ? C->getSExtValue() in mayAlias()
28267 : (LSN->getAddressingMode() == ISD::PRE_DEC) in mayAlias()
28268 ? -1 * C->getSExtValue() in mayAlias()
28270 TypeSize Size = LSN->getMemoryVT().getStoreSize(); in mayAlias()
28271 return {LSN->isVolatile(), LSN->isAtomic(), in mayAlias()
28272 LSN->getBasePtr(), Offset /*base offset*/, in mayAlias()
28273 LocationSize::precise(Size), LSN->getMemOperand()}; in mayAlias()
28278 LN->getOperand(1), in mayAlias()
28279 (LN->hasOffset()) ? LN->getOffset() : 0, in mayAlias()
28280 (LN->hasOffset()) ? LocationSize::precise(LN->getSize()) in mayAlias()
28310 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) || in mayAlias()
28311 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore())) in mayAlias()
28337 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) || in mayAlias()
28338 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore())) in mayAlias()
28346 int64_t SrcValOffset0 = MUC0.MMO->getOffset(); in mayAlias()
28347 int64_t SrcValOffset1 = MUC1.MMO->getOffset(); in mayAlias()
28348 Align OrigAlignment0 = MUC0.MMO->getBaseAlign(); in mayAlias()
28349 Align OrigAlignment1 = MUC1.MMO->getBaseAlign(); in mayAlias()
28380 if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && in mayAlias()
28388 Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset; in mayAlias()
28390 Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset; in mayAlias()
28395 if (AA->isNoAlias( in mayAlias()
28396 MemoryLocation(MUC0.MMO->getValue(), Loc0, in mayAlias()
28397 UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()), in mayAlias()
28398 MemoryLocation(MUC1.MMO->getValue(), Loc1, in mayAlias()
28399 UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes()))) in mayAlias()
28407 /// Walk up chain skipping non-aliasing memory nodes,
28416 const bool IsLoad = isa<LoadSDNode>(N) && cast<LoadSDNode>(N)->isSimple(); in GatherAllAliases()
28423 auto ImproveChain = [&](SDValue &C) -> bool { in GatherAllAliases()
28434 cast<LSBaseSDNode>(C.getNode())->isSimple(); in GatherAllAliases()
28478 // FIXME: The depth check could be made to return the last non-aliasing in GatherAllAliases()
28497 Chains.push_back(Chain.getOperand(--n)); in GatherAllAliases()
28514 /// Walk up chain skipping non-aliasing memory nodes, looking for a better chain
28545 // we improve the chains of all the potential candidates up-front
28548 // to go from a partially-merged state to the desired final
28549 // fully-merged state.
28576 if (St->getMemoryVT().isZeroSized()) in parallelizeChainedStores()
28579 // BaseIndexOffset assumes that offsets are fixed-size, which in parallelizeChainedStores()
28582 if (St->getMemoryVT().isScalableVT()) in parallelizeChainedStores()
28586 Intervals.insert(0, (St->getMemoryVT().getSizeInBits() + 7) / 8, in parallelizeChainedStores()
28589 while (StoreSDNode *Chain = dyn_cast<StoreSDNode>(STChain->getChain())) { in parallelizeChainedStores()
28590 if (Chain->getMemoryVT().isScalableVector()) in parallelizeChainedStores()
28594 if (!SDValue(Chain, 0)->hasOneUse()) in parallelizeChainedStores()
28597 if (!Chain->isSimple() || Chain->isIndexed()) in parallelizeChainedStores()
28606 int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8; in parallelizeChainedStores()
28614 if (I != Intervals.begin() && (--I).stop() <= Offset) in parallelizeChainedStores()
28628 SDValue NewChain = STChain->getChain(); in parallelizeChainedStores()
28631 StoreSDNode *S = ChainedStores[--I]; in parallelizeChainedStores()
28634 S, BetterChain, S->getOperand(1), S->getOperand(2), S->getOperand(3))); in parallelizeChainedStores()
28642 if (St->isTruncatingStore()) in parallelizeChainedStores()
28643 NewST = DAG.getTruncStore(BetterChain, SDLoc(St), St->getValue(), in parallelizeChainedStores()
28644 St->getBasePtr(), St->getMemoryVT(), in parallelizeChainedStores()
28645 St->getMemOperand()); in parallelizeChainedStores()
28647 NewST = DAG.getStore(BetterChain, SDLoc(St), St->getValue(), in parallelizeChainedStores()
28648 St->getBasePtr(), St->getMemOperand()); in parallelizeChainedStores()
28655 auto hasImprovedChain = [&](SDValue ST) -> bool { in parallelizeChainedStores()
28656 return ST->getOperand(0) != NewChain; in parallelizeChainedStores()
28667 for (const SDValue &Op : TF->ops()) in parallelizeChainedStores()
28692 SDValue BetterChain = FindBetterChain(St, St->getChain()); in findBetterNeighborChains()
28693 if (St->getChain() != BetterChain) { in findBetterNeighborChains()